From eda5bc26f44ee9a6f83dcf8c91f17296d7fc509d Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Mon, 12 Feb 2024 14:52:43 +0100 Subject: Move into version control --- src/armadillo/include/armadillo_bits/mul_gemv.hpp | 495 ++++++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 src/armadillo/include/armadillo_bits/mul_gemv.hpp (limited to 'src/armadillo/include/armadillo_bits/mul_gemv.hpp') diff --git a/src/armadillo/include/armadillo_bits/mul_gemv.hpp b/src/armadillo/include/armadillo_bits/mul_gemv.hpp new file mode 100644 index 0000000..2580e4a --- /dev/null +++ b/src/armadillo/include/armadillo_bits/mul_gemv.hpp @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright 2008-2016 Conrad Sanderson (http://conradsanderson.id.au) +// Copyright 2008-2016 National ICT Australia (NICTA) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------ + + +//! \addtogroup gemv +//! @{ + + + +//! for tiny square matrices, size <= 4x4 +template +class gemv_emul_tinysq + { + public: + + + template + struct pos + { + static constexpr uword n2 = (do_trans_A == false) ? (row + col*2) : (col + row*2); + static constexpr uword n3 = (do_trans_A == false) ? (row + col*3) : (col + row*3); + static constexpr uword n4 = (do_trans_A == false) ? (row + col*4) : (col + row*4); + }; + + + + template + arma_inline + static + void + assign(eT* y, const eT acc, const eT alpha, const eT beta) + { + if(use_beta == false) + { + y[i] = (use_alpha == false) ? acc : alpha*acc; + } + else + { + const eT tmp = y[i]; + + y[i] = beta*tmp + ( (use_alpha == false) ? acc : alpha*acc ); + } + } + + + + template + arma_cold + inline + static + void + apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) ) + { + arma_extra_debug_sigprint(); + + const eT* Am = A.memptr(); + + switch(A.n_rows) + { + case 1: + { + const eT acc = Am[0] * x[0]; + + assign(y, acc, alpha, beta); + } + break; + + + case 2: + { + const eT x0 = x[0]; + const eT x1 = x[1]; + + const eT acc0 = Am[pos<0,0>::n2]*x0 + Am[pos<0,1>::n2]*x1; + const eT acc1 = Am[pos<1,0>::n2]*x0 + Am[pos<1,1>::n2]*x1; + + assign(y, acc0, alpha, beta); + assign(y, acc1, alpha, beta); + } + break; + + + case 3: + { + const eT x0 = x[0]; + const eT x1 = x[1]; + const eT x2 = x[2]; + + const eT acc0 = Am[pos<0,0>::n3]*x0 + Am[pos<0,1>::n3]*x1 + Am[pos<0,2>::n3]*x2; + const eT acc1 = Am[pos<1,0>::n3]*x0 + Am[pos<1,1>::n3]*x1 + Am[pos<1,2>::n3]*x2; + const eT acc2 = Am[pos<2,0>::n3]*x0 + Am[pos<2,1>::n3]*x1 + Am[pos<2,2>::n3]*x2; + + assign(y, acc0, alpha, beta); + assign(y, acc1, alpha, beta); + assign(y, acc2, alpha, beta); + } + break; + + + case 4: + { + const eT x0 = x[0]; + const eT x1 = x[1]; + const eT x2 = x[2]; + const eT x3 = x[3]; + + const eT acc0 = Am[pos<0,0>::n4]*x0 + Am[pos<0,1>::n4]*x1 + Am[pos<0,2>::n4]*x2 + Am[pos<0,3>::n4]*x3; + const eT acc1 = Am[pos<1,0>::n4]*x0 + Am[pos<1,1>::n4]*x1 + Am[pos<1,2>::n4]*x2 + Am[pos<1,3>::n4]*x3; + const eT acc2 = Am[pos<2,0>::n4]*x0 + Am[pos<2,1>::n4]*x1 + Am[pos<2,2>::n4]*x2 + Am[pos<2,3>::n4]*x3; + const eT acc3 = Am[pos<3,0>::n4]*x0 + Am[pos<3,1>::n4]*x1 + Am[pos<3,2>::n4]*x2 + Am[pos<3,3>::n4]*x3; + + assign(y, acc0, alpha, beta); + assign(y, acc1, alpha, beta); + assign(y, acc2, alpha, beta); + assign(y, acc3, alpha, beta); + } + break; + + + default: + ; + } + } + + }; + + + +class gemv_emul_helper + { + public: + + template + arma_hot + inline + static + typename arma_not_cx::result + dot_row_col( const TA& A, const eT* x, const uword row, const uword N ) + { + eT acc1 = eT(0); + eT acc2 = eT(0); + + uword i,j; + for(i=0, j=1; j < N; i+=2, j+=2) + { + const eT xi = x[i]; + const eT xj = x[j]; + + acc1 += A.at(row,i) * xi; + acc2 += A.at(row,j) * xj; + } + + if(i < N) + { + acc1 += A.at(row,i) * x[i]; + } + + return (acc1 + acc2); + } + + + + template + arma_hot + inline + static + typename arma_cx_only::result + dot_row_col( const TA& A, const eT* x, const uword row, const uword N ) + { + typedef typename get_pod_type::result T; + + T val_real = T(0); + T val_imag = T(0); + + for(uword i=0; i& Ai = A.at(row,i); + const std::complex& xi = x[i]; + + const T a = Ai.real(); + const T b = Ai.imag(); + + const T c = xi.real(); + const T d = xi.imag(); + + val_real += (a*c) - (b*d); + val_imag += (a*d) + (b*c); + } + + return std::complex(val_real, val_imag); + } + + }; + + + +//! \brief +//! Partial emulation of BLAS gemv(). +//! 'y' is assumed to have been set to the correct size (ie. taking into account the transpose) + +template +class gemv_emul + { + public: + + template + arma_hot + inline + static + void + apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) ) + { + arma_extra_debug_sigprint(); + + const uword A_n_rows = A.n_rows; + const uword A_n_cols = A.n_cols; + + if(do_trans_A == false) + { + if(A_n_rows == 1) + { + const eT acc = op_dot::direct_dot_arma(A_n_cols, A.memptr(), x); + + if( (use_alpha == false) && (use_beta == false) ) { y[0] = acc; } + else if( (use_alpha == true ) && (use_beta == false) ) { y[0] = alpha*acc; } + else if( (use_alpha == false) && (use_beta == true ) ) { y[0] = acc + beta*y[0]; } + else if( (use_alpha == true ) && (use_beta == true ) ) { y[0] = alpha*acc + beta*y[0]; } + } + else + for(uword row=0; row < A_n_rows; ++row) + { + const eT acc = gemv_emul_helper::dot_row_col(A, x, row, A_n_cols); + + if( (use_alpha == false) && (use_beta == false) ) { y[row] = acc; } + else if( (use_alpha == true ) && (use_beta == false) ) { y[row] = alpha*acc; } + else if( (use_alpha == false) && (use_beta == true ) ) { y[row] = acc + beta*y[row]; } + else if( (use_alpha == true ) && (use_beta == true ) ) { y[row] = alpha*acc + beta*y[row]; } + } + } + else + if(do_trans_A == true) + { + if(is_cx::no) + { + for(uword col=0; col < A_n_cols; ++col) + { + // col is interpreted as row when storing the results in 'y' + + + // const eT* A_coldata = A.colptr(col); + // + // eT acc = eT(0); + // for(uword row=0; row < A_n_rows; ++row) + // { + // acc += A_coldata[row] * x[row]; + // } + + const eT acc = op_dot::direct_dot_arma(A_n_rows, A.colptr(col), x); + + if( (use_alpha == false) && (use_beta == false) ) { y[col] = acc; } + else if( (use_alpha == true ) && (use_beta == false) ) { y[col] = alpha*acc; } + else if( (use_alpha == false) && (use_beta == true ) ) { y[col] = acc + beta*y[col]; } + else if( (use_alpha == true ) && (use_beta == true ) ) { y[col] = alpha*acc + beta*y[col]; } + } + } + else + { + Mat AA; + + op_htrans::apply_mat_noalias(AA, A); + + gemv_emul::apply(y, AA, x, alpha, beta); + } + } + } + + }; + + + +//! \brief +//! Wrapper for BLAS gemv function, using template arguments to control the arguments passed to gemv. +//! 'y' is assumed to have been set to the correct size (ie. taking into account the transpose) + +template +class gemv + { + public: + + template + inline + static + void + apply_blas_type( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) ) + { + arma_extra_debug_sigprint(); + + if( (A.n_rows <= 4) && (A.n_rows == A.n_cols) && (is_cx::no) ) + { + gemv_emul_tinysq::apply(y, A, x, alpha, beta); + } + else + { + #if defined(ARMA_USE_ATLAS) + { + arma_debug_assert_atlas_size(A); + + if(is_cx::no) + { + // use gemm() instead of gemv() to work around a speed issue in Atlas 3.8.4 + + arma_extra_debug_print("atlas::cblas_gemm()"); + + atlas::cblas_gemm + ( + atlas_CblasColMajor, + (do_trans_A) ? ( is_cx::yes ? atlas_CblasConjTrans : atlas_CblasTrans ) : atlas_CblasNoTrans, + atlas_CblasNoTrans, + (do_trans_A) ? A.n_cols : A.n_rows, + 1, + (do_trans_A) ? A.n_rows : A.n_cols, + (use_alpha) ? alpha : eT(1), + A.mem, + A.n_rows, + x, + (do_trans_A) ? A.n_rows : A.n_cols, + (use_beta) ? beta : eT(0), + y, + (do_trans_A) ? A.n_cols : A.n_rows + ); + } + else + { + arma_extra_debug_print("atlas::cblas_gemv()"); + + atlas::cblas_gemv + ( + atlas_CblasColMajor, + (do_trans_A) ? ( is_cx::yes ? atlas_CblasConjTrans : atlas_CblasTrans ) : atlas_CblasNoTrans, + A.n_rows, + A.n_cols, + (use_alpha) ? alpha : eT(1), + A.mem, + A.n_rows, + x, + 1, + (use_beta) ? beta : eT(0), + y, + 1 + ); + } + } + #elif defined(ARMA_USE_BLAS) + { + arma_extra_debug_print("blas::gemv()"); + + arma_debug_assert_blas_size(A); + + const char trans_A = (do_trans_A) ? ( is_cx::yes ? 'C' : 'T' ) : 'N'; + const blas_int m = blas_int(A.n_rows); + const blas_int n = blas_int(A.n_cols); + const eT local_alpha = (use_alpha) ? alpha : eT(1); + //const blas_int lda = A.n_rows; + const blas_int inc = blas_int(1); + const eT local_beta = (use_beta) ? beta : eT(0); + + arma_extra_debug_print( arma_str::format("blas::gemv(): trans_A = %c") % trans_A ); + + blas::gemv + ( + &trans_A, + &m, + &n, + &local_alpha, + A.mem, + &m, // lda + x, + &inc, + &local_beta, + y, + &inc + ); + } + #else + { + gemv_emul::apply(y,A,x,alpha,beta); + } + #endif + } + + } + + + + template + arma_inline + static + void + apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) ) + { + gemv_emul::apply(y,A,x,alpha,beta); + } + + + + template + arma_inline + static + void + apply + ( + float* y, + const TA& A, + const float* x, + const float alpha = float(1), + const float beta = float(0) + ) + { + gemv::apply_blas_type(y,A,x,alpha,beta); + } + + + + template + arma_inline + static + void + apply + ( + double* y, + const TA& A, + const double* x, + const double alpha = double(1), + const double beta = double(0) + ) + { + gemv::apply_blas_type(y,A,x,alpha,beta); + } + + + + template + arma_inline + static + void + apply + ( + std::complex* y, + const TA& A, + const std::complex* x, + const std::complex alpha = std::complex(1), + const std::complex beta = std::complex(0) + ) + { + gemv::apply_blas_type(y,A,x,alpha,beta); + } + + + + template + arma_inline + static + void + apply + ( + std::complex* y, + const TA& A, + const std::complex* x, + const std::complex alpha = std::complex(1), + const std::complex beta = std::complex(0) + ) + { + gemv::apply_blas_type(y,A,x,alpha,beta); + } + + + + }; + + +//! @} -- cgit v1.2.1