diff options
Diffstat (limited to 'Eigen/src/Core/functors')
| -rw-r--r-- | Eigen/src/Core/functors/AssignmentFunctors.h | 13 | ||||
| -rw-r--r-- | Eigen/src/Core/functors/BinaryFunctors.h | 183 | ||||
| -rw-r--r-- | Eigen/src/Core/functors/NullaryFunctors.h | 61 | ||||
| -rw-r--r-- | Eigen/src/Core/functors/StlFunctors.h | 36 | ||||
| -rw-r--r-- | Eigen/src/Core/functors/UnaryFunctors.h | 1361 |
5 files changed, 917 insertions, 737 deletions
diff --git a/Eigen/src/Core/functors/AssignmentFunctors.h b/Eigen/src/Core/functors/AssignmentFunctors.h index 4153b877c..bf64ef4ed 100644 --- a/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Eigen/src/Core/functors/AssignmentFunctors.h @@ -144,7 +144,7 @@ template<typename Scalar> struct swap_assign_op { EIGEN_EMPTY_STRUCT_CTOR(swap_assign_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { -#ifdef __CUDACC__ +#ifdef EIGEN_GPUCC // FIXME is there some kind of cuda::swap? Scalar t=b; const_cast<Scalar&>(b)=a; a=t; #else @@ -157,7 +157,16 @@ template<typename Scalar> struct functor_traits<swap_assign_op<Scalar> > { enum { Cost = 3 * NumTraits<Scalar>::ReadCost, - PacketAccess = packet_traits<Scalar>::Vectorizable + PacketAccess = + #if defined(EIGEN_VECTORIZE_AVX) && EIGEN_COMP_CLANG && (EIGEN_COMP_CLANG<800 || defined(__apple_build_version__)) + // This is a partial workaround for a bug in clang generating bad code + // when mixing 256/512 bits loads and 128 bits moves. + // See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1684 + // https://bugs.llvm.org/show_bug.cgi?id=40815 + 0 + #else + packet_traits<Scalar>::Vectorizable + #endif }; }; diff --git a/Eigen/src/Core/functors/BinaryFunctors.h b/Eigen/src/Core/functors/BinaryFunctors.h index 96747bac7..63f09ab93 100644 --- a/Eigen/src/Core/functors/BinaryFunctors.h +++ b/Eigen/src/Core/functors/BinaryFunctors.h @@ -39,32 +39,26 @@ struct scalar_sum_op : binary_op_base<LhsScalar,RhsScalar> EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a + b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a + b; } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::padd(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux(a); } }; template<typename LhsScalar,typename RhsScalar> struct functor_traits<scalar_sum_op<LhsScalar,RhsScalar> > { enum { - Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, // rough estimate! + Cost = (int(NumTraits<LhsScalar>::AddCost) + int(NumTraits<RhsScalar>::AddCost)) / 2, // rough estimate! PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasAdd && packet_traits<RhsScalar>::HasAdd // TODO vectorize mixed sum }; }; -/** \internal - * \brief Template specialization to deprecate the summation of boolean expressions. - * This is required to solve Bug 426. - * \sa DenseBase::count(), DenseBase::any(), ArrayBase::cast(), MatrixBase::cast() - */ -template<> struct scalar_sum_op<bool,bool> : scalar_sum_op<int,int> { - EIGEN_DEPRECATED - scalar_sum_op() {} -}; + +template<> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalar_sum_op<bool,bool>::operator() (const bool& a, const bool& b) const { return a || b; } /** \internal @@ -83,23 +77,27 @@ struct scalar_product_op : binary_op_base<LhsScalar,RhsScalar> EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a * b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return a * b; } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return internal::pmul(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const { return internal::predux_mul(a); } }; template<typename LhsScalar,typename RhsScalar> struct functor_traits<scalar_product_op<LhsScalar,RhsScalar> > { enum { - Cost = (NumTraits<LhsScalar>::MulCost + NumTraits<RhsScalar>::MulCost)/2, // rough estimate! + Cost = (int(NumTraits<LhsScalar>::MulCost) + int(NumTraits<RhsScalar>::MulCost))/2, // rough estimate! PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasMul && packet_traits<RhsScalar>::HasMul // TODO vectorize mixed product }; }; +template<> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalar_product_op<bool,bool>::operator() (const bool& a, const bool& b) const { return a && b; } + + /** \internal * \brief Template functor to compute the conjugate product of two scalars * @@ -116,11 +114,11 @@ struct scalar_conj_product_op : binary_op_base<LhsScalar,RhsScalar> typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_conj_product_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_conj_product_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return conj_helper<LhsScalar,RhsScalar,Conj,false>().pmul(a,b); } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const { return conj_helper<Packet,Packet,Conj,false>().pmul(a,b); } }; template<typename LhsScalar,typename RhsScalar> @@ -136,21 +134,28 @@ struct functor_traits<scalar_conj_product_op<LhsScalar,RhsScalar> > { * * \sa class CwiseBinaryOp, MatrixBase::cwiseMin, class VectorwiseOp, MatrixBase::minCoeff() */ -template<typename LhsScalar,typename RhsScalar> +template<typename LhsScalar,typename RhsScalar, int NaNPropagation> struct scalar_min_op : binary_op_base<LhsScalar,RhsScalar> { typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_min_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_min_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return numext::mini(a, b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { + return internal::pmin<NaNPropagation>(a, b); + } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::pmin(a,b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const + { + return internal::pmin<NaNPropagation>(a,b); + } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const - { return internal::predux_min(a); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const + { + return internal::predux_min<NaNPropagation>(a); + } }; -template<typename LhsScalar,typename RhsScalar> -struct functor_traits<scalar_min_op<LhsScalar,RhsScalar> > { + +template<typename LhsScalar,typename RhsScalar, int NaNPropagation> +struct functor_traits<scalar_min_op<LhsScalar,RhsScalar, NaNPropagation> > { enum { Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, PacketAccess = internal::is_same<LhsScalar, RhsScalar>::value && packet_traits<LhsScalar>::HasMin @@ -162,21 +167,28 @@ struct functor_traits<scalar_min_op<LhsScalar,RhsScalar> > { * * \sa class CwiseBinaryOp, MatrixBase::cwiseMax, class VectorwiseOp, MatrixBase::maxCoeff() */ -template<typename LhsScalar,typename RhsScalar> -struct scalar_max_op : binary_op_base<LhsScalar,RhsScalar> +template<typename LhsScalar,typename RhsScalar, int NaNPropagation> +struct scalar_max_op : binary_op_base<LhsScalar,RhsScalar> { typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_max_op>::ReturnType result_type; EIGEN_EMPTY_STRUCT_CTOR(scalar_max_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const { return numext::maxi(a, b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const LhsScalar& a, const RhsScalar& b) const { + return internal::pmax<NaNPropagation>(a,b); + } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const - { return internal::pmax(a,b); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) const + { + return internal::pmax<NaNPropagation>(a,b); + } template<typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type predux(const Packet& a) const - { return internal::predux_max(a); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type predux(const Packet& a) const + { + return internal::predux_max<NaNPropagation>(a); + } }; -template<typename LhsScalar,typename RhsScalar> -struct functor_traits<scalar_max_op<LhsScalar,RhsScalar> > { + +template<typename LhsScalar,typename RhsScalar, int NaNPropagation> +struct functor_traits<scalar_max_op<LhsScalar,RhsScalar, NaNPropagation> > { enum { Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, PacketAccess = internal::is_same<LhsScalar, RhsScalar>::value && packet_traits<LhsScalar>::HasMax @@ -253,9 +265,8 @@ struct scalar_cmp_op<LhsScalar,RhsScalar, cmp_NEQ> : binary_op_base<LhsScalar,Rh EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const LhsScalar& a, const RhsScalar& b) const {return a!=b;} }; - /** \internal - * \brief Template functor to compute the hypot of two scalars + * \brief Template functor to compute the hypot of two \b positive \b and \b real scalars * * \sa MatrixBase::stableNorm(), class Redux */ @@ -263,22 +274,15 @@ template<typename Scalar> struct scalar_hypot_op<Scalar,Scalar> : binary_op_base<Scalar,Scalar> { EIGEN_EMPTY_STRUCT_CTOR(scalar_hypot_op) -// typedef typename NumTraits<Scalar>::Real result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& _x, const Scalar& _y) const + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar &x, const Scalar &y) const { - EIGEN_USING_STD_MATH(sqrt) - Scalar p, qp; - if(_x>_y) - { - p = _x; - qp = _y / p; - } - else - { - p = _y; - qp = _x / p; - } - return p * sqrt(Scalar(1) + qp*qp); + // This functor is used by hypotNorm only for which it is faster to first apply abs + // on all coefficients prior to reduction through hypot. + // This way we avoid calling abs on positive and real entries, and this also permits + // to seamlessly handle complexes. Otherwise we would have to handle both real and complexes + // through the same functor... + return internal::positive_real_hypot(x,y); } }; template<typename Scalar> @@ -294,6 +298,7 @@ struct functor_traits<scalar_hypot_op<Scalar,Scalar> > { /** \internal * \brief Template functor to compute the pow of two scalars + * See the specification of pow in https://en.cppreference.com/w/cpp/numeric/math/pow */ template<typename Scalar, typename Exponent> struct scalar_pow_op : binary_op_base<Scalar,Exponent> @@ -308,16 +313,31 @@ struct scalar_pow_op : binary_op_base<Scalar,Exponent> EIGEN_SCALAR_BINARY_OP_PLUGIN } #endif + EIGEN_DEVICE_FUNC inline result_type operator() (const Scalar& a, const Exponent& b) const { return numext::pow(a, b); } + + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + { + return generic_pow(a,b); + } }; + template<typename Scalar, typename Exponent> struct functor_traits<scalar_pow_op<Scalar,Exponent> > { - enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; + enum { + Cost = 5 * NumTraits<Scalar>::MulCost, + PacketAccess = (!NumTraits<Scalar>::IsComplex && !NumTraits<Scalar>::IsInteger && + packet_traits<Scalar>::HasExp && packet_traits<Scalar>::HasLog && + packet_traits<Scalar>::HasRound && packet_traits<Scalar>::HasCmp && + // Temporarly disable packet access for half/bfloat16 until + // accuracy is improved. + !is_same<Scalar, half>::value && !is_same<Scalar, bfloat16>::value + ) + }; }; - - //---------- non associative binary functors ---------- /** \internal @@ -344,7 +364,7 @@ struct scalar_difference_op : binary_op_base<LhsScalar,RhsScalar> template<typename LhsScalar,typename RhsScalar> struct functor_traits<scalar_difference_op<LhsScalar,RhsScalar> > { enum { - Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, + Cost = (int(NumTraits<LhsScalar>::AddCost) + int(NumTraits<RhsScalar>::AddCost)) / 2, PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasSub && packet_traits<RhsScalar>::HasSub }; }; @@ -389,11 +409,14 @@ struct functor_traits<scalar_quotient_op<LhsScalar,RhsScalar> > { struct scalar_boolean_and_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_and_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a && b; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + { return internal::pand(a,b); } }; template<> struct functor_traits<scalar_boolean_and_op> { enum { Cost = NumTraits<bool>::AddCost, - PacketAccess = false + PacketAccess = true }; }; @@ -405,11 +428,14 @@ template<> struct functor_traits<scalar_boolean_and_op> { struct scalar_boolean_or_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_or_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a || b; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + { return internal::por(a,b); } }; template<> struct functor_traits<scalar_boolean_or_op> { enum { Cost = NumTraits<bool>::AddCost, - PacketAccess = false + PacketAccess = true }; }; @@ -421,11 +447,44 @@ template<> struct functor_traits<scalar_boolean_or_op> { struct scalar_boolean_xor_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_xor_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a, const bool& b) const { return a ^ b; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + { return internal::pxor(a,b); } }; template<> struct functor_traits<scalar_boolean_xor_op> { enum { Cost = NumTraits<bool>::AddCost, - PacketAccess = false + PacketAccess = true + }; +}; + +/** \internal + * \brief Template functor to compute the absolute difference of two scalars + * + * \sa class CwiseBinaryOp, MatrixBase::absolute_difference + */ +template<typename LhsScalar,typename RhsScalar> +struct scalar_absolute_difference_op : binary_op_base<LhsScalar,RhsScalar> +{ + typedef typename ScalarBinaryOpTraits<LhsScalar,RhsScalar,scalar_absolute_difference_op>::ReturnType result_type; +#ifndef EIGEN_SCALAR_BINARY_OP_PLUGIN + EIGEN_EMPTY_STRUCT_CTOR(scalar_absolute_difference_op) +#else + scalar_absolute_difference_op() { + EIGEN_SCALAR_BINARY_OP_PLUGIN + } +#endif + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const LhsScalar& a, const RhsScalar& b) const + { return numext::absdiff(a,b); } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a, const Packet& b) const + { return internal::pabsdiff(a,b); } +}; +template<typename LhsScalar,typename RhsScalar> +struct functor_traits<scalar_absolute_difference_op<LhsScalar,RhsScalar> > { + enum { + Cost = (NumTraits<LhsScalar>::AddCost+NumTraits<RhsScalar>::AddCost)/2, + PacketAccess = is_same<LhsScalar,RhsScalar>::value && packet_traits<LhsScalar>::HasAbsDiff }; }; @@ -443,7 +502,7 @@ template<typename BinaryOp> struct bind1st_op : BinaryOp { typedef typename BinaryOp::second_argument_type second_argument_type; typedef typename BinaryOp::result_type result_type; - bind1st_op(const first_argument_type &val) : m_value(val) {} + EIGEN_DEVICE_FUNC explicit bind1st_op(const first_argument_type &val) : m_value(val) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const second_argument_type& b) const { return BinaryOp::operator()(m_value,b); } @@ -462,7 +521,7 @@ template<typename BinaryOp> struct bind2nd_op : BinaryOp { typedef typename BinaryOp::second_argument_type second_argument_type; typedef typename BinaryOp::result_type result_type; - bind2nd_op(const second_argument_type &val) : m_value(val) {} + EIGEN_DEVICE_FUNC explicit bind2nd_op(const second_argument_type &val) : m_value(val) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const first_argument_type& a) const { return BinaryOp::operator()(a,m_value); } diff --git a/Eigen/src/Core/functors/NullaryFunctors.h b/Eigen/src/Core/functors/NullaryFunctors.h index b03be0269..192f225dd 100644 --- a/Eigen/src/Core/functors/NullaryFunctors.h +++ b/Eigen/src/Core/functors/NullaryFunctors.h @@ -37,26 +37,27 @@ template<typename Scalar> struct functor_traits<scalar_identity_op<Scalar> > { enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = false, IsRepeatable = true }; }; -template <typename Scalar, typename Packet, bool IsInteger> struct linspaced_op_impl; +template <typename Scalar, bool IsInteger> struct linspaced_op_impl; -template <typename Scalar, typename Packet> -struct linspaced_op_impl<Scalar,Packet,/*IsInteger*/false> +template <typename Scalar> +struct linspaced_op_impl<Scalar,/*IsInteger*/false> { - linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : - m_low(low), m_high(high), m_size1(num_steps==1 ? 1 : num_steps-1), m_step(num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1)), + typedef typename NumTraits<Scalar>::Real RealScalar; + + EIGEN_DEVICE_FUNC linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : + m_low(low), m_high(high), m_size1(num_steps==1 ? 1 : num_steps-1), m_step(num_steps==1 ? Scalar() : Scalar((high-low)/RealScalar(num_steps-1))), m_flip(numext::abs(high)<numext::abs(low)) {} template<typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { - typedef typename NumTraits<Scalar>::Real RealScalar; if(m_flip) - return (i==0)? m_low : (m_high - RealScalar(m_size1-i)*m_step); + return (i==0)? m_low : Scalar(m_high - RealScalar(m_size1-i)*m_step); else - return (i==m_size1)? m_high : (m_low + RealScalar(i)*m_step); + return (i==m_size1)? m_high : Scalar(m_low + RealScalar(i)*m_step); } - template<typename IndexType> + template<typename Packet, typename IndexType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { // Principle: @@ -65,17 +66,17 @@ struct linspaced_op_impl<Scalar,Packet,/*IsInteger*/false> { Packet pi = plset<Packet>(Scalar(i-m_size1)); Packet res = padd(pset1<Packet>(m_high), pmul(pset1<Packet>(m_step), pi)); - if(i==0) - res = pinsertfirst(res, m_low); - return res; + if (EIGEN_PREDICT_TRUE(i != 0)) return res; + Packet mask = pcmp_lt(pset1<Packet>(0), plset<Packet>(0)); + return pselect<Packet>(mask, res, pset1<Packet>(m_low)); } else { Packet pi = plset<Packet>(Scalar(i)); Packet res = padd(pset1<Packet>(m_low), pmul(pset1<Packet>(m_step), pi)); - if(i==m_size1-unpacket_traits<Packet>::size+1) - res = pinsertlast(res, m_high); - return res; + if(EIGEN_PREDICT_TRUE(i != m_size1-unpacket_traits<Packet>::size+1)) return res; + Packet mask = pcmp_lt(plset<Packet>(0), pset1<Packet>(unpacket_traits<Packet>::size-1)); + return pselect<Packet>(mask, res, pset1<Packet>(m_high)); } } @@ -86,10 +87,10 @@ struct linspaced_op_impl<Scalar,Packet,/*IsInteger*/false> const bool m_flip; }; -template <typename Scalar, typename Packet> -struct linspaced_op_impl<Scalar,Packet,/*IsInteger*/true> +template <typename Scalar> +struct linspaced_op_impl<Scalar,/*IsInteger*/true> { - linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : + EIGEN_DEVICE_FUNC linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) : m_low(low), m_multiplier((high-low)/convert_index<Scalar>(num_steps<=1 ? 1 : num_steps-1)), m_divisor(convert_index<Scalar>((high>=low?num_steps:-num_steps)+(high-low))/((numext::abs(high-low)+1)==0?1:(numext::abs(high-low)+1))), @@ -115,8 +116,8 @@ struct linspaced_op_impl<Scalar,Packet,/*IsInteger*/true> // Forward declaration (we default to random access which does not really give // us a speed gain when using packet access but it allows to use the functor in // nested expressions). -template <typename Scalar, typename PacketType> struct linspaced_op; -template <typename Scalar, typename PacketType> struct functor_traits< linspaced_op<Scalar,PacketType> > +template <typename Scalar> struct linspaced_op; +template <typename Scalar> struct functor_traits< linspaced_op<Scalar> > { enum { @@ -126,9 +127,9 @@ template <typename Scalar, typename PacketType> struct functor_traits< linspaced IsRepeatable = true }; }; -template <typename Scalar, typename PacketType> struct linspaced_op +template <typename Scalar> struct linspaced_op { - linspaced_op(const Scalar& low, const Scalar& high, Index num_steps) + EIGEN_DEVICE_FUNC linspaced_op(const Scalar& low, const Scalar& high, Index num_steps) : impl((num_steps==1 ? high : low),high,num_steps) {} @@ -136,11 +137,11 @@ template <typename Scalar, typename PacketType> struct linspaced_op EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (IndexType i) const { return impl(i); } template<typename Packet,typename IndexType> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { return impl.packetOp(i); } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(IndexType i) const { return impl.template packetOp<Packet>(i); } // This proxy object handles the actual required temporaries and the different // implementations (integer vs. floating point). - const linspaced_op_impl<Scalar,PacketType,NumTraits<Scalar>::IsInteger> impl; + const linspaced_op_impl<Scalar,NumTraits<Scalar>::IsInteger> impl; }; // Linear access is automatically determined from the operator() prototypes available for the given functor. @@ -166,12 +167,12 @@ struct has_unary_operator<scalar_identity_op<Scalar>,IndexType> { enum { value = template<typename Scalar,typename IndexType> struct has_binary_operator<scalar_identity_op<Scalar>,IndexType> { enum { value = 1}; }; -template<typename Scalar, typename PacketType,typename IndexType> -struct has_nullary_operator<linspaced_op<Scalar,PacketType>,IndexType> { enum { value = 0}; }; -template<typename Scalar, typename PacketType,typename IndexType> -struct has_unary_operator<linspaced_op<Scalar,PacketType>,IndexType> { enum { value = 1}; }; -template<typename Scalar, typename PacketType,typename IndexType> -struct has_binary_operator<linspaced_op<Scalar,PacketType>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_nullary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 0}; }; +template<typename Scalar,typename IndexType> +struct has_unary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 1}; }; +template<typename Scalar,typename IndexType> +struct has_binary_operator<linspaced_op<Scalar>,IndexType> { enum { value = 0}; }; template<typename Scalar,typename IndexType> struct has_nullary_operator<scalar_random_op<Scalar>,IndexType> { enum { value = 1}; }; diff --git a/Eigen/src/Core/functors/StlFunctors.h b/Eigen/src/Core/functors/StlFunctors.h index 6df3fa501..4570c9b63 100644 --- a/Eigen/src/Core/functors/StlFunctors.h +++ b/Eigen/src/Core/functors/StlFunctors.h @@ -12,6 +12,28 @@ namespace Eigen { +// Portable replacements for certain functors. +namespace numext { + +template<typename T = void> +struct equal_to { + typedef bool result_type; + EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { + return lhs == rhs; + } +}; + +template<typename T = void> +struct not_equal_to { + typedef bool result_type; + EIGEN_DEVICE_FUNC bool operator()(const T& lhs, const T& rhs) const { + return lhs != rhs; + } +}; + +} + + namespace internal { // default functor traits for STL functors: @@ -69,10 +91,18 @@ struct functor_traits<std::equal_to<T> > { enum { Cost = 1, PacketAccess = false }; }; template<typename T> +struct functor_traits<numext::equal_to<T> > + : functor_traits<std::equal_to<T> > {}; + +template<typename T> struct functor_traits<std::not_equal_to<T> > { enum { Cost = 1, PacketAccess = false }; }; -#if (__cplusplus < 201103L) && (EIGEN_COMP_MSVC <= 1900) +template<typename T> +struct functor_traits<numext::not_equal_to<T> > + : functor_traits<std::not_equal_to<T> > {}; + +#if (EIGEN_COMP_CXXVER < 11) // std::binder* are deprecated since c++11 and will be removed in c++17 template<typename T> struct functor_traits<std::binder2nd<T> > @@ -83,13 +113,17 @@ struct functor_traits<std::binder1st<T> > { enum { Cost = functor_traits<T>::Cost, PacketAccess = false }; }; #endif +#if (EIGEN_COMP_CXXVER < 17) +// std::unary_negate is deprecated since c++17 and will be removed in c++20 template<typename T> struct functor_traits<std::unary_negate<T> > { enum { Cost = 1 + functor_traits<T>::Cost, PacketAccess = false }; }; +// std::binary_negate is deprecated since c++17 and will be removed in c++20 template<typename T> struct functor_traits<std::binary_negate<T> > { enum { Cost = 1 + functor_traits<T>::Cost, PacketAccess = false }; }; +#endif #ifdef EIGEN_STDEXT_SUPPORT diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index c396db393..16136d185 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h @@ -15,52 +15,40 @@ namespace Eigen { namespace internal { /** \internal - * \brief Template functor to compute the opposite of a scalar - * - * \sa class CwiseUnaryOp, MatrixBase::operator- - */ -template <typename Scalar> -struct scalar_opposite_op { + * \brief Template functor to compute the opposite of a scalar + * + * \sa class CwiseUnaryOp, MatrixBase::operator- + */ +template<typename Scalar> struct scalar_opposite_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_opposite_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar - operator()(const Scalar& a) const { - return -a; - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet - packetOp(const Packet& a) const { - return internal::pnegate(a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_opposite_op<Scalar> > { - enum { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return -a; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::pnegate(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_opposite_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasNegate - }; + PacketAccess = packet_traits<Scalar>::HasNegate }; }; /** \internal - * \brief Template functor to compute the absolute value of a scalar - * - * \sa class CwiseUnaryOp, Cwise::abs - */ -template <typename Scalar> -struct scalar_abs_op { + * \brief Template functor to compute the absolute value of a scalar + * + * \sa class CwiseUnaryOp, Cwise::abs + */ +template<typename Scalar> struct scalar_abs_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_abs_op) typedef typename NumTraits<Scalar>::Real result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type - operator()(const Scalar& a) const { - return numext::abs(a); - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet - packetOp(const Packet& a) const { - return internal::pabs(a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_abs_op<Scalar> > { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs(a); } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::pabs(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_abs_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasAbs @@ -68,372 +56,370 @@ struct functor_traits<scalar_abs_op<Scalar> > { }; /** \internal - * \brief Template functor to compute the score of a scalar, to chose a pivot - * - * \sa class CwiseUnaryOp - */ -template <typename Scalar> -struct scalar_score_coeff_op : scalar_abs_op<Scalar> { + * \brief Template functor to compute the score of a scalar, to chose a pivot + * + * \sa class CwiseUnaryOp + */ +template<typename Scalar> struct scalar_score_coeff_op : scalar_abs_op<Scalar> +{ typedef void Score_is_abs; }; -template <typename Scalar> -struct functor_traits<scalar_score_coeff_op<Scalar> > - : functor_traits<scalar_abs_op<Scalar> > {}; +template<typename Scalar> +struct functor_traits<scalar_score_coeff_op<Scalar> > : functor_traits<scalar_abs_op<Scalar> > {}; -/* Avoid recomputing abs when we know the score and they are the same. Not a - * true Eigen functor. */ -template <typename Scalar, typename = void> -struct abs_knowing_score { +/* Avoid recomputing abs when we know the score and they are the same. Not a true Eigen functor. */ +template<typename Scalar, typename=void> struct abs_knowing_score +{ EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits<Scalar>::Real result_type; - template <typename Score> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type - operator()(const Scalar& a, const Score&) const { - return numext::abs(a); - } + template<typename Score> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a, const Score&) const { return numext::abs(a); } }; -template <typename Scalar> -struct abs_knowing_score<Scalar, - typename scalar_score_coeff_op<Scalar>::Score_is_abs> { +template<typename Scalar> struct abs_knowing_score<Scalar, typename scalar_score_coeff_op<Scalar>::Score_is_abs> +{ EIGEN_EMPTY_STRUCT_CTOR(abs_knowing_score) typedef typename NumTraits<Scalar>::Real result_type; - template <typename Scal> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type - operator()(const Scal&, const result_type& a) const { - return a; - } + template<typename Scal> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scal&, const result_type& a) const { return a; } }; /** \internal - * \brief Template functor to compute the squared absolute value of a scalar - * - * \sa class CwiseUnaryOp, Cwise::abs2 - */ -template <typename Scalar> -struct scalar_abs2_op { + * \brief Template functor to compute the squared absolute value of a scalar + * + * \sa class CwiseUnaryOp, Cwise::abs2 + */ +template<typename Scalar> struct scalar_abs2_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_abs2_op) typedef typename NumTraits<Scalar>::Real result_type; EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const result_type operator()(const Scalar& a) const { - return numext::abs2(a); - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet - packetOp(const Packet& a) const { - return internal::pmul(a, a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_abs2_op<Scalar> > { - enum { - Cost = NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasAbs2 - }; -}; + EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::abs2(a); } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::pmul(a,a); } +}; +template<typename Scalar> +struct functor_traits<scalar_abs2_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasAbs2 }; }; /** \internal - * \brief Template functor to compute the conjugate of a complex value - * - * \sa class CwiseUnaryOp, MatrixBase::conjugate() - */ -template <typename Scalar> -struct scalar_conjugate_op { + * \brief Template functor to compute the conjugate of a complex value + * + * \sa class CwiseUnaryOp, MatrixBase::conjugate() + */ +template<typename Scalar> struct scalar_conjugate_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_conjugate_op) EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& a) const { - using numext::conj; - return conj(a); - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet - packetOp(const Packet& a) const { - return internal::pconj(a); - } + EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::conj(a); } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { return internal::pconj(a); } }; -template <typename Scalar> -struct functor_traits<scalar_conjugate_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_conjugate_op<Scalar> > +{ enum { - Cost = NumTraits<Scalar>::IsComplex ? NumTraits<Scalar>::AddCost : 0, + Cost = 0, + // Yes the cost is zero even for complexes because in most cases for which + // the cost is used, conjugation turns to be a no-op. Some examples: + // cost(a*conj(b)) == cost(a*b) + // cost(a+conj(b)) == cost(a+b) + // <etc. + // If we don't set it to zero, then: + // A.conjugate().lazyProduct(B.conjugate()) + // will bake its operands. We definitely don't want that! PacketAccess = packet_traits<Scalar>::HasConj }; }; /** \internal - * \brief Template functor to compute the phase angle of a complex - * - * \sa class CwiseUnaryOp, Cwise::arg - */ -template <typename Scalar> -struct scalar_arg_op { + * \brief Template functor to compute the phase angle of a complex + * + * \sa class CwiseUnaryOp, Cwise::arg + */ +template<typename Scalar> struct scalar_arg_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_arg_op) typedef typename NumTraits<Scalar>::Real result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type - operator()(const Scalar& a) const { - using numext::arg; - return arg(a); - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet - packetOp(const Packet& a) const { - return internal::parg(a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_arg_op<Scalar> > { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const result_type operator() (const Scalar& a) const { return numext::arg(a); } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::parg(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_arg_op<Scalar> > +{ enum { - Cost = NumTraits<Scalar>::IsComplex ? 5 * NumTraits<Scalar>::MulCost - : NumTraits<Scalar>::AddCost, + Cost = NumTraits<Scalar>::IsComplex ? 5 * NumTraits<Scalar>::MulCost : NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasArg }; }; /** \internal - * \brief Template functor to cast a scalar to another type - * - * \sa class CwiseUnaryOp, MatrixBase::cast() - */ -template <typename Scalar, typename NewType> + * \brief Template functor to cast a scalar to another type + * + * \sa class CwiseUnaryOp, MatrixBase::cast() + */ +template<typename Scalar, typename NewType> struct scalar_cast_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op) typedef NewType result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const NewType - operator()(const Scalar& a) const { - return cast<Scalar, NewType>(a); - } -}; -template <typename Scalar, typename NewType> -struct functor_traits<scalar_cast_op<Scalar, NewType> > { - enum { - Cost = is_same<Scalar, NewType>::value ? 0 : NumTraits<NewType>::AddCost, - PacketAccess = false - }; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const NewType operator() (const Scalar& a) const { return cast<Scalar, NewType>(a); } }; +template<typename Scalar, typename NewType> +struct functor_traits<scalar_cast_op<Scalar,NewType> > +{ enum { Cost = is_same<Scalar, NewType>::value ? 0 : NumTraits<NewType>::AddCost, PacketAccess = false }; }; /** \internal - * \brief Template functor to extract the real part of a complex - * - * \sa class CwiseUnaryOp, MatrixBase::real() - */ -template <typename Scalar> + * \brief Template functor to arithmetically shift a scalar right by a number of bits + * + * \sa class CwiseUnaryOp, MatrixBase::shift_right() + */ +template<typename Scalar, int N> +struct scalar_shift_right_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_right_op) + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const + { return a >> N; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::parithmetic_shift_right<N>(a); } +}; +template<typename Scalar, int N> +struct functor_traits<scalar_shift_right_op<Scalar,N> > +{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasShift }; }; + +/** \internal + * \brief Template functor to logically shift a scalar left by a number of bits + * + * \sa class CwiseUnaryOp, MatrixBase::shift_left() + */ +template<typename Scalar, int N> +struct scalar_shift_left_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_shift_left_op) + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const + { return a << N; } + template<typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const + { return internal::plogical_shift_left<N>(a); } +}; +template<typename Scalar, int N> +struct functor_traits<scalar_shift_left_op<Scalar,N> > +{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = packet_traits<Scalar>::HasShift }; }; + +/** \internal + * \brief Template functor to extract the real part of a complex + * + * \sa class CwiseUnaryOp, MatrixBase::real() + */ +template<typename Scalar> struct scalar_real_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_real_op) typedef typename NumTraits<Scalar>::Real result_type; EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE result_type operator()(const Scalar& a) const { - return numext::real(a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_real_op<Scalar> > { - enum { Cost = 0, PacketAccess = false }; + EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::real(a); } }; +template<typename Scalar> +struct functor_traits<scalar_real_op<Scalar> > +{ enum { Cost = 0, PacketAccess = false }; }; /** \internal - * \brief Template functor to extract the imaginary part of a complex - * - * \sa class CwiseUnaryOp, MatrixBase::imag() - */ -template <typename Scalar> + * \brief Template functor to extract the imaginary part of a complex + * + * \sa class CwiseUnaryOp, MatrixBase::imag() + */ +template<typename Scalar> struct scalar_imag_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_op) typedef typename NumTraits<Scalar>::Real result_type; EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE result_type operator()(const Scalar& a) const { - return numext::imag(a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_imag_op<Scalar> > { - enum { Cost = 0, PacketAccess = false }; + EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { return numext::imag(a); } }; +template<typename Scalar> +struct functor_traits<scalar_imag_op<Scalar> > +{ enum { Cost = 0, PacketAccess = false }; }; /** \internal - * \brief Template functor to extract the real part of a complex as a reference - * - * \sa class CwiseUnaryOp, MatrixBase::real() - */ -template <typename Scalar> + * \brief Template functor to extract the real part of a complex as a reference + * + * \sa class CwiseUnaryOp, MatrixBase::real() + */ +template<typename Scalar> struct scalar_real_ref_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_real_ref_op) typedef typename NumTraits<Scalar>::Real result_type; EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE result_type& operator()(const Scalar& a) const { - return numext::real_ref(*const_cast<Scalar*>(&a)); - } -}; -template <typename Scalar> -struct functor_traits<scalar_real_ref_op<Scalar> > { - enum { Cost = 0, PacketAccess = false }; + EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::real_ref(*const_cast<Scalar*>(&a)); } }; +template<typename Scalar> +struct functor_traits<scalar_real_ref_op<Scalar> > +{ enum { Cost = 0, PacketAccess = false }; }; /** \internal - * \brief Template functor to extract the imaginary part of a complex as a - * reference - * - * \sa class CwiseUnaryOp, MatrixBase::imag() - */ -template <typename Scalar> + * \brief Template functor to extract the imaginary part of a complex as a reference + * + * \sa class CwiseUnaryOp, MatrixBase::imag() + */ +template<typename Scalar> struct scalar_imag_ref_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_imag_ref_op) typedef typename NumTraits<Scalar>::Real result_type; EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE result_type& operator()(const Scalar& a) const { - return numext::imag_ref(*const_cast<Scalar*>(&a)); - } -}; -template <typename Scalar> -struct functor_traits<scalar_imag_ref_op<Scalar> > { - enum { Cost = 0, PacketAccess = false }; + EIGEN_STRONG_INLINE result_type& operator() (const Scalar& a) const { return numext::imag_ref(*const_cast<Scalar*>(&a)); } }; +template<typename Scalar> +struct functor_traits<scalar_imag_ref_op<Scalar> > +{ enum { Cost = 0, PacketAccess = false }; }; /** \internal - * - * \brief Template functor to compute the exponential of a scalar - * - * \sa class CwiseUnaryOp, Cwise::exp() - */ -template <typename Scalar> -struct scalar_exp_op { + * + * \brief Template functor to compute the exponential of a scalar + * + * \sa class CwiseUnaryOp, Cwise::exp() + */ +template<typename Scalar> struct scalar_exp_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_exp_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::exp(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::exp(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pexp(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexp(a); } }; template <typename Scalar> struct functor_traits<scalar_exp_op<Scalar> > { enum { PacketAccess = packet_traits<Scalar>::HasExp, - // The following numbers are based on the AVX implementation. + // The following numbers are based on the AVX implementation. #ifdef EIGEN_VECTORIZE_FMA // Haswell can issue 2 add/mul/madd per cycle. Cost = - (sizeof(Scalar) == 4 - // float: 8 pmadd, 4 pmul, 2 padd/psub, 6 other - ? (8 * NumTraits<Scalar>::AddCost + 6 * NumTraits<Scalar>::MulCost) - // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other - : (14 * NumTraits<Scalar>::AddCost + - 6 * NumTraits<Scalar>::MulCost + - scalar_div_cost<Scalar, packet_traits<Scalar>::HasDiv>::value)) + (sizeof(Scalar) == 4 + // float: 8 pmadd, 4 pmul, 2 padd/psub, 6 other + ? (8 * NumTraits<Scalar>::AddCost + 6 * NumTraits<Scalar>::MulCost) + // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other + : (14 * NumTraits<Scalar>::AddCost + + 6 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value)) #else Cost = - (sizeof(Scalar) == 4 - // float: 7 pmadd, 6 pmul, 4 padd/psub, 10 other - ? (21 * NumTraits<Scalar>::AddCost + - 13 * NumTraits<Scalar>::MulCost) - // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other - : (23 * NumTraits<Scalar>::AddCost + - 12 * NumTraits<Scalar>::MulCost + - scalar_div_cost<Scalar, packet_traits<Scalar>::HasDiv>::value)) + (sizeof(Scalar) == 4 + // float: 7 pmadd, 6 pmul, 4 padd/psub, 10 other + ? (21 * NumTraits<Scalar>::AddCost + 13 * NumTraits<Scalar>::MulCost) + // double: 7 pmadd, 5 pmul, 3 padd/psub, 1 div, 13 other + : (23 * NumTraits<Scalar>::AddCost + + 12 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value)) #endif }; }; /** \internal - * - * \brief Template functor to compute the logarithm of a scalar - * - * \sa class CwiseUnaryOp, ArrayBase::log() - */ + * + * \brief Template functor to compute the exponential of a scalar - 1. + * + * \sa class CwiseUnaryOp, ArrayBase::expm1() + */ +template<typename Scalar> struct scalar_expm1_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_expm1_op) + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::expm1(a); } + template <typename Packet> + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pexpm1(a); } +}; template <typename Scalar> -struct scalar_log_op { +struct functor_traits<scalar_expm1_op<Scalar> > { + enum { + PacketAccess = packet_traits<Scalar>::HasExpm1, + Cost = functor_traits<scalar_exp_op<Scalar> >::Cost // TODO measure cost of expm1 + }; +}; + +/** \internal + * + * \brief Template functor to compute the logarithm of a scalar + * + * \sa class CwiseUnaryOp, ArrayBase::log() + */ +template<typename Scalar> struct scalar_log_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::log(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::plog(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog(a); } }; template <typename Scalar> struct functor_traits<scalar_log_op<Scalar> > { enum { PacketAccess = packet_traits<Scalar>::HasLog, - Cost = (PacketAccess - // The following numbers are based on the AVX implementation. + Cost = + (PacketAccess + // The following numbers are based on the AVX implementation. #ifdef EIGEN_VECTORIZE_FMA - // 8 pmadd, 6 pmul, 8 padd/psub, 16 other, can issue 2 - // add/mul/madd per cycle. - ? (20 * NumTraits<Scalar>::AddCost + - 7 * NumTraits<Scalar>::MulCost) + // 8 pmadd, 6 pmul, 8 padd/psub, 16 other, can issue 2 add/mul/madd per cycle. + ? (20 * NumTraits<Scalar>::AddCost + 7 * NumTraits<Scalar>::MulCost) #else - // 8 pmadd, 6 pmul, 8 padd/psub, 20 other - ? (36 * NumTraits<Scalar>::AddCost + - 14 * NumTraits<Scalar>::MulCost) + // 8 pmadd, 6 pmul, 8 padd/psub, 20 other + ? (36 * NumTraits<Scalar>::AddCost + 14 * NumTraits<Scalar>::MulCost) #endif - // Measured cost of std::log. - : sizeof(Scalar) == 4 ? 40 : 85) + // Measured cost of std::log. + : sizeof(Scalar)==4 ? 40 : 85) }; }; /** \internal - * - * \brief Template functor to compute the logarithm of 1 plus a scalar value - * - * \sa class CwiseUnaryOp, ArrayBase::log1p() - */ -template <typename Scalar> -struct scalar_log1p_op { + * + * \brief Template functor to compute the logarithm of 1 plus a scalar value + * + * \sa class CwiseUnaryOp, ArrayBase::log1p() + */ +template<typename Scalar> struct scalar_log1p_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log1p_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::log1p(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::log1p(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::plog1p(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog1p(a); } }; template <typename Scalar> struct functor_traits<scalar_log1p_op<Scalar> > { enum { PacketAccess = packet_traits<Scalar>::HasLog1p, - Cost = functor_traits<scalar_log_op<Scalar> >::Cost // TODO measure cost of - // log1p + Cost = functor_traits<scalar_log_op<Scalar> >::Cost // TODO measure cost of log1p }; }; /** \internal - * - * \brief Template functor to compute the base-10 logarithm of a scalar - * - * \sa class CwiseUnaryOp, Cwise::log10() - */ -template <typename Scalar> -struct scalar_log10_op { + * + * \brief Template functor to compute the base-10 logarithm of a scalar + * + * \sa class CwiseUnaryOp, Cwise::log10() + */ +template<typename Scalar> struct scalar_log10_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_log10_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - EIGEN_USING_STD_MATH(log10) return log10(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { EIGEN_USING_STD(log10) return log10(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::plog10(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog10(a); } }; -template <typename Scalar> -struct functor_traits<scalar_log10_op<Scalar> > { - enum { - Cost = 5 * NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasLog10 - }; +template<typename Scalar> +struct functor_traits<scalar_log10_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasLog10 }; }; + +/** \internal + * + * \brief Template functor to compute the base-2 logarithm of a scalar + * + * \sa class CwiseUnaryOp, Cwise::log2() + */ +template<typename Scalar> struct scalar_log2_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_log2_op) + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return Scalar(EIGEN_LOG2E) * numext::log(a); } + template <typename Packet> + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::plog2(a); } }; +template<typename Scalar> +struct functor_traits<scalar_log2_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasLog }; }; /** \internal - * \brief Template functor to compute the square root of a scalar - * \sa class CwiseUnaryOp, Cwise::sqrt() - */ -template <typename Scalar> -struct scalar_sqrt_op { + * \brief Template functor to compute the square root of a scalar + * \sa class CwiseUnaryOp, Cwise::sqrt() + */ +template<typename Scalar> struct scalar_sqrt_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::sqrt(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sqrt(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::psqrt(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psqrt(a); } }; template <typename Scalar> struct functor_traits<scalar_sqrt_op<Scalar> > { @@ -452,47 +438,50 @@ struct functor_traits<scalar_sqrt_op<Scalar> > { }; }; +// Boolean specialization to eliminate -Wimplicit-conversion-floating-point-to-bool warnings. +template<> struct scalar_sqrt_op<bool> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_op) + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } + template <typename Packet> + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return a; } +}; +template <> +struct functor_traits<scalar_sqrt_op<bool> > { + enum { Cost = 1, PacketAccess = packet_traits<bool>::Vectorizable }; +}; + /** \internal - * \brief Template functor to compute the reciprocal square root of a scalar - * \sa class CwiseUnaryOp, Cwise::rsqrt() - */ -template <typename Scalar> -struct scalar_rsqrt_op { + * \brief Template functor to compute the reciprocal square root of a scalar + * \sa class CwiseUnaryOp, Cwise::rsqrt() + */ +template<typename Scalar> struct scalar_rsqrt_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_rsqrt_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return Scalar(1) / numext::sqrt(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::rsqrt(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::prsqrt(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::prsqrt(a); } }; -template <typename Scalar> -struct functor_traits<scalar_rsqrt_op<Scalar> > { - enum { +template<typename Scalar> +struct functor_traits<scalar_rsqrt_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasRsqrt }; }; /** \internal - * \brief Template functor to compute the cosine of a scalar - * \sa class CwiseUnaryOp, ArrayBase::cos() - */ -template <typename Scalar> -struct scalar_cos_op { + * \brief Template functor to compute the cosine of a scalar + * \sa class CwiseUnaryOp, ArrayBase::cos() + */ +template<typename Scalar> struct scalar_cos_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cos_op) - EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a) const { - return numext::cos(a); - } + EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return numext::cos(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pcos(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcos(a); } }; -template <typename Scalar> -struct functor_traits<scalar_cos_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_cos_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasCos @@ -500,45 +489,38 @@ struct functor_traits<scalar_cos_op<Scalar> > { }; /** \internal - * \brief Template functor to compute the sine of a scalar - * \sa class CwiseUnaryOp, ArrayBase::sin() - */ -template <typename Scalar> -struct scalar_sin_op { + * \brief Template functor to compute the sine of a scalar + * \sa class CwiseUnaryOp, ArrayBase::sin() + */ +template<typename Scalar> struct scalar_sin_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sin_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::sin(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sin(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::psin(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psin(a); } }; -template <typename Scalar> -struct functor_traits<scalar_sin_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_sin_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasSin }; }; + /** \internal - * \brief Template functor to compute the tan of a scalar - * \sa class CwiseUnaryOp, ArrayBase::tan() - */ -template <typename Scalar> -struct scalar_tan_op { + * \brief Template functor to compute the tan of a scalar + * \sa class CwiseUnaryOp, ArrayBase::tan() + */ +template<typename Scalar> struct scalar_tan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_tan_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::tan(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::tan(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::ptan(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::ptan(a); } }; -template <typename Scalar> -struct functor_traits<scalar_tan_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_tan_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasTan @@ -546,22 +528,18 @@ struct functor_traits<scalar_tan_op<Scalar> > { }; /** \internal - * \brief Template functor to compute the arc cosine of a scalar - * \sa class CwiseUnaryOp, ArrayBase::acos() - */ -template <typename Scalar> -struct scalar_acos_op { + * \brief Template functor to compute the arc cosine of a scalar + * \sa class CwiseUnaryOp, ArrayBase::acos() + */ +template<typename Scalar> struct scalar_acos_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_acos_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::acos(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::acos(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pacos(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pacos(a); } }; -template <typename Scalar> -struct functor_traits<scalar_acos_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_acos_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasACos @@ -569,45 +547,38 @@ struct functor_traits<scalar_acos_op<Scalar> > { }; /** \internal - * \brief Template functor to compute the arc sine of a scalar - * \sa class CwiseUnaryOp, ArrayBase::asin() - */ -template <typename Scalar> -struct scalar_asin_op { + * \brief Template functor to compute the arc sine of a scalar + * \sa class CwiseUnaryOp, ArrayBase::asin() + */ +template<typename Scalar> struct scalar_asin_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_asin_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::asin(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::asin(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pasin(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pasin(a); } }; -template <typename Scalar> -struct functor_traits<scalar_asin_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_asin_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasASin }; }; + /** \internal - * \brief Template functor to compute the atan of a scalar - * \sa class CwiseUnaryOp, ArrayBase::atan() - */ -template <typename Scalar> -struct scalar_atan_op { + * \brief Template functor to compute the atan of a scalar + * \sa class CwiseUnaryOp, ArrayBase::atan() + */ +template<typename Scalar> struct scalar_atan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_atan_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::atan(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::atan(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::patan(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::patan(a); } }; -template <typename Scalar> -struct functor_traits<scalar_atan_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_atan_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasATan @@ -615,229 +586,279 @@ struct functor_traits<scalar_atan_op<Scalar> > { }; /** \internal - * \brief Template functor to compute the tanh of a scalar - * \sa class CwiseUnaryOp, ArrayBase::tanh() - */ + * \brief Template functor to compute the tanh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::tanh() + */ template <typename Scalar> struct scalar_tanh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::tanh(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::tanh(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { - return ptanh(x); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& x) const { return ptanh(x); } }; template <typename Scalar> struct functor_traits<scalar_tanh_op<Scalar> > { enum { PacketAccess = packet_traits<Scalar>::HasTanh, - Cost = - ((EIGEN_FAST_MATH && is_same<Scalar, float>::value) + Cost = ( (EIGEN_FAST_MATH && is_same<Scalar,float>::value) // The following numbers are based on the AVX implementation, #ifdef EIGEN_VECTORIZE_FMA - // Haswell can issue 2 add/mul/madd per cycle. - // 9 pmadd, 2 pmul, 1 div, 2 other - ? (2 * NumTraits<Scalar>::AddCost + - 6 * NumTraits<Scalar>::MulCost + - scalar_div_cost<Scalar, packet_traits<Scalar>::HasDiv>::value) + // Haswell can issue 2 add/mul/madd per cycle. + // 9 pmadd, 2 pmul, 1 div, 2 other + ? (2 * NumTraits<Scalar>::AddCost + + 6 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value) #else - ? (11 * NumTraits<Scalar>::AddCost + - 11 * NumTraits<Scalar>::MulCost + - scalar_div_cost<Scalar, packet_traits<Scalar>::HasDiv>::value) + ? (11 * NumTraits<Scalar>::AddCost + + 11 * NumTraits<Scalar>::MulCost + + scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value) #endif - // This number assumes a naive implementation of tanh - : (6 * NumTraits<Scalar>::AddCost + - 3 * NumTraits<Scalar>::MulCost + - 2 * scalar_div_cost<Scalar, - packet_traits<Scalar>::HasDiv>::value + - functor_traits<scalar_exp_op<Scalar> >::Cost)) + // This number assumes a naive implementation of tanh + : (6 * NumTraits<Scalar>::AddCost + + 3 * NumTraits<Scalar>::MulCost + + 2 * scalar_div_cost<Scalar,packet_traits<Scalar>::HasDiv>::value + + functor_traits<scalar_exp_op<Scalar> >::Cost)) }; }; +#if EIGEN_HAS_CXX11_MATH /** \internal - * \brief Template functor to compute the sinh of a scalar - * \sa class CwiseUnaryOp, ArrayBase::sinh() - */ + * \brief Template functor to compute the atanh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::atanh() + */ +template <typename Scalar> +struct scalar_atanh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::atanh(a); } +}; + template <typename Scalar> -struct scalar_sinh_op { +struct functor_traits<scalar_atanh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + +/** \internal + * \brief Template functor to compute the sinh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::sinh() + */ +template<typename Scalar> struct scalar_sinh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_sinh_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::sinh(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::sinh(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::psinh(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psinh(a); } }; -template <typename Scalar> -struct functor_traits<scalar_sinh_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_sinh_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasSinh }; }; +#if EIGEN_HAS_CXX11_MATH /** \internal - * \brief Template functor to compute the cosh of a scalar - * \sa class CwiseUnaryOp, ArrayBase::cosh() - */ + * \brief Template functor to compute the asinh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::asinh() + */ +template <typename Scalar> +struct scalar_asinh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::asinh(a); } +}; + template <typename Scalar> -struct scalar_cosh_op { +struct functor_traits<scalar_asinh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + +/** \internal + * \brief Template functor to compute the cosh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::cosh() + */ +template<typename Scalar> struct scalar_cosh_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cosh_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return numext::cosh(a); - } + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const { return numext::cosh(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pcosh(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pcosh(a); } }; -template <typename Scalar> -struct functor_traits<scalar_cosh_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_cosh_op<Scalar> > +{ enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasCosh }; }; +#if EIGEN_HAS_CXX11_MATH /** \internal - * \brief Template functor to compute the inverse of a scalar - * \sa class CwiseUnaryOp, Cwise::inverse() - */ + * \brief Template functor to compute the acosh of a scalar + * \sa class CwiseUnaryOp, ArrayBase::acosh() + */ template <typename Scalar> +struct scalar_acosh_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op) + EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { return numext::acosh(a); } +}; + +template <typename Scalar> +struct functor_traits<scalar_acosh_op<Scalar> > { + enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false }; +}; +#endif + +/** \internal + * \brief Template functor to compute the inverse of a scalar + * \sa class CwiseUnaryOp, Cwise::inverse() + */ +template<typename Scalar> struct scalar_inverse_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_inverse_op) - EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a) const { - return Scalar(1) / a; - } - template <typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { - return internal::pdiv(pset1<Packet>(Scalar(1)), a); - } + EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return Scalar(1)/a; } + template<typename Packet> + EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const + { return internal::pdiv(pset1<Packet>(Scalar(1)),a); } }; template <typename Scalar> struct functor_traits<scalar_inverse_op<Scalar> > { enum { - Cost = NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasDiv + PacketAccess = packet_traits<Scalar>::HasDiv, + Cost = scalar_div_cost<Scalar, PacketAccess>::value }; }; /** \internal - * \brief Template functor to compute the square of a scalar - * \sa class CwiseUnaryOp, Cwise::square() - */ -template <typename Scalar> + * \brief Template functor to compute the square of a scalar + * \sa class CwiseUnaryOp, Cwise::square() + */ +template<typename Scalar> struct scalar_square_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) - EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a) const { - return a * a; - } - template <typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { - return internal::pmul(a, a); - } -}; -template <typename Scalar> -struct functor_traits<scalar_square_op<Scalar> > { - enum { - Cost = NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasMul - }; -}; + EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a; } + template<typename Packet> + EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const + { return internal::pmul(a,a); } +}; +template<typename Scalar> +struct functor_traits<scalar_square_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasMul }; }; + +// Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. +template<> +struct scalar_square_op<bool> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_square_op) + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } + template<typename Packet> + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const + { return a; } +}; +template<> +struct functor_traits<scalar_square_op<bool> > +{ enum { Cost = 0, PacketAccess = packet_traits<bool>::Vectorizable }; }; /** \internal - * \brief Template functor to compute the cube of a scalar - * \sa class CwiseUnaryOp, Cwise::cube() - */ -template <typename Scalar> + * \brief Template functor to compute the cube of a scalar + * \sa class CwiseUnaryOp, Cwise::cube() + */ +template<typename Scalar> struct scalar_cube_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) - EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a) const { - return a * a * a; - } + EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a) const { return a*a*a; } + template<typename Packet> + EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const + { return internal::pmul(a,pmul(a,a)); } +}; +template<typename Scalar> +struct functor_traits<scalar_cube_op<Scalar> > +{ enum { Cost = 2*NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasMul }; }; + +// Boolean specialization to avoid -Wint-in-bool-context warnings on GCC. +template<> +struct scalar_cube_op<bool> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_cube_op) + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline bool operator() (const bool& a) const { return a; } + template<typename Packet> + EIGEN_DEPRECATED EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const + { return a; } +}; +template<> +struct functor_traits<scalar_cube_op<bool> > +{ enum { Cost = 0, PacketAccess = packet_traits<bool>::Vectorizable }; }; + +/** \internal + * \brief Template functor to compute the rounded value of a scalar + * \sa class CwiseUnaryOp, ArrayBase::round() + */ +template<typename Scalar> struct scalar_round_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_round_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::round(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline const Packet packetOp(const Packet& a) const { - return internal::pmul(a, pmul(a, a)); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pround(a); } }; -template <typename Scalar> -struct functor_traits<scalar_cube_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_round_op<Scalar> > +{ enum { - Cost = 2 * NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasMul + Cost = NumTraits<Scalar>::MulCost, + PacketAccess = packet_traits<Scalar>::HasRound }; }; /** \internal - * \brief Template functor to compute the rounded value of a scalar - * \sa class CwiseUnaryOp, ArrayBase::round() - */ -template <typename Scalar> -struct scalar_round_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_round_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar - operator()(const Scalar& a) const { - return numext::round(a); - } + * \brief Template functor to compute the floor of a scalar + * \sa class CwiseUnaryOp, ArrayBase::floor() + */ +template<typename Scalar> struct scalar_floor_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_floor_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::floor(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pround(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pfloor(a); } }; -template <typename Scalar> -struct functor_traits<scalar_round_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_floor_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasRound + PacketAccess = packet_traits<Scalar>::HasFloor }; }; /** \internal - * \brief Template functor to compute the floor of a scalar - * \sa class CwiseUnaryOp, ArrayBase::floor() - */ -template <typename Scalar> -struct scalar_floor_op { - EIGEN_EMPTY_STRUCT_CTOR(scalar_floor_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar - operator()(const Scalar& a) const { - return numext::floor(a); - } + * \brief Template functor to compute the rounded (with current rounding mode) value of a scalar + * \sa class CwiseUnaryOp, ArrayBase::rint() + */ +template<typename Scalar> struct scalar_rint_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_rint_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::rint(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pfloor(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::print(a); } }; -template <typename Scalar> -struct functor_traits<scalar_floor_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_rint_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasFloor + PacketAccess = packet_traits<Scalar>::HasRint }; }; /** \internal - * \brief Template functor to compute the ceil of a scalar - * \sa class CwiseUnaryOp, ArrayBase::ceil() - */ -template <typename Scalar> -struct scalar_ceil_op { + * \brief Template functor to compute the ceil of a scalar + * \sa class CwiseUnaryOp, ArrayBase::ceil() + */ +template<typename Scalar> struct scalar_ceil_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_ceil_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar - operator()(const Scalar& a) const { - return numext::ceil(a); - } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::ceil(a); } template <typename Packet> - EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { - return internal::pceil(a); - } + EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pceil(a); } }; -template <typename Scalar> -struct functor_traits<scalar_ceil_op<Scalar> > { +template<typename Scalar> +struct functor_traits<scalar_ceil_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = packet_traits<Scalar>::HasCeil @@ -845,184 +866,216 @@ struct functor_traits<scalar_ceil_op<Scalar> > { }; /** \internal - * \brief Template functor to compute whether a scalar is NaN - * \sa class CwiseUnaryOp, ArrayBase::isnan() - */ -template <typename Scalar> -struct scalar_isnan_op { + * \brief Template functor to compute whether a scalar is NaN + * \sa class CwiseUnaryOp, ArrayBase::isnan() + */ +template<typename Scalar> struct scalar_isnan_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isnan_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type - operator()(const Scalar& a) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(SYCL_DEVICE_ONLY) + return numext::isnan(a); +#else return (numext::isnan)(a); +#endif } }; -template <typename Scalar> -struct functor_traits<scalar_isnan_op<Scalar> > { - enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = false }; +template<typename Scalar> +struct functor_traits<scalar_isnan_op<Scalar> > +{ + enum { + Cost = NumTraits<Scalar>::MulCost, + PacketAccess = false + }; }; /** \internal - * \brief Template functor to check whether a scalar is +/-inf - * \sa class CwiseUnaryOp, ArrayBase::isinf() - */ -template <typename Scalar> -struct scalar_isinf_op { + * \brief Template functor to check whether a scalar is +/-inf + * \sa class CwiseUnaryOp, ArrayBase::isinf() + */ +template<typename Scalar> struct scalar_isinf_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isinf_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type - operator()(const Scalar& a) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(SYCL_DEVICE_ONLY) + return numext::isinf(a); +#else return (numext::isinf)(a); +#endif } }; -template <typename Scalar> -struct functor_traits<scalar_isinf_op<Scalar> > { - enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = false }; +template<typename Scalar> +struct functor_traits<scalar_isinf_op<Scalar> > +{ + enum { + Cost = NumTraits<Scalar>::MulCost, + PacketAccess = false + }; }; /** \internal - * \brief Template functor to check whether a scalar has a finite value - * \sa class CwiseUnaryOp, ArrayBase::isfinite() - */ -template <typename Scalar> -struct scalar_isfinite_op { + * \brief Template functor to check whether a scalar has a finite value + * \sa class CwiseUnaryOp, ArrayBase::isfinite() + */ +template<typename Scalar> struct scalar_isfinite_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_isfinite_op) typedef bool result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type - operator()(const Scalar& a) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type operator() (const Scalar& a) const { +#if defined(SYCL_DEVICE_ONLY) + return numext::isfinite(a); +#else return (numext::isfinite)(a); +#endif } }; -template <typename Scalar> -struct functor_traits<scalar_isfinite_op<Scalar> > { - enum { Cost = NumTraits<Scalar>::MulCost, PacketAccess = false }; +template<typename Scalar> +struct functor_traits<scalar_isfinite_op<Scalar> > +{ + enum { + Cost = NumTraits<Scalar>::MulCost, + PacketAccess = false + }; }; /** \internal - * \brief Template functor to compute the logical not of a boolean - * - * \sa class CwiseUnaryOp, ArrayBase::operator! - */ -template <typename Scalar> -struct scalar_boolean_not_op { + * \brief Template functor to compute the logical not of a boolean + * + * \sa class CwiseUnaryOp, ArrayBase::operator! + */ +template<typename Scalar> struct scalar_boolean_not_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_boolean_not_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const bool& a) const { - return !a; - } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator() (const bool& a) const { return !a; } }; -template <typename Scalar> +template<typename Scalar> struct functor_traits<scalar_boolean_not_op<Scalar> > { - enum { Cost = NumTraits<bool>::AddCost, PacketAccess = false }; + enum { + Cost = NumTraits<bool>::AddCost, + PacketAccess = false + }; }; /** \internal - * \brief Template functor to compute the signum of a scalar - * \sa class CwiseUnaryOp, Cwise::sign() - */ -template <typename Scalar, bool iscpx = (NumTraits<Scalar>::IsComplex != 0)> -struct scalar_sign_op; -template <typename Scalar> -struct scalar_sign_op<Scalar, false> { + * \brief Template functor to compute the signum of a scalar + * \sa class CwiseUnaryOp, Cwise::sign() + */ +template<typename Scalar,bool is_complex=(NumTraits<Scalar>::IsComplex!=0), bool is_integer=(NumTraits<Scalar>::IsInteger!=0) > struct scalar_sign_op; +template<typename Scalar> +struct scalar_sign_op<Scalar, false, true> { EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - return Scalar((a > Scalar(0)) - (a < Scalar(0))); + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const + { + return Scalar( (a>Scalar(0)) - (a<Scalar(0)) ); } - // TODO - // template <typename Packet> - // EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return - // internal::psign(a); } + //TODO + //template <typename Packet> + //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; -template <typename Scalar> -struct scalar_sign_op<Scalar, true> { + +template<typename Scalar> +struct scalar_sign_op<Scalar, false, false> { EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) - EIGEN_DEVICE_FUNC inline const Scalar operator()(const Scalar& a) const { - typedef typename NumTraits<Scalar>::Real real_type; - real_type aa = numext::abs(a); - if (aa == real_type(0)) return Scalar(0); - aa = real_type(1) / aa; - return Scalar(real(a) * aa, imag(a) * aa); + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const + { + return (numext::isnan)(a) ? a : Scalar( (a>Scalar(0)) - (a<Scalar(0)) ); } - // TODO - // template <typename Packet> - // EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return - // internal::psign(a); } + //TODO + //template <typename Packet> + //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } }; -template <typename Scalar> -struct functor_traits<scalar_sign_op<Scalar> > { - enum { - Cost = NumTraits<Scalar>::IsComplex - ? (8 * NumTraits<Scalar>::MulCost) // roughly - : (3 * NumTraits<Scalar>::AddCost), + +template<typename Scalar, bool is_integer> +struct scalar_sign_op<Scalar,true, is_integer> { + EIGEN_EMPTY_STRUCT_CTOR(scalar_sign_op) + EIGEN_DEVICE_FUNC inline const Scalar operator() (const Scalar& a) const + { + typedef typename NumTraits<Scalar>::Real real_type; + real_type aa = numext::abs(a); + if (aa==real_type(0)) + return Scalar(0); + aa = real_type(1)/aa; + return Scalar(a.real()*aa, a.imag()*aa ); + } + //TODO + //template <typename Packet> + //EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::psign(a); } +}; +template<typename Scalar> +struct functor_traits<scalar_sign_op<Scalar> > +{ enum { + Cost = + NumTraits<Scalar>::IsComplex + ? ( 8*NumTraits<Scalar>::MulCost ) // roughly + : ( 3*NumTraits<Scalar>::AddCost), PacketAccess = packet_traits<Scalar>::HasSign }; }; /** \internal - * \brief Template functor to compute the logistic function of a scalar - * \sa class CwiseUnaryOp, ArrayBase::logistic() - */ + * \brief Template functor to compute the logistic function of a scalar + * \sa class CwiseUnaryOp, ArrayBase::logistic() + */ template <typename T> struct scalar_logistic_op { EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { - const T one = T(1); - return one / (one + numext::exp(-x)); + return packetOp(x); } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& x) const { + template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& x) const { const Packet one = pset1<Packet>(T(1)); return pdiv(one, padd(one, pexp(pnegate(x)))); } }; -template <typename T> -struct functor_traits<scalar_logistic_op<T> > { - enum { - Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost * 6, - PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasDiv && - packet_traits<T>::HasNegate && packet_traits<T>::HasExp - }; -}; +#ifndef EIGEN_GPU_COMPILE_PHASE /** \internal - * \brief Template specialization of the logistic function for float. - * - * Uses just a 9/10-degree rational interpolant which - * interpolates 1/(1+exp(-x)) - 0.5 up to a couple of ulp in the range - * [-18, 18], outside of which the fl(logistic(x)) = {0|1}. The shifted - * logistic is interpolated because it was easier to make the fit converge. - * - */ - + * \brief Template specialization of the logistic function for float. + * + * Uses just a 9/10-degree rational interpolant which + * interpolates 1/(1+exp(-x)) - 0.5 up to a couple of ulps in the range + * [-9, 18]. Below -9 we use the more accurate approximation + * 1/(1+exp(-x)) ~= exp(x), and above 18 the logistic function is 1 withing + * one ulp. The shifted logistic is interpolated because it was easier to + * make the fit converge. + * + */ template <> struct scalar_logistic_op<float> { EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(const float& x) const { - const float one = 1.0f; - return one / (one + numext::exp(-x)); + return packetOp(x); } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet - packetOp(const Packet& _x) const { - // Clamp the inputs to the range [-18, 18] since anything outside - // this range is 0.0f or 1.0f in single-precision. - const Packet x = pmax(pmin(_x, pset1<Packet>(18.0)), pset1<Packet>(-18.0)); + template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& _x) const { + const Packet cutoff_lower = pset1<Packet>(-9.f); + const Packet lt_mask = pcmp_lt<Packet>(_x, cutoff_lower); + const bool any_small = predux_any(lt_mask); + + // The upper cut-off is the smallest x for which the rational approximation evaluates to 1. + // Choosing this value saves us a few instructions clamping the results at the end. +#ifdef EIGEN_VECTORIZE_FMA + const Packet cutoff_upper = pset1<Packet>(15.7243833541870117f); +#else + const Packet cutoff_upper = pset1<Packet>(15.6437711715698242f); +#endif + const Packet x = pmin(_x, cutoff_upper); // The monomial coefficients of the numerator polynomial (odd). - const Packet alpha_1 = pset1<Packet>(2.48287947061529e-01); - const Packet alpha_3 = pset1<Packet>(8.51377133304701e-03); - const Packet alpha_5 = pset1<Packet>(6.08574864600143e-05); - const Packet alpha_7 = pset1<Packet>(1.15627324459942e-07); - const Packet alpha_9 = pset1<Packet>(4.37031012579801e-11); + const Packet alpha_1 = pset1<Packet>(2.48287947061529e-01f); + const Packet alpha_3 = pset1<Packet>(8.51377133304701e-03f); + const Packet alpha_5 = pset1<Packet>(6.08574864600143e-05f); + const Packet alpha_7 = pset1<Packet>(1.15627324459942e-07f); + const Packet alpha_9 = pset1<Packet>(4.37031012579801e-11f); // The monomial coefficients of the denominator polynomial (even). - const Packet beta_0 = pset1<Packet>(9.93151921023180e-01); - const Packet beta_2 = pset1<Packet>(1.16817656904453e-01); - const Packet beta_4 = pset1<Packet>(1.70198817374094e-03); - const Packet beta_6 = pset1<Packet>(6.29106785017040e-06); - const Packet beta_8 = pset1<Packet>(5.76102136993427e-09); - const Packet beta_10 = pset1<Packet>(6.10247389755681e-13); + const Packet beta_0 = pset1<Packet>(9.93151921023180e-01f); + const Packet beta_2 = pset1<Packet>(1.16817656904453e-01f); + const Packet beta_4 = pset1<Packet>(1.70198817374094e-03f); + const Packet beta_6 = pset1<Packet>(6.29106785017040e-06f); + const Packet beta_8 = pset1<Packet>(5.76102136993427e-09f); + const Packet beta_10 = pset1<Packet>(6.10247389755681e-13f); // Since the polynomials are odd/even, we need x^2. const Packet x2 = pmul(x, x); @@ -1034,21 +1087,45 @@ struct scalar_logistic_op<float> { p = pmadd(x2, p, alpha_1); p = pmul(x, p); - // Evaluate the denominator polynomial p. + // Evaluate the denominator polynomial q. Packet q = pmadd(x2, beta_10, beta_8); q = pmadd(x2, q, beta_6); q = pmadd(x2, q, beta_4); q = pmadd(x2, q, beta_2); q = pmadd(x2, q, beta_0); - // Divide the numerator by the denominator and shift it up. - return pmax(pmin(padd(pdiv(p, q), pset1<Packet>(0.5)), pset1<Packet>(1.0)), - pset1<Packet>(0.0)); + const Packet logistic = padd(pdiv(p, q), pset1<Packet>(0.5f)); + if (EIGEN_PREDICT_FALSE(any_small)) { + const Packet exponential = pexp(_x); + return pselect(lt_mask, exponential, logistic); + } else { + return logistic; + } } }; +#endif // #ifndef EIGEN_GPU_COMPILE_PHASE + +template <typename T> +struct functor_traits<scalar_logistic_op<T> > { + enum { + // The cost estimate for float here here is for the common(?) case where + // all arguments are greater than -9. + Cost = scalar_div_cost<T, packet_traits<T>::HasDiv>::value + + (internal::is_same<T, float>::value + ? NumTraits<T>::AddCost * 15 + NumTraits<T>::MulCost * 11 + : NumTraits<T>::AddCost * 2 + + functor_traits<scalar_exp_op<T> >::Cost), + PacketAccess = + packet_traits<T>::HasAdd && packet_traits<T>::HasDiv && + (internal::is_same<T, float>::value + ? packet_traits<T>::HasMul && packet_traits<T>::HasMax && + packet_traits<T>::HasMin + : packet_traits<T>::HasNegate && packet_traits<T>::HasExp) + }; +}; -} // end namespace internal +} // end namespace internal -} // end namespace Eigen +} // end namespace Eigen -#endif // EIGEN_FUNCTORS_H +#endif // EIGEN_FUNCTORS_H |