diff options
Diffstat (limited to 'Eigen/src/Core/Redux.h')
| -rw-r--r-- | Eigen/src/Core/Redux.h | 215 |
1 files changed, 156 insertions, 59 deletions
diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 50548fa9a..b6e8f8887 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h @@ -27,8 +27,9 @@ template<typename Func, typename Derived> struct redux_traits { public: + typedef typename find_best_packet<typename Derived::Scalar,Derived::SizeAtCompileTime>::type PacketType; enum { - PacketSize = packet_traits<typename Derived::Scalar>::size, + PacketSize = unpacket_traits<PacketType>::size, InnerMaxSize = int(Derived::IsRowMajor) ? Derived::MaxColsAtCompileTime : Derived::MaxRowsAtCompileTime @@ -37,8 +38,8 @@ public: enum { MightVectorize = (int(Derived::Flags)&ActualPacketAccessBit) && (functor_traits<Func>::PacketAccess), - MayLinearVectorize = MightVectorize && (int(Derived::Flags)&LinearAccessBit), - MaySliceVectorize = MightVectorize && int(InnerMaxSize)>=3*PacketSize + MayLinearVectorize = bool(MightVectorize) && (int(Derived::Flags)&LinearAccessBit), + MaySliceVectorize = bool(MightVectorize) && int(InnerMaxSize)>=3*PacketSize }; public: @@ -50,21 +51,34 @@ public: public: enum { - Cost = ( Derived::SizeAtCompileTime == Dynamic - || Derived::CoeffReadCost == Dynamic - || (Derived::SizeAtCompileTime!=1 && functor_traits<Func>::Cost == Dynamic) - ) ? Dynamic - : Derived::SizeAtCompileTime * Derived::CoeffReadCost - + (Derived::SizeAtCompileTime-1) * functor_traits<Func>::Cost, + Cost = Derived::SizeAtCompileTime == Dynamic ? HugeCost + : Derived::SizeAtCompileTime * Derived::CoeffReadCost + (Derived::SizeAtCompileTime-1) * functor_traits<Func>::Cost, UnrollingLimit = EIGEN_UNROLLING_LIMIT * (int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize)) }; public: enum { - Unrolling = Cost != Dynamic && Cost <= UnrollingLimit - ? CompleteUnrolling - : NoUnrolling + Unrolling = Cost <= UnrollingLimit ? CompleteUnrolling : NoUnrolling }; + +#ifdef EIGEN_DEBUG_ASSIGN + static void debug() + { + std::cerr << "Xpr: " << typeid(typename Derived::XprType).name() << std::endl; + std::cerr.setf(std::ios::hex, std::ios::basefield); + EIGEN_DEBUG_VAR(Derived::Flags) + std::cerr.unsetf(std::ios::hex); + EIGEN_DEBUG_VAR(InnerMaxSize) + EIGEN_DEBUG_VAR(PacketSize) + EIGEN_DEBUG_VAR(MightVectorize) + EIGEN_DEBUG_VAR(MayLinearVectorize) + EIGEN_DEBUG_VAR(MaySliceVectorize) + EIGEN_DEBUG_VAR(Traversal) + EIGEN_DEBUG_VAR(UnrollingLimit) + EIGEN_DEBUG_VAR(Unrolling) + std::cerr << std::endl; + } +#endif }; /*************************************************************************** @@ -82,6 +96,7 @@ struct redux_novec_unroller typedef typename Derived::Scalar Scalar; + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) { return func(redux_novec_unroller<Func, Derived, Start, HalfLength>::run(mat,func), @@ -99,6 +114,7 @@ struct redux_novec_unroller<Func, Derived, Start, 1> typedef typename Derived::Scalar Scalar; + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func&) { return mat.coeffByOuterInner(outer, inner); @@ -112,6 +128,7 @@ template<typename Func, typename Derived, int Start> struct redux_novec_unroller<Func, Derived, Start, 0> { typedef typename Derived::Scalar Scalar; + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Derived&, const Func&) { return Scalar(); } }; @@ -121,12 +138,12 @@ template<typename Func, typename Derived, int Start, int Length> struct redux_vec_unroller { enum { - PacketSize = packet_traits<typename Derived::Scalar>::size, + PacketSize = redux_traits<Func, Derived>::PacketSize, HalfLength = Length/2 }; typedef typename Derived::Scalar Scalar; - typedef typename packet_traits<Scalar>::type PacketScalar; + typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; static EIGEN_STRONG_INLINE PacketScalar run(const Derived &mat, const Func& func) { @@ -140,18 +157,18 @@ template<typename Func, typename Derived, int Start> struct redux_vec_unroller<Func, Derived, Start, 1> { enum { - index = Start * packet_traits<typename Derived::Scalar>::size, + index = Start * redux_traits<Func, Derived>::PacketSize, outer = index / int(Derived::InnerSizeAtCompileTime), inner = index % int(Derived::InnerSizeAtCompileTime), - alignment = (Derived::Flags & AlignedBit) ? Aligned : Unaligned + alignment = Derived::Alignment }; typedef typename Derived::Scalar Scalar; - typedef typename packet_traits<Scalar>::type PacketScalar; + typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; static EIGEN_STRONG_INLINE PacketScalar run(const Derived &mat, const Func&) { - return mat.template packetByOuterInner<alignment>(outer, inner); + return mat.template packetByOuterInner<alignment,PacketScalar>(outer, inner); } }; @@ -169,8 +186,8 @@ template<typename Func, typename Derived> struct redux_impl<Func, Derived, DefaultTraversal, NoUnrolling> { typedef typename Derived::Scalar Scalar; - typedef typename Derived::Index Index; - static EIGEN_STRONG_INLINE Scalar run(const Derived& mat, const Func& func) + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); Scalar res; @@ -193,19 +210,19 @@ template<typename Func, typename Derived> struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling> { typedef typename Derived::Scalar Scalar; - typedef typename packet_traits<Scalar>::type PacketScalar; - typedef typename Derived::Index Index; + typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; - static Scalar run(const Derived& mat, const Func& func) + static Scalar run(const Derived &mat, const Func& func) { const Index size = mat.size(); - eigen_assert(size && "you are using an empty matrix"); - const Index packetSize = packet_traits<Scalar>::size; - const Index alignedStart = internal::first_aligned(mat); + + const Index packetSize = redux_traits<Func, Derived>::PacketSize; + const int packetAlignment = unpacket_traits<PacketScalar>::alignment; enum { - alignment = bool(Derived::Flags & DirectAccessBit) || bool(Derived::Flags & AlignedBit) - ? Aligned : Unaligned + alignment0 = (bool(Derived::Flags & DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar)) ? int(packetAlignment) : int(Unaligned), + alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Derived::Alignment) }; + const Index alignedStart = internal::first_default_aligned(mat.nestedExpression()); const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize); const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize); const Index alignedEnd2 = alignedStart + alignedSize2; @@ -213,19 +230,19 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling> Scalar res; if(alignedSize) { - PacketScalar packet_res0 = mat.template packet<alignment>(alignedStart); + PacketScalar packet_res0 = mat.template packet<alignment,PacketScalar>(alignedStart); if(alignedSize>packetSize) // we have at least two packets to partly unroll the loop { - PacketScalar packet_res1 = mat.template packet<alignment>(alignedStart+packetSize); + PacketScalar packet_res1 = mat.template packet<alignment,PacketScalar>(alignedStart+packetSize); for(Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize) { - packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment>(index)); - packet_res1 = func.packetOp(packet_res1, mat.template packet<alignment>(index+packetSize)); + packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(index)); + packet_res1 = func.packetOp(packet_res1, mat.template packet<alignment,PacketScalar>(index+packetSize)); } packet_res0 = func.packetOp(packet_res0,packet_res1); if(alignedEnd>alignedEnd2) - packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment>(alignedEnd2)); + packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(alignedEnd2)); } res = func.predux(packet_res0); @@ -247,29 +264,29 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling> } }; -template<typename Func, typename Derived> -struct redux_impl<Func, Derived, SliceVectorizedTraversal, NoUnrolling> +// NOTE: for SliceVectorizedTraversal we simply bypass unrolling +template<typename Func, typename Derived, int Unrolling> +struct redux_impl<Func, Derived, SliceVectorizedTraversal, Unrolling> { typedef typename Derived::Scalar Scalar; - typedef typename packet_traits<Scalar>::type PacketScalar; - typedef typename Derived::Index Index; + typedef typename redux_traits<Func, Derived>::PacketType PacketType; - static Scalar run(const Derived& mat, const Func& func) + EIGEN_DEVICE_FUNC static Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); const Index innerSize = mat.innerSize(); const Index outerSize = mat.outerSize(); enum { - packetSize = packet_traits<Scalar>::size + packetSize = redux_traits<Func, Derived>::PacketSize }; const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize; Scalar res; if(packetedInnerSize) { - PacketScalar packet_res = mat.template packet<Unaligned>(0,0); + PacketType packet_res = mat.template packet<Unaligned,PacketType>(0,0); for(Index j=0; j<outerSize; ++j) for(Index i=(j==0?packetSize:0); i<packetedInnerSize; i+=Index(packetSize)) - packet_res = func.packetOp(packet_res, mat.template packetByOuterInner<Unaligned>(j,i)); + packet_res = func.packetOp(packet_res, mat.template packetByOuterInner<Unaligned,PacketType>(j,i)); res = func.predux(packet_res); for(Index j=0; j<outerSize; ++j) @@ -290,22 +307,90 @@ template<typename Func, typename Derived> struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> { typedef typename Derived::Scalar Scalar; - typedef typename packet_traits<Scalar>::type PacketScalar; + + typedef typename redux_traits<Func, Derived>::PacketType PacketScalar; enum { - PacketSize = packet_traits<Scalar>::size, + PacketSize = redux_traits<Func, Derived>::PacketSize, Size = Derived::SizeAtCompileTime, VectorizedSize = (Size / PacketSize) * PacketSize }; - static EIGEN_STRONG_INLINE Scalar run(const Derived& mat, const Func& func) + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); - Scalar res = func.predux(redux_vec_unroller<Func, Derived, 0, Size / PacketSize>::run(mat,func)); - if (VectorizedSize != Size) - res = func(res,redux_novec_unroller<Func, Derived, VectorizedSize, Size-VectorizedSize>::run(mat,func)); - return res; + if (VectorizedSize > 0) { + Scalar res = func.predux(redux_vec_unroller<Func, Derived, 0, Size / PacketSize>::run(mat,func)); + if (VectorizedSize != Size) + res = func(res,redux_novec_unroller<Func, Derived, VectorizedSize, Size-VectorizedSize>::run(mat,func)); + return res; + } + else { + return redux_novec_unroller<Func, Derived, 0, Size>::run(mat,func); + } } }; +// evaluator adaptor +template<typename _XprType> +class redux_evaluator +{ +public: + typedef _XprType XprType; + EIGEN_DEVICE_FUNC explicit redux_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} + + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + enum { + MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime, + MaxColsAtCompileTime = XprType::MaxColsAtCompileTime, + // TODO we should not remove DirectAccessBit and rather find an elegant way to query the alignment offset at runtime from the evaluator + Flags = evaluator<XprType>::Flags & ~DirectAccessBit, + IsRowMajor = XprType::IsRowMajor, + SizeAtCompileTime = XprType::SizeAtCompileTime, + InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime, + CoeffReadCost = evaluator<XprType>::CoeffReadCost, + Alignment = evaluator<XprType>::Alignment + }; + + EIGEN_DEVICE_FUNC Index rows() const { return m_xpr.rows(); } + EIGEN_DEVICE_FUNC Index cols() const { return m_xpr.cols(); } + EIGEN_DEVICE_FUNC Index size() const { return m_xpr.size(); } + EIGEN_DEVICE_FUNC Index innerSize() const { return m_xpr.innerSize(); } + EIGEN_DEVICE_FUNC Index outerSize() const { return m_xpr.outerSize(); } + + EIGEN_DEVICE_FUNC + CoeffReturnType coeff(Index row, Index col) const + { return m_evaluator.coeff(row, col); } + + EIGEN_DEVICE_FUNC + CoeffReturnType coeff(Index index) const + { return m_evaluator.coeff(index); } + + template<int LoadMode, typename PacketType> + PacketType packet(Index row, Index col) const + { return m_evaluator.template packet<LoadMode,PacketType>(row, col); } + + template<int LoadMode, typename PacketType> + PacketType packet(Index index) const + { return m_evaluator.template packet<LoadMode,PacketType>(index); } + + EIGEN_DEVICE_FUNC + CoeffReturnType coeffByOuterInner(Index outer, Index inner) const + { return m_evaluator.coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + + template<int LoadMode, typename PacketType> + PacketType packetByOuterInner(Index outer, Index inner) const + { return m_evaluator.template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + + const XprType & nestedExpression() const { return m_xpr; } + +protected: + internal::evaluator<XprType> m_evaluator; + const XprType &m_xpr; +}; + } // end namespace internal /*************************************************************************** @@ -316,18 +401,21 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> /** \returns the result of a full redux operation on the whole matrix or vector using \a func * * The template parameter \a BinaryOp is the type of the functor \a func which must be - * an associative operator. Both current STL and TR1 functor styles are handled. + * an associative operator. Both current C++98 and C++11 functor styles are handled. * * \sa DenseBase::sum(), DenseBase::minCoeff(), DenseBase::maxCoeff(), MatrixBase::colwise(), MatrixBase::rowwise() */ template<typename Derived> template<typename Func> -EIGEN_STRONG_INLINE typename internal::result_of<Func(typename internal::traits<Derived>::Scalar)>::type +typename internal::traits<Derived>::Scalar DenseBase<Derived>::redux(const Func& func) const { - typedef typename internal::remove_all<typename Derived::Nested>::type ThisNested; - return internal::redux_impl<Func, ThisNested> - ::run(derived(), func); + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); + + typedef typename internal::redux_evaluator<Derived> ThisEvaluator; + ThisEvaluator thisEval(derived()); + + return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func); } /** \returns the minimum of all coefficients of \c *this. @@ -337,7 +425,7 @@ template<typename Derived> EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::minCoeff() const { - return this->redux(Eigen::internal::scalar_min_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_min_op<Scalar,Scalar>()); } /** \returns the maximum of all coefficients of \c *this. @@ -347,10 +435,12 @@ template<typename Derived> EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::maxCoeff() const { - return this->redux(Eigen::internal::scalar_max_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_max_op<Scalar,Scalar>()); } -/** \returns the sum of all coefficients of *this +/** \returns the sum of all coefficients of \c *this + * + * If \c *this is empty, then the value 0 is returned. * * \sa trace(), prod(), mean() */ @@ -360,7 +450,7 @@ DenseBase<Derived>::sum() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) return Scalar(0); - return this->redux(Eigen::internal::scalar_sum_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>()); } /** \returns the mean of all coefficients of *this @@ -371,7 +461,14 @@ template<typename Derived> EIGEN_STRONG_INLINE typename internal::traits<Derived>::Scalar DenseBase<Derived>::mean() const { - return Scalar(this->redux(Eigen::internal::scalar_sum_op<Scalar>())) / Scalar(this->size()); +#ifdef __INTEL_COMPILER + #pragma warning push + #pragma warning ( disable : 2259 ) +#endif + return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>())) / Scalar(this->size()); +#ifdef __INTEL_COMPILER + #pragma warning pop +#endif } /** \returns the product of all coefficients of *this @@ -387,7 +484,7 @@ DenseBase<Derived>::prod() const { if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0)) return Scalar(1); - return this->redux(Eigen::internal::scalar_product_op<Scalar>()); + return derived().redux(Eigen::internal::scalar_product_op<Scalar>()); } /** \returns the trace of \c *this, i.e. the sum of the coefficients on the main diagonal. |