diff --git a/CLAUDE.md b/CLAUDE.md index 7f12787..90f5f76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,7 +48,7 @@ impl/kernels ─ impl/cpu ─ api/cpu + src/lib-cpu ┐ Namespace `ff::::` via the `FF_NAMESPACE_BEGIN` macros — do **not** hard-code `ff::cpu`. - **`include/fastfields/impl/cpu/`**, **`impl/cuda/`** — the loops over - elements (thread pool / OpenMP; `__global__` kernels + `CUHOST` launchers). + elements (thread pool / OpenMP; `__global__` kernels + `FF_CUHOST` launchers). Header-only, templated, dynamic sizes. `ff::cpu::…` / `ff::cuda::…`. - **`include/fastfields/api/cpu/`**, **`api/cuda/`** + **`src/lib-cpu/`**, **`src/lib-cuda/`** — the dtype-dispatch boundary. Public symbols take diff --git a/MIGRATION.md b/MIGRATION.md index c63dcb4..ea04782 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -135,7 +135,7 @@ CPU path against a brute-force / reference implementation, as `test_distance.cpp spurious extra arg (no matching overload); triggered by `restrict::loop`. 10. **cpu-impl/{resize,restrict,splinc}.h** — wrong include prefix `"lib/…"` → `"kernels/…"`; impl namespace was plain `ff::` but the kernels live in - `ff::cpu::` (`FF_DEVICE`) so it must be `FF_NAMESPACE_BEGIN(FF)/(FF_DEVICE)/()` + `ff::cpu::` (`FF_DEVICE`) so it must be `FF_NAMESPACE_BEGIN(FF_NS)/(FF_DEVICE)/()` like distance; `index2offset_nd()` runtime-ndim → dynamic overload; `jf::has_atomic_add` → `has_atomic_add`. diff --git a/include/fastfields/api/checks.h b/include/fastfields/api/checks.h index 9574b1c..9369c1d 100644 --- a/include/fastfields/api/checks.h +++ b/include/fastfields/api/checks.h @@ -4,7 +4,7 @@ #include "fastfields/core/dlpack.h" #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * Assert that a set of DLTensors all live on the same device. @@ -55,6 +55,6 @@ inline void require_same_device(const DLTensor & ref, const DLTensor & t, const require_same_device(ref, rest...); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_CHECKS diff --git a/include/fastfields/api/cpu/pushpull_dispatch.h b/include/fastfields/api/cpu/pushpull_dispatch.h index 50ec072..a8790c0 100644 --- a/include/fastfields/api/cpu/pushpull_dispatch.h +++ b/include/fastfields/api/cpu/pushpull_dispatch.h @@ -23,7 +23,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/pushpull.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduce/accumulation type used by the sampling kernels. Match jitfields @@ -34,7 +34,7 @@ typedef double reduce_t; * DISPATCH * ***********************************************************************/ -#define PP_DTYPE(D, I, B, FN, args...) \ +#define FF_PP_DTYPE(D, I, B, FN, args...) \ switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return (use_32bits ? FN(args) \ @@ -45,25 +45,25 @@ typedef double reduce_t; }; default: break; \ } -// The template argument fed to PP_DTYPE is FF_BOUND_ / FF_SPLINE_ +// The template argument fed to FF_PP_DTYPE is FF_BOUND_ / FF_SPLINE_ // (kernels/bounds.h, kernels/spline.h) -- the condition/order itself when it // is statically compiled, `Dynamic` otherwise per BOUNDFLAGS/SPLINEFLAGS. The // switch labels stay exhaustive on the *runtime* value either way; only the // instantiated template argument collapses onto the shared Dynamic path. -#define PP_BOUND(D, I, FN, args...) \ +#define FF_PP_BOUND(D, I, FN, args...) \ switch (bnd) { \ - case bound_t::Zero: PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \ - case bound_t::Replicate: PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \ - case bound_t::DCT1: PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \ - case bound_t::DCT2: PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \ - case bound_t::DST1: PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \ - case bound_t::DST2: PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \ - case bound_t::DFT: PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \ - case bound_t::NoCheck: PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \ + case bound_t::Zero: FF_PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \ + case bound_t::Replicate: FF_PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \ + case bound_t::DCT1: FF_PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \ + case bound_t::DCT2: FF_PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \ + case bound_t::DST1: FF_PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \ + case bound_t::DST2: FF_PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \ + case bound_t::DFT: FF_PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \ + case bound_t::NoCheck: FF_PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \ default: throw std::invalid_argument("Unsupported boundary condition");\ } -// There used to be a second, hand-duplicated PP_ORDER (behind `-DFF_TEST_SPARSE`) +// There used to be a second, hand-duplicated FF_PP_ORDER (behind `-DFF_TEST_SPARSE`) // that hard-coded a *covering* subset of the order x bound matrix -- literally // rejecting (throwing) most bound/order combinations at runtime -- purely to // keep the test build's compile time down. That is now redundant with, and @@ -72,29 +72,29 @@ typedef double reduce_t; // instantiation (the actual compile-cost win FF_TEST_SPARSE was chasing), // while every combination stays fully *functional* (just via the Dynamic // runtime path instead of a dedicated static one) rather than throwing. -// There is therefore only one PP_ORDER/PP_BOUND now; which combinations are +// There is therefore only one FF_PP_ORDER/FF_PP_BOUND now; which combinations are // statically instantiated and which share Dynamic is entirely a BOUNDFLAGS/ // SPLINEFLAGS *build-time* choice (Makefile: a sparser default for the `test` // target, the full static matrix for the library), not a code-level branch. -#define PP_ORDER(D, FN, args...) \ +#define FF_PP_ORDER(D, FN, args...) \ switch (spl) { \ - case spline_t::Nearest: PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \ - case spline_t::Linear: PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \ - case spline_t::Quadratic: PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \ - case spline_t::Cubic: PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \ - case spline_t::FourthOrder: PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \ - case spline_t::FifthOrder: PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \ - case spline_t::SixthOrder: PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \ - case spline_t::SeventhOrder: PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \ + case spline_t::Nearest: FF_PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \ + case spline_t::Linear: FF_PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \ + case spline_t::Quadratic: FF_PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \ + case spline_t::Cubic: FF_PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \ + case spline_t::FourthOrder: FF_PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \ + case spline_t::FifthOrder: FF_PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \ + case spline_t::SixthOrder: FF_PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \ + case spline_t::SeventhOrder: FF_PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \ default: throw std::invalid_argument("Unsupported spline order"); \ } -#define DISPATCH_PP(FN, args...) \ +#define FF_DISPATCH_PP(FN, args...) \ { \ switch (ndim) { \ - case 1: PP_ORDER(1, FN, args); break; \ - case 2: PP_ORDER(2, FN, args); break; \ - case 3: PP_ORDER(3, FN, args); break; \ + case 1: FF_PP_ORDER(1, FN, args); break; \ + case 2: FF_PP_ORDER(2, FN, args); break; \ + case 3: FF_PP_ORDER(3, FN, args); break; \ default: throw std::invalid_argument("Only 1D, 2D and 3D are supported"); \ }; \ throw std::invalid_argument("Unsupported data type"); \ @@ -102,6 +102,6 @@ typedef double reduce_t; FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CPU_PUSHPULL_DISPATCH diff --git a/include/fastfields/api/cuda/pushpull_dispatch.h b/include/fastfields/api/cuda/pushpull_dispatch.h index 626d0e1..a8117b1 100644 --- a/include/fastfields/api/cuda/pushpull_dispatch.h +++ b/include/fastfields/api/cuda/pushpull_dispatch.h @@ -23,7 +23,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/pushpull.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduce/accumulation type used by the sampling kernels. Match jitfields @@ -34,7 +34,7 @@ typedef double reduce_t; * DISPATCH * ***********************************************************************/ -#define PP_DTYPE(D, I, B, FN, args...) \ +#define FF_PP_DTYPE(D, I, B, FN, args...) \ switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return (use_32bits ? FN(args) \ @@ -45,45 +45,45 @@ typedef double reduce_t; }; default: break; \ } -// The template argument fed to PP_DTYPE is FF_BOUND_ / FF_SPLINE_ +// The template argument fed to FF_PP_DTYPE is FF_BOUND_ / FF_SPLINE_ // (kernels/bounds.h, kernels/spline.h) -- the condition/order itself when it // is statically compiled, `Dynamic` otherwise per BOUNDFLAGS/SPLINEFLAGS. The // switch labels stay exhaustive on the *runtime* value either way; only the // instantiated template argument collapses onto the shared Dynamic path. // This is what actually keeps ptxas's memory bounded -- see BOUNDFLAGS/ // SPLINEFLAGS in the Makefile. -#define PP_BOUND(D, I, FN, args...) \ +#define FF_PP_BOUND(D, I, FN, args...) \ switch (bnd) { \ - case bound_t::Zero: PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \ - case bound_t::Replicate: PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \ - case bound_t::DCT1: PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \ - case bound_t::DCT2: PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \ - case bound_t::DST1: PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \ - case bound_t::DST2: PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \ - case bound_t::DFT: PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \ - case bound_t::NoCheck: PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \ + case bound_t::Zero: FF_PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \ + case bound_t::Replicate: FF_PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \ + case bound_t::DCT1: FF_PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \ + case bound_t::DCT2: FF_PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \ + case bound_t::DST1: FF_PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \ + case bound_t::DST2: FF_PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \ + case bound_t::DFT: FF_PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \ + case bound_t::NoCheck: FF_PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \ default: throw std::invalid_argument("Unsupported boundary condition");\ } -#define PP_ORDER(D, FN, args...) \ +#define FF_PP_ORDER(D, FN, args...) \ switch (spl) { \ - case spline_t::Nearest: PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \ - case spline_t::Linear: PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \ - case spline_t::Quadratic: PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \ - case spline_t::Cubic: PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \ - case spline_t::FourthOrder: PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \ - case spline_t::FifthOrder: PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \ - case spline_t::SixthOrder: PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \ - case spline_t::SeventhOrder: PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \ + case spline_t::Nearest: FF_PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \ + case spline_t::Linear: FF_PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \ + case spline_t::Quadratic: FF_PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \ + case spline_t::Cubic: FF_PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \ + case spline_t::FourthOrder: FF_PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \ + case spline_t::FifthOrder: FF_PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \ + case spline_t::SixthOrder: FF_PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \ + case spline_t::SeventhOrder: FF_PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \ default: throw std::invalid_argument("Unsupported spline order"); \ } -#define DISPATCH_PP(FN, args...) \ +#define FF_DISPATCH_PP(FN, args...) \ { \ switch (ndim) { \ - case 1: PP_ORDER(1, FN, args); break; \ - case 2: PP_ORDER(2, FN, args); break; \ - case 3: PP_ORDER(3, FN, args); break; \ + case 1: FF_PP_ORDER(1, FN, args); break; \ + case 2: FF_PP_ORDER(2, FN, args); break; \ + case 3: FF_PP_ORDER(3, FN, args); break; \ default: throw std::invalid_argument("Only 1D, 2D and 3D are supported"); \ }; \ throw std::invalid_argument("Unsupported data type"); \ @@ -91,6 +91,6 @@ typedef double reduce_t; FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CUDA_PUSHPULL_DISPATCH diff --git a/include/fastfields/api/cuda/stream.h b/include/fastfields/api/cuda/stream.h index 5a0f086..a37975a 100644 --- a/include/fastfields/api/cuda/stream.h +++ b/include/fastfields/api/cuda/stream.h @@ -18,7 +18,7 @@ #include #include "fastfields/core/cuda_switch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // intptr_t -> cudaStream_t (0 == the default stream). @@ -28,6 +28,6 @@ static inline cudaStream_t _reg_stream(intptr_t stream) } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CUDA_STREAM diff --git a/include/fastfields/api/distance.h b/include/fastfields/api/distance.h index 072d439..1244cfb 100644 --- a/include/fastfields/api/distance.h +++ b/include/fastfields/api/distance.h @@ -4,7 +4,7 @@ #include #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T @@ -175,6 +175,6 @@ void dt_mesh( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_DISTANCE diff --git a/include/fastfields/api/posdef.h b/include/fastfields/api/posdef.h index 6786066..2230b5f 100644 --- a/include/fastfields/api/posdef.h +++ b/include/fastfields/api/posdef.h @@ -4,7 +4,7 @@ #include #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * Compact symmetric ("Sym") positive-definite matrix operations. @@ -79,6 +79,6 @@ void sym_invert_( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_POSDEF diff --git a/include/fastfields/api/pushpull.h b/include/fastfields/api/pushpull.h index 4cb399a..6743e40 100644 --- a/include/fastfields/api/pushpull.h +++ b/include/fastfields/api/pushpull.h @@ -6,7 +6,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Sample ("pull") a spline-encoded volume at arbitrary coordinates. @@ -227,6 +227,6 @@ void grad_backward( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_PUSHPULL diff --git a/include/fastfields/api/reg_field.h b/include/fastfields/api/reg_field.h index 63218e5..ebd0bba 100644 --- a/include/fastfields/api/reg_field.h +++ b/include/fastfields/api/reg_field.h @@ -6,7 +6,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Apply a spatial regulariser operator to a multi-channel field. @@ -375,6 +375,6 @@ void field_relax_rls( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_REG_FIELD diff --git a/include/fastfields/api/reg_flow.h b/include/fastfields/api/reg_flow.h index b853f05..d092bbe 100644 --- a/include/fastfields/api/reg_flow.h +++ b/include/fastfields/api/reg_flow.h @@ -6,7 +6,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Apply a spatial regulariser operator to a vector flow field. @@ -396,6 +396,6 @@ void flow_relax_rls( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_REG_FLOW diff --git a/include/fastfields/api/resize.h b/include/fastfields/api/resize.h index a2fac57..e8d8456 100644 --- a/include/fastfields/api/resize.h +++ b/include/fastfields/api/resize.h @@ -6,7 +6,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Resample (prolongation) a tensor to a new shape using spline @@ -67,6 +67,6 @@ void resample( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_RESIZE diff --git a/include/fastfields/api/restrict.h b/include/fastfields/api/restrict.h index cdaa9c8..f438f10 100644 --- a/include/fastfields/api/restrict.h +++ b/include/fastfields/api/restrict.h @@ -6,7 +6,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Restriction: the adjoint (transpose) of the resize prolongation. @@ -66,6 +66,6 @@ void restriction( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_RESTRICT diff --git a/include/fastfields/api/solve_field.h b/include/fastfields/api/solve_field.h index 969275e..4bfa7d8 100644 --- a/include/fastfields/api/solve_field.h +++ b/include/fastfields/api/solve_field.h @@ -5,7 +5,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -33,10 +33,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Solve `(H + L) x = g` by Jacobi-preconditioned conjugate gradients. @@ -92,6 +92,6 @@ void field_cg( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_SOLVE_FIELD diff --git a/include/fastfields/api/splinc.h b/include/fastfields/api/splinc.h index 3f3990a..fb6b436 100644 --- a/include/fastfields/api/splinc.h +++ b/include/fastfields/api/splinc.h @@ -8,7 +8,7 @@ #ifndef FF_LIB_BOUND_SPLINE_T #define FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound_t) using T = int8_t; @@ -36,10 +36,10 @@ static constexpr T SixthOrder = 6; static constexpr T SeventhOrder = 7; FF_NAMESPACE_END(spline_t) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_BOUND_SPLINE_T -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * @brief Assert that `bound` is a boundary condition the prefilter implements. @@ -113,6 +113,6 @@ void spline_coeff( intptr_t stream = 0 ); -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_LIB_SPLINC diff --git a/include/fastfields/core/autocast.h b/include/fastfields/core/autocast.h index de3af9e..88c6c7b 100644 --- a/include/fastfields/core/autocast.h +++ b/include/fastfields/core/autocast.h @@ -6,7 +6,7 @@ #include "fastfields/core/dlpack.h" #include "fastfields/core/cuda_switch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // DLPack allows DLTensor.strides == NULL to mean "compact row-major". The @@ -198,6 +198,6 @@ inline void free_if_needed(OutPointer ptr) } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_AUTOCAST diff --git a/include/fastfields/core/cuda_switch.h b/include/fastfields/core/cuda_switch.h index 2805d31..d85669b 100755 --- a/include/fastfields/core/cuda_switch.h +++ b/include/fastfields/core/cuda_switch.h @@ -12,19 +12,19 @@ #ifndef __host__ #define __host__ #endif -#define CUGLOB -#define CUHOST -#define CUDEV -#define CUHOSTDEV +#define FF_CUGLOB +#define FF_CUHOST +#define FF_CUDEV +#define FF_CUHOSTDEV #define FF_DEVICE cpu #include #else -#define CUGLOB __global__ -#define CUHOST __host__ -#define CUDEV __device__ -#define CUHOSTDEV __host__ __device__ +#define FF_CUGLOB __global__ +#define FF_CUHOST __host__ +#define FF_CUDEV __device__ +#define FF_CUHOSTDEV __host__ __device__ #define FF_DEVICE cuda #ifdef __CUDACC_RTC__ diff --git a/include/fastfields/core/defines.h b/include/fastfields/core/defines.h index 27b8258..3bc5819 100644 --- a/include/fastfields/core/defines.h +++ b/include/fastfields/core/defines.h @@ -8,7 +8,21 @@ // from the hub. One file, one guard -- so a bare `#include "defines.h"` can no // longer resolve to a different header than the author meant. -#define FF ff +// `FF_NS` is the project's root namespace spelled once, so that +// `FF_NAMESPACE_BEGIN(FF_NS)` is the only place any header names it. +// +// It was a bare two-letter macro until the public-macro prefixing pass. An +// all-caps two-letter name in an installed header takes that name away from +// every translation unit downstream of us, and it is an entirely plausible +// downstream identifier (an enum member, a constant, a template parameter). +// +// `#undef`-ing it at the end of this header is NOT an alternative to renaming +// it -- it is actively wrong. The macro is consumed by ~105 *other* files +// after they include this one, so undefining it here would quietly turn every +// `FF_NAMESPACE_BEGIN(FF_NS)` into a namespace literally named `FF_NS`, +// rather than into an error. The `#undef` idiom only works for a macro used +// within the header that defines it, which this is not. +#define FF_NS ff #define FF_NAMESPACE_BEGIN(NAME) namespace NAME { #define FF_NAMESPACE_END(NAME) } #define FF_NAMESPACE_BEGIN_DEVICE FF_NAMESPACE_BEGIN(FF_DEVICE) diff --git a/include/fastfields/core/dispatch.h b/include/fastfields/core/dispatch.h index c2a1a74..c164876 100644 --- a/include/fastfields/core/dispatch.h +++ b/include/fastfields/core/dispatch.h @@ -121,7 +121,7 @@ * NON-TENSOR ARGUMENT MARSHALLING * ***********************************************************************/ -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) /** * Build a length-`nc` penalty-weight vector from the ABI's `const double *`. @@ -144,6 +144,6 @@ inline std::vector as_weights(const double * w, int64_t nc) return v; } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CORE_DISPATCH diff --git a/include/fastfields/impl/cpu/distance_euclidean.h b/include/fastfields/impl/cpu/distance_euclidean.h index ba8b5a3..1ffc483 100755 --- a/include/fastfields/impl/cpu/distance_euclidean.h +++ b/include/fastfields/impl/cpu/distance_euclidean.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/parallel.h" #include "fastfields/impl/kernels/utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_e) @@ -55,6 +55,6 @@ dt( FF_NAMESPACE_END(distance_e) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CPU_DISTANCE_EUCLIDEAN diff --git a/include/fastfields/impl/cpu/distance_l1.h b/include/fastfields/impl/cpu/distance_l1.h index c9a81e3..7597a4e 100755 --- a/include/fastfields/impl/cpu/distance_l1.h +++ b/include/fastfields/impl/cpu/distance_l1.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/parallel.h" #include "fastfields/impl/kernels/utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_l1) @@ -34,6 +34,6 @@ dt( FF_NAMESPACE_END(distance_l1) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CPU_DISTANCE_L1 diff --git a/include/fastfields/impl/cpu/distance_mesh.h b/include/fastfields/impl/cpu/distance_mesh.h index e18c22a..452396a 100755 --- a/include/fastfields/impl/cpu/distance_mesh.h +++ b/include/fastfields/impl/cpu/distance_mesh.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_mesh) @@ -713,7 +713,7 @@ dt( FF_NAMESPACE_END(distance_mesh) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_MESH diff --git a/include/fastfields/impl/cpu/distance_spline.h b/include/fastfields/impl/cpu/distance_spline.h index 97e096b..73eaf98 100755 --- a/include/fastfields/impl/cpu/distance_spline.h +++ b/include/fastfields/impl/cpu/distance_spline.h @@ -5,7 +5,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_spline) @@ -181,6 +181,6 @@ mindist_gaussnewton( FF_NAMESPACE_END(distance_spline) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_CPU_DISTANCE_SPLINE diff --git a/include/fastfields/impl/cpu/posdef.h b/include/fastfields/impl/cpu/posdef.h index 2c73062..b658d8f 100755 --- a/include/fastfields/impl/cpu/posdef.h +++ b/include/fastfields/impl/cpu/posdef.h @@ -5,7 +5,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(posdef) @@ -465,6 +465,6 @@ void sym_invert_( FF_NAMESPACE_END(posdef) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_POSDEF_CPU diff --git a/include/fastfields/impl/cpu/pushpull.h b/include/fastfields/impl/cpu/pushpull.h index 35ac8d0..769f403 100755 --- a/include/fastfields/impl/cpu/pushpull.h +++ b/include/fastfields/impl/cpu/pushpull.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/parallel.h" #include "fastfields/impl/kernels/utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -18,7 +18,7 @@ FF_NAMESPACE_BEGIN(pushpull) // 0 : reject coordinates past the first/last voxel *centres* // -1 : reject coordinates past the first/last voxel *edges* template -inline CUDEV bool infov_dyn(int extrapolate, const scalar_t * loc, const offset_t * size) +inline FF_CUDEV bool infov_dyn(int extrapolate, const scalar_t * loc, const offset_t * size) { if (extrapolate > 0) return InFOV< 1, ndim>::infov(loc, size); if (extrapolate == 0) return InFOV< 0, ndim>::infov(loc, size); @@ -771,6 +771,6 @@ void grad_backward( FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PUSHPULL_CPU diff --git a/include/fastfields/impl/cpu/reg_field.h b/include/fastfields/impl/cpu/reg_field.h index 3619693..4fada7b 100755 --- a/include/fastfields/impl/cpu/reg_field.h +++ b/include/fastfields/impl/cpu/reg_field.h @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/regularisers/field.h" #include "fastfields/impl/kernels/posdef.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -1836,6 +1836,6 @@ void relax_bending_jrls_( FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FIELD_CPU diff --git a/include/fastfields/impl/cpu/reg_flow.h b/include/fastfields/impl/cpu/reg_flow.h index 5eedecd..0f075c0 100755 --- a/include/fastfields/impl/cpu/reg_flow.h +++ b/include/fastfields/impl/cpu/reg_flow.h @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/regularisers/flow.h" #include "fastfields/impl/kernels/posdef.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -1494,7 +1494,7 @@ void relax_lame_jrls_( FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FLOW_CPU diff --git a/include/fastfields/impl/cpu/resize.h b/include/fastfields/impl/cpu/resize.h index 6a35a3c..b9f55cf 100755 --- a/include/fastfields/impl/cpu/resize.h +++ b/include/fastfields/impl/cpu/resize.h @@ -5,9 +5,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -#define uchar_t unsigned char - -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(resize) @@ -62,8 +60,8 @@ void loopnd( const scalar_t * inp, // (*batch, *shape) tensor reduce_t shift, const reduce_t * _scale, // [*shape] vector - const uchar_t * _order, // [*shape] vector - const uchar_t * _bnd, // [*shape] vector + const unsigned char * _order, // [*shape] vector + const unsigned char * _bnd, // [*shape] vector const offset_t * size_out, // [*batch, *shape] vector const offset_t * size_inp, // [*batch, *shape] vector const offset_t * stride_out, // [*batch, *shape] vector @@ -97,6 +95,6 @@ void loopnd( FF_NAMESPACE_END(resize) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_RESIZE_LOOP diff --git a/include/fastfields/impl/cpu/restrict.h b/include/fastfields/impl/cpu/restrict.h index a75522e..505e6d5 100755 --- a/include/fastfields/impl/cpu/restrict.h +++ b/include/fastfields/impl/cpu/restrict.h @@ -5,9 +5,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -#define uchar_t unsigned char - -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(restrict) @@ -180,8 +178,8 @@ void loopnd( const scalar_t * inp, // (*batch, *shape) tensor reduce_t shift, const reduce_t * _scale, // [*shape] vector - const uchar_t * _order, // [*shape] vector - const uchar_t * _bnd, // [*shape] vector + const unsigned char * _order, // [*shape] vector + const unsigned char * _bnd, // [*shape] vector const offset_t * size_out, // [*batch, *shape] vector const offset_t * size_inp, // [*batch, *shape] vector const offset_t * stride_out, // [*batch, *shape] vector @@ -270,6 +268,6 @@ void loopnd( FF_NAMESPACE_END(restrict) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_RESTRICT_LOOP diff --git a/include/fastfields/impl/cpu/solve_field.h b/include/fastfields/impl/cpu/solve_field.h index b404c54..d6fd080 100644 --- a/include/fastfields/impl/cpu/solve_field.h +++ b/include/fastfields/impl/cpu/solve_field.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/batch.h" #include "fastfields/impl/kernels/parallel.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(solve_field) @@ -143,6 +143,6 @@ void axpby_( FF_NAMESPACE_END(solve_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_SOLVE_FIELD_CPU diff --git a/include/fastfields/impl/cpu/splinc.h b/include/fastfields/impl/cpu/splinc.h index f8b30d9..e723887 100755 --- a/include/fastfields/impl/cpu/splinc.h +++ b/include/fastfields/impl/cpu/splinc.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/kernels/parallel.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(splinc) @@ -40,6 +40,6 @@ void loop( FF_NAMESPACE_END(splinc) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_SPLINC_LOOP diff --git a/include/fastfields/impl/cuda/distance_euclidean.h b/include/fastfields/impl/cuda/distance_euclidean.h index acbab69..3708f88 100755 --- a/include/fastfields/impl/cuda/distance_euclidean.h +++ b/include/fastfields/impl/cuda/distance_euclidean.h @@ -6,13 +6,13 @@ #include #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_e) // CUDA kernel template -CUGLOB void dt_kernel( +FF_CUGLOB void dt_kernel( offset_t ndim , // number of dimensions scalar_t * f , // pointer to data [*batch, n] char * buf , // buffer (n*(offset_t + 2 * scalar_t)) @@ -47,7 +47,7 @@ CUGLOB void dt_kernel( // Templated entrypoint that launches the CUDA kernel template -CUHOST void dt( +FF_CUHOST void dt( offset_t ndim , // number of dimensions scalar_t * f , // pointer to data [*batch, n] scalar_t w , // pixel spacing @@ -90,4 +90,4 @@ CUHOST void dt( FF_NAMESPACE_END(distance_e) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/distance_l1.h b/include/fastfields/impl/cuda/distance_l1.h index e595e7b..6f7e86c 100755 --- a/include/fastfields/impl/cuda/distance_l1.h +++ b/include/fastfields/impl/cuda/distance_l1.h @@ -6,13 +6,13 @@ #include #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_l1) // CUDA kernel template -CUGLOB void dt_kernel( +FF_CUGLOB void dt_kernel( offset_t ndim , // number of dimensions scalar_t * f , // pointer to data [*batch, n] scalar_t w , // pixel spacing @@ -36,7 +36,7 @@ CUGLOB void dt_kernel( // Templated entrypoint that launches the CUDA kernel template -CUHOST void dt( +FF_CUHOST void dt( offset_t ndim , // number of dimensions scalar_t * f , // pointer to data [*batch, n] scalar_t w , // pixel spacing @@ -69,4 +69,4 @@ CUHOST void dt( FF_NAMESPACE_END(distance_l1) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/distance_mesh.h b/include/fastfields/impl/cuda/distance_mesh.h index ea81742..7775e5e 100755 --- a/include/fastfields/impl/cuda/distance_mesh.h +++ b/include/fastfields/impl/cuda/distance_mesh.h @@ -8,7 +8,7 @@ #include // std::unique_ptr #include // std::is_trivially_copyable -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_mesh) @@ -22,7 +22,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUHOST inline void +FF_CUHOST inline void build_tree( // (2M) array of *constructed* `Node` objects. It used to be a raw // `void*` byte buffer that was `reinterpret_cast` here; `Node` is a @@ -70,7 +70,7 @@ build_tree( // it to a *device* address space, where the host vtables do not exist at all, // is also concretely wrong -- the device would dispatch through host // pointers. So we do not transfer `Node` at all. Instead the host BVH builder -// (`MeshDist::build_tree`, unchanged and still `CUHOST`) writes real `Node` +// (`MeshDist::build_tree`, unchanged and still `FF_CUHOST`) writes real `Node` // objects into host memory, and a translation pass flattens them into this // plain struct, which *is* trivially copyable and is what actually crosses // the H2D boundary. @@ -100,7 +100,7 @@ struct DeviceNode // Flatten the host-built `Node` tree into the POD mirror that is uploaded. // One pass over the `2M` nodes; `nodes` must be a fully constructed array. template -CUHOST inline void +FF_CUHOST inline void flatten_tree( DeviceNode * out, const typename MeshDist::Node * nodes, @@ -144,7 +144,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUHOST inline void +FF_CUHOST inline void build_normals( scalar_t * _normfaces , // (M, D) tensor scalar_t * _normvertices , // (N, D) tensor @@ -185,11 +185,11 @@ build_normals( ***********************************************************************/ // Host-only: it returns an `allocHost` buffer, and its only caller -// (`copyTensorToContiguous`) is itself CUHOST. Declaring it CUHOSTDEV made +// (`copyTensorToContiguous`) is itself FF_CUHOST. Declaring it FF_CUHOSTDEV made // nvcc emit `warning #20014-D: calling a __host__ function from a // __host__ __device__ function is not allowed` for every instantiation. template -CUHOST inline offset_t * contiguousStrides(const offset_t * size, int ndim) +FF_CUHOST inline offset_t * contiguousStrides(const offset_t * size, int ndim) { offset_t * stride = allocHost(ndim); stride[ndim-1] = static_cast(1); @@ -199,7 +199,7 @@ CUHOST inline offset_t * contiguousStrides(const offset_t * size, int ndim) } template -CUGLOB inline void +FF_CUGLOB inline void copy_tensor_kernel( offset_t ndim, scalar_t * out, @@ -221,7 +221,7 @@ copy_tensor_kernel( } template -CUHOST inline +FF_CUHOST inline scalar_t * copyTensorToContiguous( offset_t ndim, const scalar_t * inp, @@ -270,7 +270,7 @@ template -CUGLOB inline void copy_faces_kernel( +FF_CUGLOB inline void copy_faces_kernel( offset_t nb_faces, // Number of faces (M) index_t * faces_out, // (M, D) output (contiguous) tensor of faces const index_t * faces_inp, // (M, D) input tensor of faces @@ -296,7 +296,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUHOST inline +FF_CUHOST inline index_t * copy_faces( offset_t nb_faces , const index_t * faces , @@ -344,7 +344,7 @@ template < typename NearestPoint, typename Point, typename Vertices, typename Faces, typename Trace > -CUDEV inline void +FF_CUDEV inline void query_dist_loop_pod( index_t & nearest_face, scalar_t & nearest_dist, @@ -534,7 +534,7 @@ template < typename offset_t, typename Point, typename Vertices, typename Faces, typename Trace > -CUDEV inline scalar_t +FF_CUDEV inline scalar_t unsigned_dist_pod( const Point & point, const Vertices & vertices, @@ -582,7 +582,7 @@ template < typename NormFaces, typename NormEdges, typename NormVertices, typename Trace > -CUDEV inline scalar_t +FF_CUDEV inline scalar_t signed_dist_pod( const Point & point, const Vertices & vertices, @@ -642,7 +642,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUGLOB inline void sdt_kernel( +FF_CUGLOB inline void sdt_kernel( offset_t nbatch , // Number of batch dimensions in coord scalar_t * dist , // (*batch) tensor -> Output placeholder for distance index_t * nearest_vertex , // (*batch) tensor -> Output placeholder for index of nearest vertex @@ -733,7 +733,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUGLOB inline void sdt_naive_kernel( +FF_CUGLOB inline void sdt_naive_kernel( offset_t nbatch , // Number of batch dimensions in coord scalar_t * dist , // (*batch) tensor -> Output placeholder for distance index_t * nearest_vertex , // (*batch) tensor -> Output placeholder for index of nearest vertex @@ -808,7 +808,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUGLOB inline void udt_kernel( +FF_CUGLOB inline void udt_kernel( offset_t nbatch , // Number of batch dimensions in coord scalar_t * dist , // (*batch) tensor -> Output placeholder for distance const scalar_t * coord , // (*batch, D) tensor -> Coordinates at which to evaluate distance @@ -867,7 +867,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUGLOB inline void udt_naive_kernel( +FF_CUGLOB inline void udt_naive_kernel( offset_t nbatch , // Number of batch dimensions in coord scalar_t * dist , // (*batch) tensor -> Output placeholder for distance const scalar_t * coord , // (*batch, D) tensor -> Coordinates at which to evaluate distance @@ -927,7 +927,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUHOST inline void +FF_CUHOST inline void sdt( offset_t nbatch, // Number of batch dimensions in coord scalar_t * dist, // (*batch) tensor -> Output placeholder for distance @@ -1244,7 +1244,7 @@ template < typename index_t, // Index (faces) data type typename offset_t // Index/Stride data type > -CUHOST inline void +FF_CUHOST inline void sdt_naive( offset_t nbatch, // Number of batch dimensions in coord scalar_t * dist, // (*batch) tensor -> Output placeholder for distance @@ -1427,7 +1427,7 @@ sdt_naive( } template -CUHOST inline void sdt( +FF_CUHOST inline void sdt( offset_t nbatch , // Number of batch dimensions in coord scalar_t * dist , // (*batch) tensor -> Output placeholder for distance index_t * nearest_vertex , // (*batch) tensor -> Output placeholder for index of nearest vertex @@ -1518,7 +1518,7 @@ CUHOST inline void sdt( // false true udt_naive udt_naive_kernel -> no launcher, throws // // The two unsigned device kernels exist and are type-checked by -// `tests/impl-cuda/compile_probe_mesh.cu`, but neither has a `CUHOST` launcher +// `tests/impl-cuda/compile_probe_mesh.cu`, but neither has a `FF_CUHOST` launcher // to upload the mesh and size the grid. Writing those is tracked separately; // until then those branches throw rather than silently returning garbage. See // fastfields-lib#5. @@ -1534,7 +1534,7 @@ CUHOST inline void sdt( // hardware is still the open acceptance bar of fastfields-lib#5. Same bar as // every other module in src/lib-cuda's MODULES. template -CUHOST inline void +FF_CUHOST inline void dt( offset_t nbatch, scalar_t * dist, @@ -1596,4 +1596,4 @@ dt( FF_NAMESPACE_END(distance_mesh) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/distance_spline.h b/include/fastfields/impl/cuda/distance_spline.h index 9725569..0a91431 100755 --- a/include/fastfields/impl/cuda/distance_spline.h +++ b/include/fastfields/impl/cuda/distance_spline.h @@ -6,7 +6,7 @@ #include "fastfields/impl/kernels/utils.h" using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_spline) @@ -20,7 +20,7 @@ template < typename scalar_t, // Value data type typename offset_t // Index/Stride data type > -CUGLOB void mindist_table( +FF_CUGLOB void mindist_table( scalar_t * time, // (*batch) tensor -> Best time scalar_t * dist, // (*batch) tensor -> Best sqdist const scalar_t * loc, // (*batch, ndim) tensor -> ND location of each point @@ -81,7 +81,7 @@ template < typename scalar_t, // Value data type typename offset_t // Index/Stride data type > -CUGLOB void mindist_brent( +FF_CUGLOB void mindist_brent( scalar_t * time, // (*batch) tensor -> Best time scalar_t * dist, // (*batch) tensor -> Best sqdist const scalar_t * loc, // (*batch, ndim) tensor -> ND location of each point @@ -141,7 +141,7 @@ template < typename scalar_t, // Value data type typename offset_t // Index/Stride data type > -CUGLOB void mindist_gaussnewton( +FF_CUGLOB void mindist_gaussnewton( scalar_t * time, // (*batch) tensor -> Best time scalar_t * dist, // (*batch) tensor -> Best sqdist const scalar_t * loc, // (*batch, ndim) tensor -> ND location of each point @@ -189,13 +189,13 @@ CUGLOB void mindist_gaussnewton( } // --------------------------------------------------------------------------- -// Host launchers (mirror cpu-impl distance_spline). These wrap the CUGLOB +// Host launchers (mirror cpu-impl distance_spline). These wrap the FF_CUGLOB // kernels above (device shape/stride copy, grid config, launch, stream). Not // implemented yet — the CUDA point-to-spline launchers are pending. Provided // with the cpu-impl signatures so the cuda-lib dispatch layer compiles + links; // they throw until authored. Compile-verified under nvcc; runtime needs a GPU. template -CUHOST inline void +FF_CUHOST inline void mindist_table( offset_t nbatch, scalar_t* /*time*/, scalar_t* /*dist*/, const scalar_t* /*loc*/, const scalar_t* /*coeff*/, const scalar_t* /*times*/, @@ -210,7 +210,7 @@ mindist_table( } template -CUHOST inline void +FF_CUHOST inline void mindist_brent( offset_t nbatch, scalar_t* /*time*/, scalar_t* /*dist*/, const scalar_t* /*loc*/, const scalar_t* /*coeff*/, const offset_t* /*size*/, @@ -224,7 +224,7 @@ mindist_brent( } template -CUHOST inline void +FF_CUHOST inline void mindist_gaussnewton( offset_t nbatch, scalar_t* /*time*/, scalar_t* /*dist*/, const scalar_t* /*loc*/, const scalar_t* /*coeff*/, const offset_t* /*size*/, @@ -239,4 +239,4 @@ mindist_gaussnewton( FF_NAMESPACE_END(distance_spline) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/posdef.h b/include/fastfields/impl/cuda/posdef.h index 379a4b7..9061096 100755 --- a/include/fastfields/impl/cuda/posdef.h +++ b/include/fastfields/impl/cuda/posdef.h @@ -7,7 +7,7 @@ #include using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(posdef) @@ -19,7 +19,7 @@ FF_NAMESPACE_BEGIN(posdef) #endif template -CUGLOB +FF_CUGLOB void sym_matvec_k(scalar_t * out, const scalar_t * hes, const scalar_t * inp, const offset_t * size, const offset_t * stride_out, @@ -49,7 +49,7 @@ void sym_matvec_k(scalar_t * out, const scalar_t * hes, const scalar_t * inp, } template -CUGLOB +FF_CUGLOB void sym_matvec_backward_k( scalar_t * out, const scalar_t * grd, const scalar_t * inp, const offset_t * size, @@ -80,7 +80,7 @@ void sym_matvec_backward_k( } template -CUGLOB +FF_CUGLOB void sym_addmatvec__k(scalar_t * out, const scalar_t * hes, const scalar_t * inp, const offset_t * size, const offset_t * stride_out, @@ -110,7 +110,7 @@ void sym_addmatvec__k(scalar_t * out, const scalar_t * hes, const scalar_t * inp } template -CUGLOB +FF_CUGLOB void sym_submatvec__k(scalar_t * out, const scalar_t * hes, const scalar_t * inp, const offset_t * size, const offset_t * stride_out, @@ -141,7 +141,7 @@ void sym_submatvec__k(scalar_t * out, const scalar_t * hes, const scalar_t * inp template -CUGLOB +FF_CUGLOB void sym_solve_k(scalar_t * out, const scalar_t * inp, const scalar_t * hes, const scalar_t * wgt, const offset_t * size, @@ -179,7 +179,7 @@ void sym_solve_k(scalar_t * out, const scalar_t * inp, template -CUGLOB +FF_CUGLOB void sym_solve__k(scalar_t * out, const scalar_t * hes, const scalar_t * wgt, const offset_t * size, @@ -212,7 +212,7 @@ void sym_solve__k(scalar_t * out, } template -CUGLOB +FF_CUGLOB void sym_invert_k(scalar_t * out, const scalar_t * hes, const offset_t * size, const offset_t * stride_out, @@ -240,7 +240,7 @@ void sym_invert_k(scalar_t * out, const scalar_t * hes, } template -CUGLOB +FF_CUGLOB void sym_invert__k(scalar_t * hes, const offset_t * size, const offset_t * stride) @@ -309,7 +309,7 @@ void sym_invert__k(scalar_t * hes, const int threads = CUDA_NUM_THREADS template -CUHOST +FF_CUHOST void sym_matvec( offset_t nbatch, offset_t nchannel, @@ -341,7 +341,7 @@ void sym_matvec( } template -CUHOST +FF_CUHOST void sym_matvec_backward( offset_t nbatch, offset_t nchannel, @@ -373,7 +373,7 @@ void sym_matvec_backward( } template -CUHOST +FF_CUHOST void sym_addmatvec_( offset_t nbatch, offset_t nchannel, @@ -405,7 +405,7 @@ void sym_addmatvec_( } template -CUHOST +FF_CUHOST void sym_submatvec_( offset_t nbatch, offset_t nchannel, @@ -437,7 +437,7 @@ void sym_submatvec_( } template -CUHOST +FF_CUHOST void sym_solve( offset_t nbatch, offset_t nchannel, @@ -473,7 +473,7 @@ void sym_solve( } template -CUHOST +FF_CUHOST void sym_solve_( offset_t nbatch, offset_t nchannel, @@ -505,7 +505,7 @@ void sym_solve_( } template -CUHOST +FF_CUHOST void sym_invert( offset_t nbatch, offset_t nchannel, @@ -534,7 +534,7 @@ void sym_invert( } template -CUHOST +FF_CUHOST void sym_invert_( offset_t nbatch, offset_t nchannel, @@ -564,4 +564,4 @@ void sym_invert_( FF_NAMESPACE_END(posdef) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/pushpull.h b/include/fastfields/impl/cuda/pushpull.h index eda1e3c..bdf79a7 100755 --- a/include/fastfields/impl/cuda/pushpull.h +++ b/include/fastfields/impl/cuda/pushpull.h @@ -9,7 +9,7 @@ #include // std::logic_error using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -24,7 +24,7 @@ FF_NAMESPACE_BEGIN(pushpull) // 0 : reject coordinates past the first/last voxel *centres* // -1 : reject coordinates past the first/last voxel *edges* template -inline CUDEV bool infov_dyn(int extrapolate, const scalar_t * loc, const offset_t * size) +inline FF_CUDEV bool infov_dyn(int extrapolate, const scalar_t * loc, const offset_t * size) { if (extrapolate > 0) return InFOV< 1, ndim>::infov(loc, size); if (extrapolate == 0) return InFOV< 0, ndim>::infov(loc, size); @@ -36,7 +36,7 @@ template -CUGLOB +FF_CUGLOB void pull( bound::BoundVec bnd, spline::SplineVec spl, @@ -99,7 +99,7 @@ template -CUGLOB +FF_CUGLOB void push( bound::BoundVec bnd, spline::SplineVec spl, @@ -159,7 +159,7 @@ template -CUGLOB +FF_CUGLOB void count( bound::BoundVec bnd, spline::SplineVec spl, @@ -211,7 +211,7 @@ template -CUGLOB +FF_CUGLOB void grad( bound::BoundVec bnd, spline::SplineVec spl, @@ -276,7 +276,7 @@ template -CUGLOB +FF_CUGLOB void hess( bound::BoundVec bnd, spline::SplineVec spl, @@ -341,7 +341,7 @@ template -CUGLOB +FF_CUGLOB void pull_backward( bound::BoundVec bnd, spline::SplineVec spl, @@ -421,7 +421,7 @@ template -CUGLOB +FF_CUGLOB void push_backward( bound::BoundVec bnd, spline::SplineVec spl, @@ -510,7 +510,7 @@ template -CUGLOB +FF_CUGLOB void count_backward( bound::BoundVec bnd, spline::SplineVec spl, @@ -573,7 +573,7 @@ template -CUGLOB +FF_CUGLOB void grad_backward( bound::BoundVec bnd, spline::SplineVec spl, @@ -661,7 +661,7 @@ void grad_backward( * HOST LAUNCHERS * * * * These mirror the fastfields-cpu-impl pushpull launchers, but launch * - * the CUGLOB kernels above over the grid. `nbatch`, `extrapolate` and * + * the FF_CUGLOB kernels above over the grid. `nbatch`, `extrapolate` and * * the bound/spline conditions are runtime arguments here exactly as * * on the CPU side; only `ndim`, `abs` and whichever bound/spline axes * * the build compiles statically remain compile-time template * @@ -713,7 +713,7 @@ void grad_backward( } // int -> cudaStream_t (0 == default stream). -CUHOST inline cudaStream_t _pp_stream(intptr_t stream) +FF_CUHOST inline cudaStream_t _pp_stream(intptr_t stream) { return reinterpret_cast(static_cast(stream)); } @@ -723,7 +723,7 @@ template -CUHOST void pull( +FF_CUHOST void pull( offset_t nbatch, int extrapolate, scalar_t * out, @@ -775,7 +775,7 @@ template -CUHOST void push( +FF_CUHOST void push( offset_t nbatch, int extrapolate, scalar_t * out, // must be pre-zeroed by the caller @@ -827,7 +827,7 @@ template -CUHOST void count( +FF_CUHOST void count( offset_t nbatch, int extrapolate, scalar_t * out, // must be pre-zeroed by the caller @@ -876,7 +876,7 @@ template -CUHOST void grad( +FF_CUHOST void grad( offset_t nbatch, int extrapolate, scalar_t * out, @@ -933,7 +933,7 @@ template -CUHOST void pull_backward( +FF_CUHOST void pull_backward( offset_t nbatch, int extrapolate, scalar_t * out, // must be pre-zeroed by the caller @@ -993,7 +993,7 @@ template -CUHOST void push_backward( +FF_CUHOST void push_backward( offset_t nbatch, int extrapolate, scalar_t * out, @@ -1053,7 +1053,7 @@ template -CUHOST void count_backward( +FF_CUHOST void count_backward( offset_t nbatch, int extrapolate, scalar_t * gout, @@ -1106,7 +1106,7 @@ template -CUHOST void grad_backward( +FF_CUHOST void grad_backward( offset_t nbatch, int extrapolate, scalar_t * out, // must be pre-zeroed by the caller @@ -1166,4 +1166,4 @@ CUHOST void grad_backward( FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/reg_field.h b/include/fastfields/impl/cuda/reg_field.h index cd6b33f..51f817d 100755 --- a/include/fastfields/impl/cuda/reg_field.h +++ b/include/fastfields/impl/cuda/reg_field.h @@ -9,7 +9,7 @@ #include // std::logic_error using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -22,7 +22,7 @@ FF_NAMESPACE_BEGIN(reg_field) template -CUGLOB +FF_CUGLOB void matvec_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -66,7 +66,7 @@ void matvec_absolute( template -CUGLOB +FF_CUGLOB void kernel_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -106,7 +106,7 @@ void kernel_absolute( template -CUGLOB +FF_CUGLOB void diag_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -148,7 +148,7 @@ void diag_absolute( template -CUGLOB +FF_CUGLOB void matvec_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -196,7 +196,7 @@ void matvec_membrane( template -CUGLOB +FF_CUGLOB void kernel_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -240,7 +240,7 @@ void kernel_membrane( template -CUGLOB +FF_CUGLOB void diag_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -282,7 +282,7 @@ void diag_membrane( template -CUGLOB +FF_CUGLOB void relax_membrane_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -365,7 +365,7 @@ void relax_membrane_( template -CUGLOB +FF_CUGLOB void matvec_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -414,7 +414,7 @@ void matvec_bending( template -CUGLOB +FF_CUGLOB void kernel_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -459,7 +459,7 @@ void kernel_bending( template -CUGLOB +FF_CUGLOB void diag_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -502,7 +502,7 @@ void diag_bending( template -CUGLOB +FF_CUGLOB void relax_bending_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -586,7 +586,7 @@ void relax_bending_( template -CUGLOB +FF_CUGLOB void matvec_absolute_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -635,7 +635,7 @@ void matvec_absolute_rls( template -CUGLOB +FF_CUGLOB void diag_absolute_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -678,7 +678,7 @@ void diag_absolute_rls( template -CUGLOB +FF_CUGLOB void relax_absolute_rls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -760,7 +760,7 @@ void relax_absolute_rls_( template -CUGLOB +FF_CUGLOB void matvec_absolute_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -808,7 +808,7 @@ void matvec_absolute_jrls( template -CUGLOB +FF_CUGLOB void diag_absolute_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -850,7 +850,7 @@ void diag_absolute_jrls( template -CUGLOB +FF_CUGLOB void relax_absolute_jrls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -931,7 +931,7 @@ void relax_absolute_jrls_( template -CUGLOB +FF_CUGLOB void matvec_membrane_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -986,7 +986,7 @@ void matvec_membrane_rls( template -CUGLOB +FF_CUGLOB void diag_membrane_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1034,7 +1034,7 @@ void diag_membrane_rls( template -CUGLOB +FF_CUGLOB void relax_membrane_rls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1124,7 +1124,7 @@ void relax_membrane_rls_( template -CUGLOB +FF_CUGLOB void matvec_membrane_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -1178,7 +1178,7 @@ void matvec_membrane_jrls( template -CUGLOB +FF_CUGLOB void diag_membrane_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1225,7 +1225,7 @@ void diag_membrane_jrls( template -CUGLOB +FF_CUGLOB void relax_membrane_jrls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1314,7 +1314,7 @@ void relax_membrane_jrls_( template -CUGLOB +FF_CUGLOB void matvec_bending_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -1369,7 +1369,7 @@ void matvec_bending_rls( template -CUGLOB +FF_CUGLOB void diag_bending_rls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1418,7 +1418,7 @@ void diag_bending_rls( template -CUGLOB +FF_CUGLOB void relax_bending_rls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1509,7 +1509,7 @@ void relax_bending_rls_( template -CUGLOB +FF_CUGLOB void matvec_bending_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -1563,7 +1563,7 @@ void matvec_bending_jrls( template -CUGLOB +FF_CUGLOB void diag_bending_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1611,7 +1611,7 @@ void diag_bending_jrls( template -CUGLOB +FF_CUGLOB void relax_bending_jrls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1698,7 +1698,7 @@ void relax_bending_jrls_( // // The device kernels above are templated on a *compile-time* number of batch // dimensions (`nbatch`) and a *compile-time* channel count (`C`). The cuda-lib -// dispatch layer only knows both at runtime, so these CUHOST launchers: +// dispatch layer only knows both at runtime, so these FF_CUHOST launchers: // 1. copy the (host) shape / stride / voxel-size / weight vectors to the // device, // 2. dispatch the runtime (`nbatch`, `nc`) pair to a bounded set of @@ -1747,7 +1747,7 @@ void relax_bending_jrls_( template -CUHOST void matvec_absolute( +FF_CUHOST void matvec_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1781,7 +1781,7 @@ CUHOST void matvec_absolute( template -CUHOST void diag_absolute( +FF_CUHOST void diag_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1814,7 +1814,7 @@ CUHOST void diag_absolute( template -CUHOST void matvec_membrane( +FF_CUHOST void matvec_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1852,7 +1852,7 @@ CUHOST void matvec_membrane( template -CUHOST void diag_membrane( +FF_CUHOST void diag_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1889,7 +1889,7 @@ CUHOST void diag_membrane( template -CUHOST void matvec_bending( +FF_CUHOST void matvec_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1929,7 +1929,7 @@ CUHOST void matvec_bending( template -CUHOST void diag_bending( +FF_CUHOST void diag_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1973,7 +1973,7 @@ CUHOST void diag_bending( template -CUHOST void kernel_absolute( +FF_CUHOST void kernel_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2004,7 +2004,7 @@ CUHOST void kernel_absolute( template -CUHOST void kernel_membrane( +FF_CUHOST void kernel_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2039,7 +2039,7 @@ CUHOST void kernel_membrane( template -CUHOST void kernel_bending( +FF_CUHOST void kernel_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2106,7 +2106,7 @@ CUHOST void kernel_bending( template -CUHOST void relax_membrane_( +FF_CUHOST void relax_membrane_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2149,7 +2149,7 @@ CUHOST void relax_membrane_( template -CUHOST void relax_bending_( +FF_CUHOST void relax_bending_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2227,7 +2227,7 @@ CUHOST void relax_bending_( template -CUHOST void matvec_absolute_rls( +FF_CUHOST void matvec_absolute_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2264,7 +2264,7 @@ CUHOST void matvec_absolute_rls( template -CUHOST void diag_absolute_rls( +FF_CUHOST void diag_absolute_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2297,7 +2297,7 @@ CUHOST void diag_absolute_rls( template -CUHOST void relax_absolute_rls_( +FF_CUHOST void relax_absolute_rls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2341,7 +2341,7 @@ CUHOST void relax_absolute_rls_( template -CUHOST void matvec_absolute_jrls( +FF_CUHOST void matvec_absolute_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2378,7 +2378,7 @@ CUHOST void matvec_absolute_jrls( template -CUHOST void diag_absolute_jrls( +FF_CUHOST void diag_absolute_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2411,7 +2411,7 @@ CUHOST void diag_absolute_jrls( template -CUHOST void relax_absolute_jrls_( +FF_CUHOST void relax_absolute_jrls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2455,7 +2455,7 @@ CUHOST void relax_absolute_jrls_( template -CUHOST void matvec_membrane_rls( +FF_CUHOST void matvec_membrane_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2498,7 +2498,7 @@ CUHOST void matvec_membrane_rls( template -CUHOST void diag_membrane_rls( +FF_CUHOST void diag_membrane_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2535,7 +2535,7 @@ CUHOST void diag_membrane_rls( template -CUHOST void relax_membrane_rls_( +FF_CUHOST void relax_membrane_rls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2584,7 +2584,7 @@ CUHOST void relax_membrane_rls_( template -CUHOST void matvec_membrane_jrls( +FF_CUHOST void matvec_membrane_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2627,7 +2627,7 @@ CUHOST void matvec_membrane_jrls( template -CUHOST void diag_membrane_jrls( +FF_CUHOST void diag_membrane_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2664,7 +2664,7 @@ CUHOST void diag_membrane_jrls( template -CUHOST void relax_membrane_jrls_( +FF_CUHOST void relax_membrane_jrls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2713,7 +2713,7 @@ CUHOST void relax_membrane_jrls_( template -CUHOST void matvec_bending_rls( +FF_CUHOST void matvec_bending_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2756,7 +2756,7 @@ CUHOST void matvec_bending_rls( template -CUHOST void diag_bending_rls( +FF_CUHOST void diag_bending_rls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2795,7 +2795,7 @@ CUHOST void diag_bending_rls( template -CUHOST void relax_bending_rls_( +FF_CUHOST void relax_bending_rls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2846,7 +2846,7 @@ CUHOST void relax_bending_rls_( template -CUHOST void matvec_bending_jrls( +FF_CUHOST void matvec_bending_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2889,7 +2889,7 @@ CUHOST void matvec_bending_jrls( template -CUHOST void diag_bending_jrls( +FF_CUHOST void diag_bending_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2928,7 +2928,7 @@ CUHOST void diag_bending_jrls( template -CUHOST void relax_bending_jrls_( +FF_CUHOST void relax_bending_jrls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, @@ -2981,4 +2981,4 @@ CUHOST void relax_bending_jrls_( FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/reg_flow.h b/include/fastfields/impl/cuda/reg_flow.h index 00cf695..fd58a60 100755 --- a/include/fastfields/impl/cuda/reg_flow.h +++ b/include/fastfields/impl/cuda/reg_flow.h @@ -9,7 +9,7 @@ #include // std::logic_error using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -22,7 +22,7 @@ FF_NAMESPACE_BEGIN(reg_flow) template -CUGLOB +FF_CUGLOB void matvec_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -68,7 +68,7 @@ void matvec_absolute( template -CUGLOB +FF_CUGLOB void kernel_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -110,7 +110,7 @@ void kernel_absolute( template -CUGLOB +FF_CUGLOB void diag_absolute( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -154,7 +154,7 @@ void diag_absolute( template -CUGLOB +FF_CUGLOB void matvec_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -201,7 +201,7 @@ void matvec_membrane( template -CUGLOB +FF_CUGLOB void kernel_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -244,7 +244,7 @@ void kernel_membrane( template -CUGLOB +FF_CUGLOB void diag_membrane( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -285,7 +285,7 @@ void diag_membrane( template -CUGLOB +FF_CUGLOB void relax_membrane_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -368,7 +368,7 @@ void relax_membrane_( template -CUGLOB +FF_CUGLOB void matvec_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -415,7 +415,7 @@ void matvec_bending( template -CUGLOB +FF_CUGLOB void kernel_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -458,7 +458,7 @@ void kernel_bending( template -CUGLOB +FF_CUGLOB void diag_bending( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -499,7 +499,7 @@ void diag_bending( template -CUGLOB +FF_CUGLOB void relax_bending_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -582,7 +582,7 @@ void relax_bending_( template -CUGLOB +FF_CUGLOB void matvec_lame( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -629,7 +629,7 @@ void matvec_lame( template -CUGLOB +FF_CUGLOB void kernel_lame( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C, C) tensor @@ -671,7 +671,7 @@ void kernel_lame( template -CUGLOB +FF_CUGLOB void diag_lame( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -711,7 +711,7 @@ void diag_lame( template -CUGLOB +FF_CUGLOB void relax_lame_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -794,7 +794,7 @@ void relax_lame_( template -CUGLOB +FF_CUGLOB void matvec_all( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -842,7 +842,7 @@ void matvec_all( template -CUGLOB +FF_CUGLOB void kernel_all( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C, C) tensor @@ -885,7 +885,7 @@ void kernel_all( template -CUGLOB +FF_CUGLOB void diag_all( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -927,7 +927,7 @@ void diag_all( template -CUGLOB +FF_CUGLOB void relax_all_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1011,7 +1011,7 @@ void relax_all_( template -CUGLOB +FF_CUGLOB void matvec_membrane_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -1063,7 +1063,7 @@ void matvec_membrane_jrls( template -CUGLOB +FF_CUGLOB void diag_membrane_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1109,7 +1109,7 @@ void diag_membrane_jrls( template -CUGLOB +FF_CUGLOB void relax_membrane_jrls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1198,7 +1198,7 @@ void relax_membrane_jrls_( template -CUGLOB +FF_CUGLOB void matvec_lame_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, C) tensor @@ -1250,7 +1250,7 @@ void matvec_lame_jrls( template -CUGLOB +FF_CUGLOB void diag_lame_jrls( bound::BoundVec bnd, scalar_t * out, // (*batch, *spatial, channels) tensor @@ -1296,7 +1296,7 @@ void diag_lame_jrls( template -CUGLOB +FF_CUGLOB void relax_lame_jrls_( bound::BoundVec bnd, scalar_t * sol, // (*batch, *spatial, C) tensor @@ -1382,7 +1382,7 @@ void relax_lame_jrls_( // // The device kernels above are templated on a *compile-time* number of batch // dimensions (`nbatch`). The cuda-lib dispatch layer only knows `nbatch` at -// runtime, so these CUHOST launchers: +// runtime, so these FF_CUHOST launchers: // 1. copy the (host) shape / stride / voxel-size vectors to the device, // 2. dispatch the runtime `nbatch` to a bounded set of compile-time // instantiations of the matching device kernel, @@ -1438,7 +1438,7 @@ void relax_lame_jrls_( template -CUHOST void matvec_absolute( +FF_CUHOST void matvec_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1472,7 +1472,7 @@ CUHOST void matvec_absolute( template -CUHOST void diag_absolute( +FF_CUHOST void diag_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1505,7 +1505,7 @@ CUHOST void diag_absolute( template -CUHOST void matvec_membrane( +FF_CUHOST void matvec_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1540,7 +1540,7 @@ CUHOST void matvec_membrane( template -CUHOST void diag_membrane( +FF_CUHOST void diag_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1574,7 +1574,7 @@ CUHOST void diag_membrane( template -CUHOST void matvec_bending( +FF_CUHOST void matvec_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1610,7 +1610,7 @@ CUHOST void matvec_bending( template -CUHOST void diag_bending( +FF_CUHOST void diag_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1645,7 +1645,7 @@ CUHOST void diag_bending( template -CUHOST void matvec_all( +FF_CUHOST void matvec_all( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1684,7 +1684,7 @@ CUHOST void matvec_all( template -CUHOST void diag_all( +FF_CUHOST void diag_all( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1731,7 +1731,7 @@ CUHOST void diag_all( template -CUHOST void matvec_membrane_jrls( +FF_CUHOST void matvec_membrane_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1769,7 +1769,7 @@ CUHOST void matvec_membrane_jrls( template -CUHOST void diag_membrane_jrls( +FF_CUHOST void diag_membrane_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1806,7 +1806,7 @@ CUHOST void diag_membrane_jrls( template -CUHOST void matvec_lame_jrls( +FF_CUHOST void matvec_lame_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1847,7 +1847,7 @@ CUHOST void matvec_lame_jrls( template -CUHOST void diag_lame_jrls( +FF_CUHOST void diag_lame_jrls( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1893,7 +1893,7 @@ CUHOST void diag_lame_jrls( template -CUHOST void kernel_absolute( +FF_CUHOST void kernel_absolute( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1924,7 +1924,7 @@ CUHOST void kernel_absolute( template -CUHOST void kernel_membrane( +FF_CUHOST void kernel_membrane( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1956,7 +1956,7 @@ CUHOST void kernel_membrane( template -CUHOST void kernel_bending( +FF_CUHOST void kernel_bending( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -1989,7 +1989,7 @@ CUHOST void kernel_bending( template -CUHOST void kernel_lame( +FF_CUHOST void kernel_lame( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2023,7 +2023,7 @@ CUHOST void kernel_lame( template -CUHOST void kernel_all( +FF_CUHOST void kernel_all( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * out, @@ -2068,7 +2068,7 @@ CUHOST void kernel_all( template -CUHOST void relax_membrane_( +FF_CUHOST void relax_membrane_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const offset_t * size, const offset_t * stride_sol, @@ -2100,7 +2100,7 @@ CUHOST void relax_membrane_( template -CUHOST void relax_bending_( +FF_CUHOST void relax_bending_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const offset_t * size, const offset_t * stride_sol, @@ -2132,7 +2132,7 @@ CUHOST void relax_bending_( template -CUHOST void relax_lame_( +FF_CUHOST void relax_lame_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const offset_t * size, const offset_t * stride_sol, @@ -2164,7 +2164,7 @@ CUHOST void relax_lame_( template -CUHOST void relax_all_( +FF_CUHOST void relax_all_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const offset_t * size, const offset_t * stride_sol, @@ -2205,7 +2205,7 @@ CUHOST void relax_all_( template -CUHOST void relax_membrane_jrls_( +FF_CUHOST void relax_membrane_jrls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const scalar_t * wgt, @@ -2240,7 +2240,7 @@ CUHOST void relax_membrane_jrls_( template -CUHOST void relax_lame_jrls_( +FF_CUHOST void relax_lame_jrls_( const bound::BoundVec & bnd, offset_t nbatch, scalar_t * sol, const scalar_t * hes, const scalar_t * grd, const scalar_t * wgt, @@ -2279,4 +2279,4 @@ CUHOST void relax_lame_jrls_( FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/resize.h b/include/fastfields/impl/cuda/resize.h index 3ce1466..4214b26 100755 --- a/include/fastfields/impl/cuda/resize.h +++ b/include/fastfields/impl/cuda/resize.h @@ -14,7 +14,7 @@ #include using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(resize) @@ -30,7 +30,7 @@ template -CUGLOB +FF_CUGLOB void kernel( scalar_t * out, // (*batch, *shape) tensor const scalar_t * inp, // (*batch, *shape) tensor @@ -68,7 +68,7 @@ void kernel( template -CUGLOB +FF_CUGLOB void kernelnd( scalar_t * out, // (*batch, *shape) tensor const scalar_t * inp, // (*batch, *shape) tensor @@ -124,7 +124,7 @@ template < spline::type IY=IX, bound::type BY=BX, spline::type IZ=IY, bound::type BZ=BY > -CUHOST +FF_CUHOST void loop( offset_t nbatch, scalar_t * out, @@ -186,4 +186,4 @@ void loop( FF_NAMESPACE_END(resize) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/restrict.h b/include/fastfields/impl/cuda/restrict.h index c5bba01..0d58f38 100755 --- a/include/fastfields/impl/cuda/restrict.h +++ b/include/fastfields/impl/cuda/restrict.h @@ -15,7 +15,7 @@ #include using namespace std; -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(restrict) @@ -32,7 +32,7 @@ template -CUGLOB +FF_CUGLOB void kernel( scalar_t * out, // (*batch, *shape) tensor const scalar_t * inp, // (*batch, *shape) tensor @@ -110,7 +110,7 @@ template -CUGLOB +FF_CUGLOB void kernel2( scalar_t * out, // (*batch, *shape) tensor const scalar_t * inp, // (*batch, *shape) tensor @@ -128,7 +128,7 @@ void kernel2( template -CUGLOB +FF_CUGLOB void kernelnd( scalar_t * out, // (*batch, *shape) tensor const scalar_t * inp, // (*batch, *shape) tensor @@ -200,7 +200,7 @@ template < spline::type IY=IX, bound::type BY=BX, spline::type IZ=IY, bound::type BZ=BY > -CUHOST +FF_CUHOST void loop( offset_t nbatch, scalar_t * out, @@ -261,4 +261,4 @@ void loop( FF_NAMESPACE_END(restrict) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/splinc.h b/include/fastfields/impl/cuda/splinc.h index 64fe38d..07cc934 100755 --- a/include/fastfields/impl/cuda/splinc.h +++ b/include/fastfields/impl/cuda/splinc.h @@ -7,7 +7,7 @@ #include #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(splinc) @@ -20,7 +20,7 @@ FF_NAMESPACE_BEGIN(splinc) template -CUGLOB +FF_CUGLOB void kernel( scalar_t * inp, const offset_t * _size, @@ -50,7 +50,7 @@ void kernel( // host array of length npoles; `inp` is device memory. template -CUHOST +FF_CUHOST void loop( offset_t nbatch, scalar_t * inp, @@ -105,4 +105,4 @@ void loop( FF_NAMESPACE_END(splinc) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/cuda/utils.h b/include/fastfields/impl/cuda/utils.h index 7d70dfd..54d2901 100644 --- a/include/fastfields/impl/cuda/utils.h +++ b/include/fastfields/impl/cuda/utils.h @@ -5,7 +5,7 @@ #include // std::range_error #include // std::numeric_limits -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -16,7 +16,7 @@ FF_NAMESPACE_BEGIN(FF_DEVICE) static constexpr int CUDA_NUM_THREADS = 1024; // Set the number of blocks for CUDA kernel launches. (Copied from PyTorch) -CUHOST inline int +FF_CUHOST inline int GET_BLOCKS( const int64_t N, const int64_t max_threads_per_block = CUDA_NUM_THREADS @@ -42,21 +42,21 @@ GET_BLOCKS( template struct _CudaBuffers { - CUHOST static inline void freeDevice(U...) {} - CUHOST static inline void freeHost (U...) {} + FF_CUHOST static inline void freeDevice(U...) {} + FF_CUHOST static inline void freeHost (U...) {} }; template struct _CudaBuffers { - CUHOST static inline void freeDevice(U0 buffer0, U... buffers) + FF_CUHOST static inline void freeDevice(U0 buffer0, U... buffers) { _CudaBuffers::freeDevice(buffer0); _CudaBuffers::freeDevice(buffers...); } - CUHOST static inline void freeHost(U0 buffer0, U... buffers) + FF_CUHOST static inline void freeHost(U0 buffer0, U... buffers) { _CudaBuffers::freeHost(buffer0); @@ -67,13 +67,13 @@ struct _CudaBuffers template struct _CudaBuffers { - CUHOST static inline void freeDevice(U0 buffer0) + FF_CUHOST static inline void freeDevice(U0 buffer0) { if (buffer0) cudaFree(static_cast(buffer0)); } - CUHOST static inline void freeHost(U0 buffer0) + FF_CUHOST static inline void freeHost(U0 buffer0) { if (buffer0) cudaFreeHost(static_cast(buffer0)); @@ -81,19 +81,19 @@ struct _CudaBuffers }; template -CUHOST inline void freeDevice(U... buffers) +FF_CUHOST inline void freeDevice(U... buffers) { return _CudaBuffers::freeDevice(buffers...); } template -CUHOST inline void freeHost(U... buffers) +FF_CUHOST inline void freeHost(U... buffers) { return _CudaBuffers::freeHost(buffers...); } template -CUHOST inline void error(F exc, const char * msg) +FF_CUHOST inline void error(F exc, const char * msg) { throw exc(msg); } @@ -109,7 +109,7 @@ template struct ff_is_same { static constexpr bool value = true; }; template -CUHOST inline O * allocDevice(S size) +FF_CUHOST inline O * allocDevice(S size) { O * out = nullptr; cudaError_t err = cudaMalloc(reinterpret_cast(&out), size * sizeof(O)); @@ -118,7 +118,7 @@ CUHOST inline O * allocDevice(S size) } template -CUHOST inline O * allocHost(S size) +FF_CUHOST inline O * allocHost(S size) { O * out = nullptr; cudaError_t err = cudaMallocHost(reinterpret_cast(&out), size * sizeof(O)); @@ -131,7 +131,7 @@ CUHOST inline O * allocHost(S size) ***********************************************************************/ template -CUHOST inline O * copyToDevice(const I * inp, S size, O * out = nullptr) +FF_CUHOST inline O * copyToDevice(const I * inp, S size, O * out = nullptr) { cudaError_t err; constexpr bool needs_tmp = ff_is_same::value; @@ -179,7 +179,7 @@ CUHOST inline O * copyToDevice(const I * inp, S size, O * out = nullptr) } template -CUHOST inline I * copyToDevice(const I * inp, S size, I * out = nullptr) +FF_CUHOST inline I * copyToDevice(const I * inp, S size, I * out = nullptr) { return copyToDevice(inp, size, out); } @@ -209,7 +209,7 @@ CUHOST inline I * copyToDevice(const I * inp, S size, I * out = nullptr) // before the free. Only the metadata upload is waited on; the caller // still gets stream-ordered kernel execution. template -CUHOST inline O * copyToDeviceAsync( +FF_CUHOST inline O * copyToDeviceAsync( const I * inp, S size, cudaStream_t stream, O * out = nullptr) { cudaError_t err; @@ -269,7 +269,7 @@ CUHOST inline O * copyToDeviceAsync( } template -CUHOST inline I * copyToDeviceAsync( +FF_CUHOST inline I * copyToDeviceAsync( const I * inp, S size, cudaStream_t stream, I * out = nullptr) { return copyToDeviceAsync(inp, size, stream, out); @@ -277,7 +277,7 @@ CUHOST inline I * copyToDeviceAsync( template -CUHOST inline O * copyToHost(const I * inp, S size, O * out = nullptr) +FF_CUHOST inline O * copyToHost(const I * inp, S size, O * out = nullptr) { O * ownout = nullptr; try @@ -317,10 +317,10 @@ CUHOST inline O * copyToHost(const I * inp, S size, O * out = nullptr) } template -CUHOST inline I * copyToHost(const I * inp, S size, I * out = nullptr) +FF_CUHOST inline I * copyToHost(const I * inp, S size, I * out = nullptr) { return copyToHost(inp, size, out); } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/include/fastfields/impl/kernels/atomic.h b/include/fastfields/impl/kernels/atomic.h index f0be333..751fde3 100755 --- a/include/fastfields/impl/kernels/atomic.h +++ b/include/fastfields/impl/kernels/atomic.h @@ -13,7 +13,7 @@ #ifndef __CUDACC__ #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) template class has_fetch_add @@ -86,7 +86,7 @@ static inline void anyAtomicAddNoReturn(T *address, T val) { return AtomicAdd::value>::atomicAddNoReturn(address, val); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) /*********************************************************************** * CUDA @@ -99,7 +99,7 @@ struct AtomicFPOp; template <> struct AtomicFPOp { template - inline CUDEV double operator() (double * address, double val, const func_t& func) { + inline FF_CUDEV double operator() (double * address, double val, const func_t& func) { unsigned long long int* address_as_ull = (unsigned long long int*)address; unsigned long long int old = *address_as_ull; unsigned long long int assumed; @@ -114,14 +114,14 @@ struct AtomicFPOp { } }; -#define ATOMIC_INTEGER_IMPL(NAME) \ +#define FF_ATOMIC_INTEGER_IMPL(NAME) \ template \ struct Atomic##NAME##IntegerImpl; \ \ template \ struct Atomic##NAME##IntegerImpl { \ template \ - inline CUDEV void operator()(T *address, T val, const func_t& func) { \ + inline FF_CUDEV void operator()(T *address, T val, const func_t& func) { \ size_t offset = (size_t)address & 3; \ unsigned int * address_as_ui = (unsigned int *)((char *)address - offset); \ unsigned int old = *address_as_ui; \ @@ -143,7 +143,7 @@ struct Atomic##NAME##IntegerImpl { template \ struct Atomic##NAME##IntegerImpl { \ template \ - inline CUDEV void operator()(T *address, T val, const func_t& func) { \ + inline FF_CUDEV void operator()(T *address, T val, const func_t& func) { \ size_t offset = (size_t)address & 2; \ unsigned int * address_as_ui = (unsigned int *)((char *)address - offset); \ bool is_32_align = offset; \ @@ -165,7 +165,7 @@ struct Atomic##NAME##IntegerImpl { template \ struct Atomic##NAME##IntegerImpl { \ template \ - inline CUDEV void operator()(T *address, T val, const func_t& func) { \ + inline FF_CUDEV void operator()(T *address, T val, const func_t& func) { \ unsigned int * address_as_ui = (unsigned int *) (address); \ unsigned int old = *address_as_ui; \ unsigned int newval; \ @@ -182,7 +182,7 @@ struct Atomic##NAME##IntegerImpl { template \ struct Atomic##NAME##IntegerImpl { \ template \ - inline CUDEV void operator()(T *address, T val, const func_t& func) { \ + inline FF_CUDEV void operator()(T *address, T val, const func_t& func) { \ unsigned long long * address_as_ui = (unsigned long long *) (address); \ unsigned long long old = *address_as_ui; \ unsigned long long newval; \ @@ -197,8 +197,8 @@ struct Atomic##NAME##IntegerImpl { }; -# define GPU_ATOMIC_INTEGER(NAME, OP, DTYPE) \ -static inline CUDEV void gpuAtomic##NAME(DTYPE *address, DTYPE val) { \ +# define FF_GPU_ATOMIC_INTEGER(NAME, OP, DTYPE) \ +static inline FF_CUDEV void gpuAtomic##NAME(DTYPE *address, DTYPE val) { \ Atomic##NAME##IntegerImpl()(address, \ val, \ [](DTYPE a, DTYPE b) { \ @@ -206,11 +206,11 @@ Atomic##NAME##IntegerImpl()(address, }); \ } \ -ATOMIC_INTEGER_IMPL(Add) +FF_ATOMIC_INTEGER_IMPL(Add) /* // Don't instantiate gpuAtomicAdd with the macro as it seems non-standard (see int32, int64) -static inline CUDEV void gpuAtomicAdd(char *address, char val) { +static inline FF_CUDEV void gpuAtomicAdd(char *address, char val) { AtomicAddIntegerImpl()(address, val, [](char a, char b) { @@ -218,7 +218,7 @@ static inline CUDEV void gpuAtomicAdd(char *address, char val) { }); } -static inline CUDEV void gpuAtomicAdd(signed char *address, signed char val) { +static inline FF_CUDEV void gpuAtomicAdd(signed char *address, signed char val) { AtomicAddIntegerImpl()(address, val, [](signed char a, signed char b) { @@ -226,7 +226,7 @@ static inline CUDEV void gpuAtomicAdd(signed char *address, signed char val) { }); } -static inline CUDEV void gpuAtomicAdd(short *address, short val) { +static inline FF_CUDEV void gpuAtomicAdd(short *address, short val) { AtomicAddIntegerImpl()(address, val, [](short a, short b) { @@ -234,11 +234,11 @@ static inline CUDEV void gpuAtomicAdd(short *address, short val) { }); } -static inline CUDEV int gpuAtomicAdd(int *address, int val) { +static inline FF_CUDEV int gpuAtomicAdd(int *address, int val) { return atomicAdd(address, val); } -static inline CUDEV void gpuAtomicAdd(long *address, long val) { +static inline FF_CUDEV void gpuAtomicAdd(long *address, long val) { #if defined(USE_ROCM) __atomic_fetch_add(address, val, __ATOMIC_RELAXED); #else @@ -250,7 +250,7 @@ static inline CUDEV void gpuAtomicAdd(long *address, long val) { #endif } -static inline CUDEV void gpuAtomicAdd(bool *address, bool val) { +static inline FF_CUDEV void gpuAtomicAdd(bool *address, bool val) { *address = address && val; } */ @@ -260,7 +260,7 @@ static inline CUDEV void gpuAtomicAdd(bool *address, bool val) { // provide this fallback for older device architectures (defining it for // sm_60+ collides with the built-in "atomicAdd(double*, double)"). #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600) -static inline CUDEV double atomicAdd(double* address, double val) +static inline FF_CUDEV double atomicAdd(double* address, double val) #if defined(__clang__) && defined(__CUDA__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wgcc-compat" @@ -276,17 +276,17 @@ static inline CUDEV double atomicAdd(double* address, double val) } #endif // __CUDA_ARCH__ < 600 -static inline CUDEV double gpuAtomicAdd(double *address, double val) { +static inline FF_CUDEV double gpuAtomicAdd(double *address, double val) { return atomicAdd(address, val); } -static inline CUDEV float gpuAtomicAdd(float *address, float val) { +static inline FF_CUDEV float gpuAtomicAdd(float *address, float val) { return atomicAdd(address, val); } /* template -static inline CUDEV void gpuAtomicAdd(complex *address, complex val) { +static inline FF_CUDEV void gpuAtomicAdd(complex *address, complex val) { gpuAtomicAdd(&address->real_, val.real_); gpuAtomicAdd(&address->imag_, val.imag_); } @@ -300,23 +300,23 @@ static inline CUDEV void gpuAtomicAdd(complex *address, complex val) { */ /* -static inline CUDEV void atomicAdd(char *address, char val) { +static inline FF_CUDEV void atomicAdd(char *address, char val) { gpuAtomicAdd(address, val); } -static inline CUDEV void atomicAdd(signed char *address, signed char val) { +static inline FF_CUDEV void atomicAdd(signed char *address, signed char val) { gpuAtomicAdd(address, val); } -static inline CUDEV void atomicAdd(short *address, short val) { +static inline FF_CUDEV void atomicAdd(short *address, short val) { gpuAtomicAdd(address, val); } -static inline CUDEV void atomicAdd(long *address, long val) { +static inline FF_CUDEV void atomicAdd(long *address, long val) { gpuAtomicAdd(address, val); } -static inline CUDEV void atomicAdd(bool *address, bool val) { +static inline FF_CUDEV void atomicAdd(bool *address, bool val) { gpuAtomicAdd(address, val); } */ @@ -330,32 +330,32 @@ static inline CUDEV void atomicAdd(bool *address, bool val) { */ /* template -static inline CUDEV void gpuAtomicAddNoReturn(complex *address, complex val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(char *address, char val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(signed char *address, signed char val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(short *address, short val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(int *address, int val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(long *address, long val) { gpuAtomicAdd(address, val); } -static inline CUDEV void gpuAtomicAddNoReturn(bool *address, bool val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(complex *address, complex val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(char *address, char val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(signed char *address, signed char val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(short *address, short val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(int *address, int val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(long *address, long val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(bool *address, bool val) { gpuAtomicAdd(address, val); } */ -static inline CUDEV void gpuAtomicAddNoReturn(double *address, double val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(double *address, double val) { gpuAtomicAdd(address, val); } /* Special case fp32 atomic. */ #if defined(USE_ROCM) -static inline CUDEV void gpuAtomicAddNoReturn(float *address, float val) { atomicAddNoRet(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(float *address, float val) { atomicAddNoRet(address, val); } #else -static inline CUDEV void gpuAtomicAddNoReturn(float *address, float val) { gpuAtomicAdd(address, val); } +static inline FF_CUDEV void gpuAtomicAddNoReturn(float *address, float val) { gpuAtomicAdd(address, val); } #endif namespace ff { template -static inline CUDEV T anyAtomicAdd(T *address, T val) { +static inline FF_CUDEV T anyAtomicAdd(T *address, T val) { return gpuAtomicAdd(address, val); } template -static inline CUDEV void anyAtomicAddNoReturn(T *address, T val) { +static inline FF_CUDEV void anyAtomicAddNoReturn(T *address, T val) { return gpuAtomicAddNoReturn(address, val); } diff --git a/include/fastfields/impl/kernels/batch.h b/include/fastfields/impl/kernels/batch.h index cfba30e..ea5ea55 100755 --- a/include/fastfields/impl/kernels/batch.h +++ b/include/fastfields/impl/kernels/batch.h @@ -24,10 +24,10 @@ #include "fastfields/core/cuda_switch.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) template -inline CUDEV +inline FF_CUDEV offset_t index2offset( offset_t index, int _ndim, @@ -50,7 +50,7 @@ offset_t index2offset( } template -inline CUDEV +inline FF_CUDEV offset_t index2offset( offset_t index, const offset_t * size, @@ -73,7 +73,7 @@ offset_t index2offset( } template -inline CUDEV +inline FF_CUDEV offset_t index2offset_nd( offset_t index, int _nall, @@ -105,7 +105,7 @@ offset_t index2offset_nd( template -inline CUDEV +inline FF_CUDEV offset_t index2offset_nd( offset_t index, const offset_t * size, @@ -140,7 +140,7 @@ offset_t index2offset_nd( // // This should be called index2offset_nd_v2. template -inline CUDEV +inline FF_CUDEV offset_t index2offset_v2( offset_t index, const offset_t * size, @@ -169,7 +169,7 @@ offset_t index2offset_v2( } template -inline CUDEV +inline FF_CUDEV offset_t index2offset_v2( offset_t index, offset_t nall, @@ -195,7 +195,7 @@ offset_t index2offset_v2( } template -inline CUDEV +inline FF_CUDEV offset_t sub2offset( const offset_t * sub, const offset_t * stride @@ -210,7 +210,7 @@ offset_t sub2offset( } template -inline CUDEV +inline FF_CUDEV offset_t sub2offset(offset_t ndim, const offset_t * sub, const offset_t * stride) { offset_t offset = 0; @@ -220,7 +220,7 @@ offset_t sub2offset(offset_t ndim, const offset_t * sub, const offset_t * stride } template -inline CUDEV +inline FF_CUDEV void index2sub( offset_t index, const offset_t * size, @@ -245,7 +245,7 @@ void index2sub( } template -inline CUDEV +inline FF_CUDEV void index2sub( offset_t nall, offset_t index, @@ -268,6 +268,6 @@ void index2sub( } } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_BATCH diff --git a/include/fastfields/impl/kernels/bounds.h b/include/fastfields/impl/kernels/bounds.h index 4206fcb..1cdecae 100755 --- a/include/fastfields/impl/kernels/bounds.h +++ b/include/fastfields/impl/kernels/bounds.h @@ -9,7 +9,7 @@ // INDEXING // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(bound) enum class type : int8_t { @@ -63,7 +63,7 @@ enum class type : int8_t { // uses DCT2/Neumann or DFT in practice. // __host__ __device__: used both as a compile-time template argument and, for a // `type::Dynamic` axis, evaluated at run time inside a device-side constructor. -CUHOSTDEV constexpr inline type transpose(type b) +FF_CUHOSTDEV constexpr inline type transpose(type b) { return b == type::DCT1 ? type::DST1 : b == type::DST1 ? type::DCT1 : @@ -91,11 +91,11 @@ struct BoundVecN { static const int max_ndim = MaxNDim; int8_t b[max_ndim]; - inline CUHOSTDEV BoundVecN() + inline FF_CUHOSTDEV BoundVecN() { for (int d = 0; d < max_ndim; ++d) b[d] = static_cast(type::Zero); } // Isotropic: the same condition on every axis (what the public ABI exposes). - explicit inline CUHOSTDEV BoundVecN(type v) + explicit inline FF_CUHOSTDEV BoundVecN(type v) { for (int d = 0; d < max_ndim; ++d) b[d] = static_cast(v); } // Anisotropic: one condition per axis, `ndim <= max_ndim` of them meaningful. @@ -106,13 +106,13 @@ struct BoundVecN { // `max_ndim`), so the specific pad value is inert; `Zero` is used because // it is this enum's own semantic default/identity value, matching `type()` // default-constructing to 0. - inline CUHOSTDEV BoundVecN(const type * v, int ndim) + inline FF_CUHOSTDEV BoundVecN(const type * v, int ndim) { for (int d = 0; d < max_ndim; ++d) b[d] = static_cast(d < ndim ? v[d] : type::Zero); } - inline CUHOSTDEV type operator[] (int d) const + inline FF_CUHOSTDEV type operator[] (int d) const { return static_cast(b[d]); } }; @@ -180,8 +180,8 @@ template using Bound = meta::Tuple; # define FF_STATIC_BOUND_NOCHECK FF_STATIC_BOUNDS #endif -#define FF_BOUND_IF_1(NAME) ::FF::bound::type::NAME -#define FF_BOUND_IF_0(NAME) ::FF::bound::type::Dynamic +#define FF_BOUND_IF_1(NAME) ::FF_NS::bound::type::NAME +#define FF_BOUND_IF_0(NAME) ::FF_NS::bound::type::Dynamic #define FF_BOUND_CAT_(A, B) A##B #define FF_BOUND_CAT(A, B) FF_BOUND_CAT_(A, B) #define FF_BOUND_SEL(FLAG, NAME) FF_BOUND_CAT(FF_BOUND_IF_, FLAG)(NAME) @@ -200,9 +200,9 @@ template using Bound = meta::Tuple; FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(bound) -using FF::bound::type; -using FF::bound::transpose; -using FF::bound::BoundVec; +using FF_NS::bound::type; +using FF_NS::bound::transpose; +using FF_NS::bound::BoundVec; // These function act on floating point coordinates and simply // apply the periodicity and reflection conditions of each boundary. @@ -215,7 +215,7 @@ template ::value struct _index { template - static inline CUDEV + static inline FF_CUDEV offset_t inbounds(offset_t coord, size_t size) { return coord; @@ -225,7 +225,7 @@ struct _index // Support length = N-1 // -> Boundary condition of a DCT-I template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_Nminus1(offset_t coord, size_t size) { if (size == 1) return static_cast(0); @@ -240,7 +240,7 @@ struct _index // Support length = N+1 // -> Boundary condition of a DST-I template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_Nplus1(offset_t coord, size_t size) { if (size == 1) static_cast(0); @@ -257,7 +257,7 @@ struct _index // Support length = N // -> Boundary condition of a DCT-II or DST-II template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_N(offset_t coord, size_t size) { if (size == 1) static_cast(0); @@ -273,7 +273,7 @@ struct _index // Support length = N // -> Boundary condition of a DFT template - static inline CUDEV + static inline FF_CUDEV offset_t circular(offset_t coord, size_t size) { if (size == 1) static_cast(0); @@ -286,7 +286,7 @@ struct _index // Clamped to (-1/2, N-1/2) // Support length = N template - static inline CUDEV + static inline FF_CUDEV offset_t replicate(offset_t coord, size_t size) { coord = coord <= -0.5 ? static_cast(-0.5) @@ -300,7 +300,7 @@ template struct _index { template - static inline CUDEV + static inline FF_CUDEV offset_t inbounds(offset_t coord, size_t size) { return coord; @@ -311,7 +311,7 @@ struct _index // -1 --> 1 // n --> n-2 template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_Nminus1(offset_t coord, size_t size) { if (size == 1) return 0; @@ -330,7 +330,7 @@ struct _index // n --> undefined [n-1] // n+1 --> n-1 template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_Nplus1(offset_t coord, size_t size) { if (size == 1) return static_cast(0); @@ -347,7 +347,7 @@ struct _index // -1 --> 0 // n --> n-1 template - static inline CUDEV + static inline FF_CUDEV offset_t reflect_N(offset_t coord, size_t size) { size_t size_twice = size*2; @@ -362,7 +362,7 @@ struct _index // -1 --> n-1 // n --> 0 template - static inline CUDEV + static inline FF_CUDEV offset_t circular(offset_t coord, size_t size) { coord = coord < 0 ? (size + coord%size) % size : coord % size; @@ -375,7 +375,7 @@ struct _index // n --> n-1 // n+1 --> n-1 template - static inline CUDEV + static inline FF_CUDEV offset_t replicate(offset_t coord, size_t size) { coord = coord <= 0 ? static_cast(0) @@ -392,21 +392,21 @@ struct _index FF_NAMESPACE_BEGIN(_sign) template -inline CUDEV int8_t inbounds(offset_t coord, size_t size) { +inline FF_CUDEV int8_t inbounds(offset_t coord, size_t size) { return coord < 0 || coord >= size ? 0 : 1; } // Boundary condition of a DCT/DFT // No sign modification based on coordinates template -constexpr inline CUDEV int8_t constant(offset_t coord, size_t size) { +constexpr inline FF_CUDEV int8_t constant(offset_t coord, size_t size) { return static_cast(1); } // Boundary condition of a DST-I // Periodic sign change based on coordinates template -inline CUDEV int8_t periodic1(offset_t coord, size_t size) { +inline FF_CUDEV int8_t periodic1(offset_t coord, size_t size) { if (size == 1) return 1; size_t size_twice = (size+1)*2; coord = coord < 0 ? size - coord - 1 : coord; @@ -419,7 +419,7 @@ inline CUDEV int8_t periodic1(offset_t coord, size_t size) { // Boundary condition of a DST-II // Periodic sign change based on coordinates template -inline CUDEV int8_t periodic2(offset_t coord, size_t size) { +inline FF_CUDEV int8_t periodic2(offset_t coord, size_t size) { coord = (coord < 0 ? size - coord - 1 : coord); return static_cast((coord/size) % 2 ? -1 : 1); } @@ -432,21 +432,21 @@ FF_NAMESPACE_END(_sign) // Check if coordinates within bounds template -inline CUDEV +inline FF_CUDEV bool inbounds(size_t coord, size_t size) { return coord >= 0 && coord < size; } template -inline CUDEV +inline FF_CUDEV bool inbounds(scalar_t coord, size_t size, scalar_t tol) { return coord >= -tol && coord < (scalar_t)(size-1)+tol; } template -inline CUDEV +inline FF_CUDEV scalar_t get(const scalar_t * ptr, offset_t offset, int8_t sign) { if (sign == -1) return -ptr[offset]; @@ -455,28 +455,28 @@ scalar_t get(const scalar_t * ptr, offset_t offset, int8_t sign) } template -inline CUDEV +inline FF_CUDEV scalar_t get(const scalar_t * ptr, offset_t offset) { return ptr[offset]; } template -inline CUDEV +inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset, int8_t sign) { return static_cast(get(ptr, offset, sign)); } template -inline CUDEV +inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset) { return static_cast(get(ptr, offset)); } template -inline CUDEV +inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) { scalar_t cval = static_cast(val); @@ -485,7 +485,7 @@ void add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) } template -inline CUDEV +inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val) { anyAtomicAdd(ptr + offset, static_cast(val)); @@ -493,64 +493,64 @@ void add(scalar_t *ptr, offset_t offset, val_t val) template struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::inbounds(coord, size); } template - static inline CUDEV int8_t sign(offset_t coord, size_t size) + static inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::inbounds(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::replicate(coord, size); } template - static constexpr inline CUDEV int8_t sign(offset_t coord, size_t size) + static constexpr inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::constant(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::reflect_Nminus1(coord, size); } template - static constexpr inline CUDEV int8_t sign(offset_t coord, size_t size) + static constexpr inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::constant(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::reflect_N(coord, size); } template - static constexpr inline CUDEV int8_t sign(offset_t coord, size_t size) + static constexpr inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::constant(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::reflect_Nplus1(coord, size); } template - static inline CUDEV int8_t sign(offset_t coord, size_t size) + static inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::periodic1(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::reflect_N(coord, size); } template - static inline CUDEV int8_t sign(offset_t coord, size_t size) + static inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::periodic2(coord, size); } }; template <> struct utils { template - static inline CUDEV offset_t index(offset_t coord, size_t size) + static inline FF_CUDEV offset_t index(offset_t coord, size_t size) { return _index::circular(coord, size); } template - static constexpr inline CUDEV int8_t sign(offset_t coord, size_t size) + static constexpr inline FF_CUDEV int8_t sign(offset_t coord, size_t size) { return _sign::constant(coord, size); } }; @@ -574,17 +574,17 @@ template <> struct utils { template struct dyn { - inline CUDEV dyn() {} - explicit inline CUDEV dyn(type) {} // runtime value: not needed, ignored + inline FF_CUDEV dyn() {} + explicit inline FF_CUDEV dyn(type) {} // runtime value: not needed, ignored - inline CUDEV type value() const { return B; } + inline FF_CUDEV type value() const { return B; } template - inline CUDEV offset_t index(offset_t coord, size_t size) const + inline FF_CUDEV offset_t index(offset_t coord, size_t size) const { return utils::template index(coord, size); } template - inline CUDEV int8_t sign(offset_t coord, size_t size) const + inline FF_CUDEV int8_t sign(offset_t coord, size_t size) const { return utils::template sign(coord, size); } }; @@ -592,16 +592,16 @@ template <> struct dyn { type bnd; - inline CUDEV dyn() : bnd(type::Zero) {} - explicit inline CUDEV dyn(type b) : bnd(b) {} + inline FF_CUDEV dyn() : bnd(type::Zero) {} + explicit inline FF_CUDEV dyn(type b) : bnd(b) {} - inline CUDEV type value() const { return bnd; } + inline FF_CUDEV type value() const { return bnd; } // Direct switches (rather than the `index_fn` / `sign_fn` function-pointer // helpers below): an indirect call cannot be inlined and is expensive on // the GPU, whereas a switch over a warp-uniform value is close to free. template - inline CUDEV offset_t index(offset_t coord, size_t size) const + inline FF_CUDEV offset_t index(offset_t coord, size_t size) const { switch (bnd) { case type::Replicate: return _index::replicate(coord, size); @@ -615,7 +615,7 @@ template <> struct dyn } template - inline CUDEV int8_t sign(offset_t coord, size_t size) const + inline FF_CUDEV int8_t sign(offset_t coord, size_t size) const { switch (bnd) { case type::Replicate: return _sign::constant(coord, size); @@ -634,11 +634,11 @@ template <> struct dyn // Not iso -> use sign template struct getutils { template - static inline CUDEV scalar_t + static inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset, int8_t sign) { return cget(ptr, offset, sign); } template - static inline CUDEV void + static inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) { return add(ptr, offset, val, sign); } }; @@ -647,33 +647,33 @@ template struct getutils { template struct getutils { template - static inline CUDEV scalar_t + static inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset, int8_t) { return bound::cget(ptr, offset); } template - static inline CUDEV void + static inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val, int8_t) { return bound::add(ptr, offset, val); } }; template struct getutils { template - static inline CUDEV scalar_t + static inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset, int8_t) { return bound::cget(ptr, offset); } template - static inline CUDEV void + static inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val, int8_t) { return bound::add(ptr, offset, val); } }; template struct getutils { template - static inline CUDEV scalar_t + static inline FF_CUDEV scalar_t cget(const scalar_t * ptr, offset_t offset, int8_t) { return bound::cget(ptr, offset); } template - static inline CUDEV void + static inline FF_CUDEV void add(scalar_t *ptr, offset_t offset, val_t val, int8_t) { return bound::add(ptr, offset, val); } }; @@ -683,31 +683,31 @@ template struct getutils { #define FF_ISO_SIGN(B) \ template <> struct getutils { \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ cget(const scalar_t * ptr, offset_t offset, int8_t sign) \ { return bound::cget(ptr, offset, sign); } \ template \ - static inline CUDEV void \ + static inline FF_CUDEV void \ add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) \ { return bound::add(ptr, offset, val, sign); } \ }; \ template <> struct getutils { \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ cget(const scalar_t * ptr, offset_t offset, int8_t sign) \ { return bound::cget(ptr, offset, sign); } \ template \ - static inline CUDEV void \ + static inline FF_CUDEV void \ add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) \ { return bound::add(ptr, offset, val, sign); } \ }; \ template <> struct getutils { \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ cget(const scalar_t * ptr, offset_t offset, int8_t sign) \ { return bound::cget(ptr, offset, sign); } \ template \ - static inline CUDEV void \ + static inline FF_CUDEV void \ add(scalar_t *ptr, offset_t offset, val_t val, int8_t sign) \ { return bound::add(ptr, offset, val, sign); } \ }; @@ -726,7 +726,7 @@ template using _index_fn_t = typename _index_fn::type; template -static inline CUDEV _index_fn_t +static inline FF_CUDEV _index_fn_t index_fn(type bound_type) { switch (bound_type) { case type::Replicate: return _index::template replicate; @@ -741,7 +741,7 @@ index_fn(type bound_type) { } template -static inline CUDEV offset_t +static inline FF_CUDEV offset_t index(type bound_type, offset_t coord, size_t size) { return index_fn(bound_type)(coord, size); // switch (bound_type) { @@ -763,7 +763,7 @@ template using _sign_fn_t = typename _sign_fn::type; template -static inline CUDEV _sign_fn_t +static inline FF_CUDEV _sign_fn_t sign_fn(type bound_type) { switch (bound_type) { case type::Replicate: return _sign::constant; @@ -778,7 +778,7 @@ sign_fn(type bound_type) { } template -static inline CUDEV int8_t +static inline FF_CUDEV int8_t sign(type bound_type, offset_t coord, size_t size) { return sign_fn(bound_type)(coord, size); // switch (bound_type) { @@ -795,6 +795,6 @@ sign(type bound_type, offset_t coord, size_t size) { FF_NAMESPACE_END(bound) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_BOUNDS diff --git a/include/fastfields/impl/kernels/distance/euclidean.h b/include/fastfields/impl/kernels/distance/euclidean.h index c1c702e..9e99bed 100755 --- a/include/fastfields/impl/kernels/distance/euclidean.h +++ b/include/fastfields/impl/kernels/distance/euclidean.h @@ -10,14 +10,14 @@ #include "fastfields/core/cuda_switch.h" #include "../utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_e) // This may be needed when working with half precision? // (I can't remember, but it's probably here for a reason) template -CUDEV inline +FF_CUDEV inline out_t mycast(inp_t x) { return static_cast(static_cast(x)); @@ -26,7 +26,7 @@ out_t mycast(inp_t x) // Compute the intersection point between two parabolas template -CUDEV +FF_CUDEV scalar_t intersection(scalar_t * f, offset_t * v, scalar_t w2, offset_t k, offset_t q, offset_t size, offset_t stride_buf) @@ -44,7 +44,7 @@ scalar_t intersection(scalar_t * f, offset_t * v, scalar_t w2, // Compute the squared distance in each voxel based on the location of // the parabolas template -CUDEV +FF_CUDEV void fillin(scalar_t * f, offset_t * v, scalar_t * z, scalar_t * d, scalar_t w2, offset_t size, offset_t stride, offset_t stride_buf) { @@ -81,7 +81,7 @@ void fillin(scalar_t * f, offset_t * v, scalar_t * z, scalar_t * d, scalar_t w2, // stride - Stride of between two voxels along the current dimension (`f`) // stride_buf - Stride of between two voxels along the current dimension (`d`) template -CUDEV +FF_CUDEV void kernel(scalar_t * f, offset_t * v, scalar_t * z, scalar_t * d, scalar_t w2, offset_t size, offset_t stride, offset_t stride_buf = 1) { @@ -119,6 +119,6 @@ void kernel(scalar_t * f, offset_t * v, scalar_t * z, scalar_t * d, scalar_t w2, FF_NAMESPACE_END(distance_e) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_E diff --git a/include/fastfields/impl/kernels/distance/l1.h b/include/fastfields/impl/kernels/distance/l1.h index 0faba93..155d767 100755 --- a/include/fastfields/impl/kernels/distance/l1.h +++ b/include/fastfields/impl/kernels/distance/l1.h @@ -7,7 +7,7 @@ #include "fastfields/core/cuda_switch.h" #include "../utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_l1) @@ -24,7 +24,7 @@ FF_NAMESPACE_BEGIN(distance_l1) // stride - Stride between two voxels along the current dimension // w - Voxel size along the current dimension template -CUDEV +FF_CUDEV void kernel(scalar_t * f, offset_t size, offset_t stride, scalar_t w) { if (size == 1) return; @@ -44,5 +44,5 @@ void kernel(scalar_t * f, offset_t size, offset_t stride, scalar_t w) FF_NAMESPACE_END(distance_l1) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_L1 diff --git a/include/fastfields/impl/kernels/distance/mesh.h b/include/fastfields/impl/kernels/distance/mesh.h index d1f51a4..8f75de5 100755 --- a/include/fastfields/impl/kernels/distance/mesh.h +++ b/include/fastfields/impl/kernels/distance/mesh.h @@ -10,13 +10,13 @@ #include // is needed by the 3D `build_normals` below. That builder runs -// on the host only -- but "host only" under nvcc means `CUHOST`, not +// on the host only -- but "host only" under nvcc means `FF_CUHOST`, not // `#ifndef __CUDACC__`: nvcc's device pass still parses (and instantiates) host // function bodies, so a `__CUDACC__` guard removes the member from *both* // passes and breaks the CUDA launcher that calls it. #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) // ============================================================================= // @@ -38,7 +38,7 @@ FF_NAMESPACE_END(distance_mesh) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_mesh) -using FF::distance_mesh::NearestEntity; +using FF_NS::distance_mesh::NearestEntity; template class MeshDistUtil {}; @@ -52,7 +52,7 @@ struct MeshDistUtil<2, scalar_t, offset_t> { static constexpr int D = 2; template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t sign( const Point & point, const NearestPoint & nearest_point, @@ -84,7 +84,7 @@ struct MeshDistUtil<2, scalar_t, offset_t> { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t sqdist_unsigned( NearestEntity & nearest_entity, NearestPoint & nearest_point, @@ -121,7 +121,7 @@ struct MeshDistUtil<2, scalar_t, offset_t> { // #ifndef __CUDACC__ // Returns pseudonormals ordered as: F, V0, V1 template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline void compute_normal(Normal & normal, const Vertices & vertices) { auto edge = vertices[1] - vertices[0]; @@ -137,7 +137,7 @@ struct MeshDistUtil<2, scalar_t, offset_t> { typename Faces, typename Vertices > - CUHOSTDEV static inline + FF_CUHOSTDEV static inline void build_normals( NormFaces & normfaces, NormVertices & normvertices, @@ -187,7 +187,7 @@ struct MeshDistUtil<3, scalar_t, offset_t> { static constexpr int D = 3; template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t sign( const Point & point, const NearestPoint & nearest_point, @@ -231,7 +231,7 @@ struct MeshDistUtil<3, scalar_t, offset_t> { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t sqdist_unsigned( NearestEntity & nearest_entity, NearestPoint & nearest_point, @@ -497,7 +497,7 @@ struct MeshDistUtil<3, scalar_t, offset_t> { } // Host-only (std::acos / std::unordered_map): the pseudonormals are - // precomputed on the host and uploaded to the device. Marked CUHOST rather + // precomputed on the host and uploaded to the device. Marked FF_CUHOST rather // than guarded by `#ifndef __CUDACC__` -- the guard hid these members from // nvcc's host pass too, so the CUDA `sdt` launcher's `build_normals` // wrapper had nothing to call. Mirrors the 2D specialisation above, whose @@ -505,7 +505,7 @@ struct MeshDistUtil<3, scalar_t, offset_t> { // Returns pseudonormals ordered as: F, V0, V1, V2 template - CUHOST static inline + FF_CUHOST static inline void compute_pseudonormals( Normals & pseudonormals, const Triangle & triangle @@ -542,7 +542,7 @@ struct MeshDistUtil<3, scalar_t, offset_t> { template - CUHOST static inline + FF_CUHOST static inline void build_normals( NormFaces & normfaces, NormVertices & normvertices, @@ -657,7 +657,7 @@ struct MeshDist { }; template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline BoundingSphere bounding_sphere(const Face & face) { BoundingSphere sphere; @@ -677,11 +677,11 @@ struct MeshDist { } // Host-only (std::sort + recursion): the BVH is built on the host and the - // node buffer is uploaded; the device walks it iteratively. Marked CUHOST + // node buffer is uploaded; the device walks it iteratively. Marked FF_CUHOST // rather than guarded by `#ifndef __CUDACC__` -- nvcc's device pass parses // and instantiates host function bodies too, so that guard removed // `build_tree` from *both* passes and left the CUDA `sdt` launcher (whose - // `CUHOST build_tree` wrapper calls it) unable to compile at all. + // `FF_CUHOST build_tree` wrapper calls it) unable to compile at all. // This logic is overly complex, but it's the only way I managed to // get std::sort to work on a strided array without copying the @@ -794,7 +794,7 @@ struct MeshDist { }; template - CUHOST static inline + FF_CUHOST static inline BoundingSphere build_tree( Node * nodes, index_t & node_id, @@ -895,7 +895,7 @@ struct MeshDist { // The point of the tree search is that we can cut long branches that // we know are already too far. template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline void query_dist_recurse( index_t & nearest_face, scalar_t & nearest_dist, @@ -966,7 +966,7 @@ struct MeshDist { // we can't use recursions in cuda (because stack size must be known at compile time) // so we must unroll the recursion, which is a pain. This works though! template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline void query_dist_loop( index_t & nearest_face, scalar_t & nearest_dist, @@ -1149,14 +1149,14 @@ struct MeshDist { } } -// #define DIST_USE_LOOP 1 +// #define FF_DIST_USE_LOOP 1 #ifdef __CUDACC__ -#define DIST_USE_LOOP +#define FF_DIST_USE_LOOP #endif template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline index_t get_nearest_vertex( const Face & nearest_face, const Point & point, @@ -1179,11 +1179,11 @@ struct MeshDist { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t _unsigned_dist( index_t & nearest_face, NearestEntity & nearest_entity, @@ -1192,13 +1192,13 @@ struct MeshDist { const Vertices & vertices, const Faces & faces, const Node * tree -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP , Trace & treetrace #endif ) { scalar_t nearest_dist = static_cast(1./0.); -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP query_dist_loop( #else query_dist_recurse( @@ -1207,14 +1207,14 @@ struct MeshDist { nearest_dist, nearest_entity, nearest_point, -#ifndef DIST_USE_LOOP +#ifndef FF_DIST_USE_LOOP static_cast(0), #endif point, vertices, faces, tree -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP ,treetrace #endif ); @@ -1222,17 +1222,17 @@ struct MeshDist { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t unsigned_dist( const Point & point, const Vertices & vertices, const Faces & faces, const Node * tree, -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP Trace & treetrace, #endif index_t * nearest_vertex = nullptr @@ -1250,7 +1250,7 @@ struct MeshDist { vertices, faces, tree -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP ,treetrace #endif ); @@ -1264,17 +1264,17 @@ struct MeshDist { template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t signed_dist( const Point & point, const Vertices & vertices, const Faces & faces, const Node * tree, -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP Trace & treetrace, #endif const NormFaces & normfaces, @@ -1298,7 +1298,7 @@ struct MeshDist { vertices, faces, tree -#ifdef DIST_USE_LOOP +#ifdef FF_DIST_USE_LOOP ,treetrace #endif ); @@ -1340,7 +1340,7 @@ struct MeshDist { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t _unsigned_dist_naive( index_t & nearest_face, NearestEntity & nearest_entity, @@ -1378,7 +1378,7 @@ struct MeshDist { } template - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t unsigned_dist_naive( const Point & point, const Vertices & vertices, @@ -1414,7 +1414,7 @@ struct MeshDist { typename NormEdges, typename NormVertices > - CUHOSTDEV static inline + FF_CUHOSTDEV static inline scalar_t signed_dist_naive( const Point & point, const Vertices & vertices, @@ -1476,6 +1476,6 @@ struct MeshDist { FF_NAMESPACE_END(distance_mesh) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_MESH_H diff --git a/include/fastfields/impl/kernels/distance/mesh_utils.h b/include/fastfields/impl/kernels/distance/mesh_utils.h index 71c9efe..f8bcb44 100755 --- a/include/fastfields/impl/kernels/distance/mesh_utils.h +++ b/include/fastfields/impl/kernels/distance/mesh_utils.h @@ -9,18 +9,18 @@ // // ============================================================================= -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_mesh) template struct StridedPointer { - CUHOSTDEV StridedPointer(value_t * ptr, offset_t stride): + FF_CUHOSTDEV StridedPointer(value_t * ptr, offset_t stride): ptr(ptr), stride(stride) {} - CUHOSTDEV value_t & operator[] (offset_t n) { return ptr[n*stride]; } - CUHOSTDEV const value_t & operator[] (offset_t n) const { return ptr[n*stride]; } + FF_CUHOSTDEV value_t & operator[] (offset_t n) { return ptr[n*stride]; } + FF_CUHOSTDEV const value_t & operator[] (offset_t n) const { return ptr[n*stride]; } value_t * ptr; offset_t stride; @@ -29,11 +29,11 @@ struct StridedPointer { template struct SizedStridedPointer { - CUHOSTDEV SizedStridedPointer(value_t * ptr, offset_t stride, offset_t size): + FF_CUHOSTDEV SizedStridedPointer(value_t * ptr, offset_t stride, offset_t size): ptr(ptr), stride(stride), size(size) {} - CUHOSTDEV value_t & operator[] (offset_t n) { return ptr[n*stride]; } - CUHOSTDEV const value_t & operator[] (offset_t n) const { return ptr[n*stride]; } + FF_CUHOSTDEV value_t & operator[] (offset_t n) { return ptr[n*stride]; } + FF_CUHOSTDEV const value_t & operator[] (offset_t n) const { return ptr[n*stride]; } value_t * ptr; offset_t stride; @@ -47,11 +47,11 @@ struct SizedStridedPointer { template struct Sized { - CUHOSTDEV virtual ~Sized() {} + FF_CUHOSTDEV virtual ~Sized() {} - CUHOSTDEV Sized(offset_t length): length(length) {} + FF_CUHOSTDEV Sized(offset_t length): length(length) {} - CUHOSTDEV inline int size() const { return length; } + FF_CUHOSTDEV inline int size() const { return length; } offset_t length; }; @@ -61,26 +61,26 @@ struct StaticSized { static constexpr long length = N; - CUHOSTDEV virtual ~StaticSized() {} + FF_CUHOSTDEV virtual ~StaticSized() {} - CUHOSTDEV inline int size() const { return length; } + FF_CUHOSTDEV inline int size() const { return length; } }; template struct AnyConstPoint { - CUHOSTDEV virtual ~AnyConstPoint() {} + FF_CUHOSTDEV virtual ~AnyConstPoint() {} - CUHOSTDEV virtual const scalar_t& operator[] (int d) const = 0; + FF_CUHOSTDEV virtual const scalar_t& operator[] (int d) const = 0; }; template struct AnyPoint { - CUHOSTDEV virtual ~AnyPoint() {} + FF_CUHOSTDEV virtual ~AnyPoint() {} - CUHOSTDEV virtual scalar_t& operator[] (int d) = 0; + FF_CUHOSTDEV virtual scalar_t& operator[] (int d) = 0; }; template @@ -104,130 +104,130 @@ struct PointMixin: public AnyPoint { using point_type = AnyPoint; using const_point_type = AnyConstPoint; - CUHOSTDEV virtual ~PointMixin() {} + FF_CUHOSTDEV virtual ~PointMixin() {} // reference to final type - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type * thisptr() { return reinterpret_cast(this); } - CUHOSTDEV inline + FF_CUHOSTDEV inline const final_type * thisptr() const { return reinterpret_cast(this); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type & thisref() { return reinterpret_cast(*this); } - CUHOSTDEV inline + FF_CUHOSTDEV inline const final_type & thisref() const { return reinterpret_cast(*this); } // in-place operations - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& copy_ (const const_point_type & other) { for (int d=0; d < D; ++d) (*this)[d] = other[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& copy_ (const const_point_type & other, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = other[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& copy_ (scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator = (const const_point_type & other) { return this->copy_(other); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator = (scalar_t alpha) { return this->copy_(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& add_ (const const_point_type & other) { for (int d=0; d < D; ++d) (*this)[d] += other[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& add_ (const const_point_type & other, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] += other[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& add_ (scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] += alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator += (const const_point_type & other) { return this->add_(other); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator += (scalar_t alpha) { return this->add_(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& sub_ (const const_point_type & other) { for (int d=0; d < D; ++d) (*this)[d] -= other[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& sub_ (const const_point_type & other, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] -= other[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& sub_ (scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] -= alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator -= (const const_point_type & other) { return this->sub_(other); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator -= (scalar_t alpha) { return this->sub_(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& mul_ (const const_point_type & other) { for (int d=0; d < D; ++d) (*this)[d] *= other[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& mul_ (const const_point_type & other, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] *= other[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& mul_ (scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] *= alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator *= (const const_point_type & other) { return this->mul_(other); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator *= (scalar_t alpha) { return this->mul_(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& div_ (const const_point_type & other) { for (int d=0; d < D; ++d) (*this)[d] /= other[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& div_ (const const_point_type & other, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] /= other[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& div_ (scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] /= alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator /= (const const_point_type & other) { return this->div_(other); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& operator /= (scalar_t alpha) { return this->div_(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& max_(const const_point_type & other) - { for (int d=0; d < D; ++d) (*this)[d] = FF::FF_DEVICE::max((*this)[d], other[d]); return thisref(); } + { for (int d=0; d < D; ++d) (*this)[d] = FF_NS::FF_DEVICE::max((*this)[d], other[d]); return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& min_(const const_point_type & other) - { for (int d=0; d < D; ++d) (*this)[d] = FF::FF_DEVICE::min((*this)[d], other[d]); return thisref(); } - CUHOSTDEV inline + { for (int d=0; d < D; ++d) (*this)[d] = FF_NS::FF_DEVICE::min((*this)[d], other[d]); return thisref(); } + FF_CUHOSTDEV inline final_type& normalize_() { scalar_t nrm = static_cast(0); @@ -240,47 +240,47 @@ struct PointMixin: public AnyPoint { // out-of-place operations (fill self) - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& addto_(const const_point_type & lhs, const const_point_type & rhs) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] + rhs[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& addto_(const const_point_type & lhs, const const_point_type & rhs, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] + rhs[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& subto_(const const_point_type & lhs, const const_point_type & rhs) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] - rhs[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& subto_(const const_point_type & lhs, const const_point_type & rhs, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] - rhs[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& multo_(const const_point_type & lhs, const const_point_type & rhs) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] * rhs[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& multo_(const const_point_type & lhs, const const_point_type & rhs, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] * rhs[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& divto_(const const_point_type & lhs, const const_point_type & rhs) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] / rhs[d]; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& divto_(const const_point_type & lhs, const const_point_type & rhs, scalar_t alpha) { for (int d=0; d < D; ++d) (*this)[d] = lhs[d] / rhs[d] * alpha; return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& maxto_(const const_point_type & lhs, const const_point_type & rhs) - { for (int d=0; d < D; ++d) (*this)[d] = FF::FF_DEVICE::max(lhs[d], rhs[d]); return thisref(); } + { for (int d=0; d < D; ++d) (*this)[d] = FF_NS::FF_DEVICE::max(lhs[d], rhs[d]); return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& minto_(const const_point_type & lhs, const const_point_type & rhs) - { for (int d=0; d < D; ++d) (*this)[d] = FF::FF_DEVICE::min(lhs[d], rhs[d]); return thisref(); } + { for (int d=0; d < D; ++d) (*this)[d] = FF_NS::FF_DEVICE::min(lhs[d], rhs[d]); return thisref(); } - CUHOSTDEV inline + FF_CUHOSTDEV inline final_type& crossto_(const const_point_type & lhs, const const_point_type & rhs) { // !! only works in 3D @@ -302,111 +302,111 @@ struct ConstPointMixin: public AnyConstPoint { using point_type = AnyPoint; using const_point_type = AnyConstPoint; - CUHOSTDEV virtual ~ConstPointMixin() {} + FF_CUHOSTDEV virtual ~ConstPointMixin() {} // out-of-place operations (return static point) - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type copy () const { return static_type(*this); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type add(const const_point_type & other) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] + other[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type add(const const_point_type & other, scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] + other[d] * alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type add(scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] + alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator+(const const_point_type & rhs) const { return this->add(rhs); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator+(scalar_t alpha) const { return this->add(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type sub(const const_point_type & other) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] - other[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type sub(const const_point_type & other, scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] - other[d] * alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type sub(scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] - alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator-(const const_point_type & rhs) const { return this->sub(rhs); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator-(scalar_t alpha) const { return this->sub(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type mul(const const_point_type & other) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] * other[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type mul(const const_point_type & other, scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] * other[d] * alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type mul(scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] * alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator*(const const_point_type & rhs) const { return this->mul(rhs); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator*(scalar_t alpha) const { return this->mul(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type div(const const_point_type & other) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] / other[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type div(const const_point_type & other, scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] / (other[d] * alpha); return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type div(scalar_t alpha) const { static_type out; for (int d=0; d < D; ++d) out[d] = (*this)[d] / alpha; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator/(const const_point_type & rhs) const { return this->div(rhs); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type operator/(scalar_t alpha) const { return this->div(alpha); } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type max(const const_point_type & other) const - { static_type out; for (int d=0; d < D; ++d) out[d] = FF::FF_DEVICE::max((*this)[d], other[d]); return out; } + { static_type out; for (int d=0; d < D; ++d) out[d] = FF_NS::FF_DEVICE::max((*this)[d], other[d]); return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type max(scalar_t alpha) const - { static_type out; for (int d=0; d < D; ++d) out[d] = FF::FF_DEVICE::max((*this)[d], alpha); return out; } + { static_type out; for (int d=0; d < D; ++d) out[d] = FF_NS::FF_DEVICE::max((*this)[d], alpha); return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type min(const const_point_type & other) const - { static_type out; for (int d=0; d < D; ++d) out[d] = FF::FF_DEVICE::min((*this)[d], other[d]); return out; } + { static_type out; for (int d=0; d < D; ++d) out[d] = FF_NS::FF_DEVICE::min((*this)[d], other[d]); return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type min(scalar_t alpha) const - { static_type out; for (int d=0; d < D; ++d) out[d] = FF::FF_DEVICE::min((*this)[d], alpha); return out; } + { static_type out; for (int d=0; d < D; ++d) out[d] = FF_NS::FF_DEVICE::min((*this)[d], alpha); return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline static_type cross(const const_point_type & other) const { // !! only works in 3D @@ -419,30 +419,30 @@ struct ConstPointMixin: public AnyConstPoint { // operations that return a scalar - CUHOSTDEV inline + FF_CUHOSTDEV inline scalar_t dot(const const_point_type & other) const { scalar_t out = static_cast(0); for (int d=0; d < D; ++d) out += (*this)[d] * other[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline scalar_t sum() const { scalar_t out = static_cast(0); for (int d=0; d < D; ++d) out += (*this)[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline scalar_t prod() const { scalar_t out = static_cast(1); for (int d=0; d < D; ++d) out *= (*this)[d]; return out; } - CUHOSTDEV inline + FF_CUHOSTDEV inline scalar_t sqnorm() const { return this->dot(*this); } - CUHOSTDEV inline + FF_CUHOSTDEV inline scalar_t norm() const { return sqrt(this->sqnorm()); } // view template - CUHOSTDEV inline + FF_CUHOSTDEV inline StaticPoint copyview() { StaticPoint out; @@ -460,27 +460,27 @@ struct StaticPoint: { using any_const_point = AnyConstPoint; - CUHOSTDEV virtual ~StaticPoint() {} - CUHOSTDEV StaticPoint() = default; - CUHOSTDEV StaticPoint(const any_const_point & other) + FF_CUHOSTDEV virtual ~StaticPoint() {} + FF_CUHOSTDEV StaticPoint() = default; + FF_CUHOSTDEV StaticPoint(const any_const_point & other) { this->copy_(other); } - CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d]; }; - CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; + FF_CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d]; }; + FF_CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; scalar_t data[D]; // reference view template - CUHOSTDEV inline + FF_CUHOSTDEV inline RefPoint view() { return RefPoint(data + begin); } template - CUHOSTDEV inline + FF_CUHOSTDEV inline ConstRefPoint view() const { return ConstRefPoint(data + begin); @@ -493,11 +493,11 @@ struct RefPoint: public PointMixin >, public ConstPointMixin > { - CUHOSTDEV virtual ~RefPoint() {} - CUHOSTDEV RefPoint(scalar_t * data): data(data) {} + FF_CUHOSTDEV virtual ~RefPoint() {} + FF_CUHOSTDEV RefPoint(scalar_t * data): data(data) {} - CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d]; }; - CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; + FF_CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d]; }; + FF_CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; scalar_t * data; @@ -505,14 +505,14 @@ struct RefPoint: // reference view template - CUHOSTDEV inline + FF_CUHOSTDEV inline RefPoint view() { return RefPoint(data + begin); } template - CUHOSTDEV inline + FF_CUHOSTDEV inline ConstRefPoint view() const { return ConstRefPoint(data + begin); @@ -523,17 +523,17 @@ template struct ConstRefPoint: public ConstPointMixin > { - CUHOSTDEV virtual ~ConstRefPoint() {} - CUHOSTDEV ConstRefPoint(const scalar_t * data): data(data) {} + FF_CUHOSTDEV virtual ~ConstRefPoint() {} + FF_CUHOSTDEV ConstRefPoint(const scalar_t * data): data(data) {} - CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; + FF_CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d]; }; const scalar_t * data; // reference view template - CUHOSTDEV inline + FF_CUHOSTDEV inline ConstRefPoint view() const { return ConstRefPoint(data + begin); @@ -545,13 +545,13 @@ struct StridedPoint: public PointMixin >, public ConstPointMixin > { - CUHOSTDEV virtual ~StridedPoint() {} - CUHOSTDEV StridedPoint(scalar_t * data, offset_t stride): data(data), stride(stride) {} + FF_CUHOSTDEV virtual ~StridedPoint() {} + FF_CUHOSTDEV StridedPoint(scalar_t * data, offset_t stride): data(data), stride(stride) {} - CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d*stride]; }; - CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d*stride]; }; + FF_CUHOSTDEV inline scalar_t& operator[] (int d) { return data[d*stride]; }; + FF_CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d*stride]; }; - CUHOSTDEV inline StridedPoint & operator= (const AnyConstPoint & other) + FF_CUHOSTDEV inline StridedPoint & operator= (const AnyConstPoint & other) { printf("assign (%ld)\n", data); for (int d=0; d - CUHOSTDEV inline + FF_CUHOSTDEV inline StridedPoint view() { return StridedPoint(data + begin, stride); } template - CUHOSTDEV inline + FF_CUHOSTDEV inline ConstStridedPoint view() const { return ConstStridedPoint(data + begin, stride); @@ -582,10 +582,10 @@ template struct ConstStridedPoint: public ConstPointMixin > { - CUHOSTDEV virtual ~ConstStridedPoint() {} - CUHOSTDEV ConstStridedPoint(const scalar_t * data, offset_t stride): data(data), stride(stride) {} + FF_CUHOSTDEV virtual ~ConstStridedPoint() {} + FF_CUHOSTDEV ConstStridedPoint(const scalar_t * data, offset_t stride): data(data), stride(stride) {} - CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d*stride]; }; + FF_CUHOSTDEV inline const scalar_t& operator[] (int d) const { return data[d*stride]; }; const scalar_t * data; offset_t stride; @@ -593,7 +593,7 @@ struct ConstStridedPoint: // reference view template - CUHOSTDEV inline + FF_CUHOSTDEV inline const ConstStridedPoint view() const { return ConstStridedPoint(data + begin, stride); @@ -609,13 +609,13 @@ struct StaticPointList: public StaticSized { using PointType = RefPoint; using ConstPointType = ConstRefPoint; - CUHOSTDEV virtual ~StaticPointList() {} + FF_CUHOSTDEV virtual ~StaticPointList() {} - CUHOSTDEV inline int size() const { return N; } + FF_CUHOSTDEV inline int size() const { return N; } - CUHOSTDEV inline PointType operator[] (int n) + FF_CUHOSTDEV inline PointType operator[] (int n) { return PointType(data + n*D); }; - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return ConstPointType(data + n*D); }; scalar_t data[N*D]; @@ -627,12 +627,12 @@ struct RefPointList { using PointType = RefPoint; using ConstPointType = ConstRefPoint; - CUHOSTDEV virtual ~RefPointList() {} - CUHOSTDEV RefPointList(scalar_t * data): data(data) {} + FF_CUHOSTDEV virtual ~RefPointList() {} + FF_CUHOSTDEV RefPointList(scalar_t * data): data(data) {} - CUHOSTDEV inline PointType operator[] (int n) + FF_CUHOSTDEV inline PointType operator[] (int n) { return PointType(data + n*D); }; - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return ConstPointType(data + n*D); }; scalar_t * data = nullptr; @@ -646,8 +646,8 @@ struct RefPointListSized: using BaseList = RefPointList; using BaseSized = Sized; - CUHOSTDEV virtual ~RefPointListSized() {} - CUHOSTDEV RefPointListSized(scalar_t * data, offset_t length): + FF_CUHOSTDEV virtual ~RefPointListSized() {} + FF_CUHOSTDEV RefPointListSized(scalar_t * data, offset_t length): BaseList(data), BaseSized(length) {} }; @@ -656,10 +656,10 @@ struct ConstRefPointList { using ConstPointType = ConstRefPoint; - CUHOSTDEV virtual ~ConstRefPointList() {} - CUHOSTDEV ConstRefPointList(const scalar_t * data): data(data) {} + FF_CUHOSTDEV virtual ~ConstRefPointList() {} + FF_CUHOSTDEV ConstRefPointList(const scalar_t * data): data(data) {} - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return ConstPointType(data + n*D); }; const scalar_t * data = nullptr; @@ -673,8 +673,8 @@ struct ConstRefPointListSized: using BaseList = ConstRefPointList; using BaseSized = Sized; - CUHOSTDEV virtual ~ConstRefPointListSized() {} - CUHOSTDEV ConstRefPointListSized(const scalar_t * data, offset_t length): + FF_CUHOSTDEV virtual ~ConstRefPointListSized() {} + FF_CUHOSTDEV ConstRefPointListSized(const scalar_t * data, offset_t length): BaseList(data), BaseSized(length) {} }; @@ -685,17 +685,17 @@ struct StridedPointList { using PointType = StridedPoint; using ConstPointType = ConstStridedPoint; - CUHOSTDEV virtual ~StridedPointList() {} - CUHOSTDEV + FF_CUHOSTDEV virtual ~StridedPointList() {} + FF_CUHOSTDEV StridedPointList(scalar_t * data, offset_t stride_elem, offset_t stride_channel): data(data), stride_elem(stride_elem), stride_channel(stride_channel) {} - CUHOSTDEV inline PointType operator[] (int n) + FF_CUHOSTDEV inline PointType operator[] (int n) { return PointType(data + n*stride_elem, stride_channel); }; - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return ConstPointType(data + n*stride_elem, stride_channel); }; scalar_t * data = nullptr; @@ -711,8 +711,8 @@ struct StridedPointListSized: using BaseList = StridedPointList; using BaseSized = Sized; - CUHOSTDEV virtual ~StridedPointListSized() {} - CUHOSTDEV StridedPointListSized(scalar_t * data, + FF_CUHOSTDEV virtual ~StridedPointListSized() {} + FF_CUHOSTDEV StridedPointListSized(scalar_t * data, offset_t stride_elem, offset_t stride_channel, offset_t length): @@ -724,14 +724,14 @@ struct ConstStridedPointList { using ConstPointType = ConstStridedPoint; - CUHOSTDEV virtual ~ConstStridedPointList() {} - CUHOSTDEV + FF_CUHOSTDEV virtual ~ConstStridedPointList() {} + FF_CUHOSTDEV ConstStridedPointList(const scalar_t * data, offset_t stride_elem, offset_t stride_channel): data(data), stride_elem(stride_elem), stride_channel(stride_channel) {} - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return ConstPointType(data + n*stride_elem, stride_channel); }; const scalar_t * data = nullptr; @@ -747,8 +747,8 @@ struct ConstStridedPointListSized: using BaseList = ConstStridedPointList; using BaseSized = Sized; - CUHOSTDEV virtual ~ConstStridedPointListSized() {} - CUHOSTDEV ConstStridedPointListSized(const scalar_t * data, + FF_CUHOSTDEV virtual ~ConstStridedPointListSized() {} + FF_CUHOSTDEV ConstStridedPointListSized(const scalar_t * data, offset_t stride_elem, offset_t stride_channel, offset_t length): @@ -802,37 +802,37 @@ struct StaticPointArray { template struct returned<0, dummy> { using type = SubArrayType; }; - CUHOSTDEV virtual ~StaticPointArray() {} + FF_CUHOSTDEV virtual ~StaticPointArray() {} template - CUHOSTDEV inline + FF_CUHOSTDEV inline typename returned<_Count::value>::type & at(int n0, T... n) { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType & at (int n0) { return reinterpret_cast(data + n0 * stride0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType & operator[] (int n0) { return this->at(n0); }; template - CUHOSTDEV inline + FF_CUHOSTDEV inline const typename returned<_Count::value>::type & at(int n0, T... n) const { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType& at(int n0) const { return reinterpret_cast(data + n0 * stride0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType& operator[] (int n0) const { return this->at(n0); @@ -849,25 +849,25 @@ struct StaticPointArray { template struct returned { using type = PointType; }; - CUHOSTDEV virtual ~StaticPointArray() {} + FF_CUHOSTDEV virtual ~StaticPointArray() {} - CUHOSTDEV inline + FF_CUHOSTDEV inline PointType& at (int n0) { return reinterpret_cast(data + n0 * D); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline PointType& operator[] (int n0) { return this->at(n0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const PointType& at (int n0) const { return reinterpret_cast(data + n0 * D); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const PointType& operator[] (int n0) const { return this->at(n0); @@ -894,20 +894,20 @@ struct RefPointArray { template struct returned<0, dummy> { using type = SubArrayType; }; - CUHOSTDEV virtual ~RefPointArray() {} + FF_CUHOSTDEV virtual ~RefPointArray() {} template - CUHOSTDEV inline + FF_CUHOSTDEV inline typename returned<_Count::value>::type & at(int n0, T... n) { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType & at(int n0) { return reinterpret_cast(data + n0 * stride0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType & operator[](int n0) { return this->at(n0); @@ -915,17 +915,17 @@ struct RefPointArray { template - CUHOSTDEV inline + FF_CUHOSTDEV inline const typename returned<_Count::value>::type & at(int n0, T... n) const { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType& at(int n0) const { return reinterpret_cast(data + n0 * stride0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType& operator[] (int n0) const { return this->at(n0); @@ -942,25 +942,25 @@ struct RefPointArray { template struct returned { using type = PointType; }; - CUHOSTDEV virtual ~RefPointArray() {} + FF_CUHOSTDEV virtual ~RefPointArray() {} - CUHOSTDEV inline + FF_CUHOSTDEV inline PointType& at(int n0) { return reinterpret_cast(data + n0 * D); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline PointType& operator[] (int n0) { return this->at(n0); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const PointType& at(int n0) const { return reinterpret_cast(data + n0 * D); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const PointType& operator[] (int n0) const { return this->at(n0); @@ -987,46 +987,46 @@ struct StridedPointArray { template struct returned<0, dummy> { using type = SubArrayType; }; - CUHOSTDEV virtual ~StridedPointArray() {} + FF_CUHOSTDEV virtual ~StridedPointArray() {} template - CUHOSTDEV + FF_CUHOSTDEV StridedPointArray(scalar_t * data, const Stride & stride): data(data), stride(stride) {} - CUHOSTDEV + FF_CUHOSTDEV StridedPointArray(scalar_t * data = nullptr): data(data), stride() { stride.copy_(1); } template - CUHOSTDEV inline + FF_CUHOSTDEV inline typename returned<_Count::value>::type at(int n0, T... n) { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType at(int n0) { return SubArrayType(data + n0 * stride[0], stride.template view<1,nbatch+1>()); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline SubArrayType operator[] (int n0) { return this->at(n0); }; template - CUHOSTDEV inline + FF_CUHOSTDEV inline const typename returned<_Count::value>::type at(int n0, T... n) const { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType at(int n0) const { return SubArrayType(data + n0 * stride[0], stride.template view<1,nbatch+1>()); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType operator[] (int n0) const { return this->at(n0); @@ -1042,26 +1042,26 @@ struct StridedPointArray { using PointType = StridedPoint; using ConstPointType = StridedPoint; - CUHOSTDEV virtual ~StridedPointArray() {} + FF_CUHOSTDEV virtual ~StridedPointArray() {} template - CUHOSTDEV + FF_CUHOSTDEV StridedPointArray(scalar_t * data, const Stride & stride): data(data), stride(stride) {} - CUHOSTDEV inline PointType at(int n) + FF_CUHOSTDEV inline PointType at(int n) { return PointType(data + n*stride[0], stride[1]); }; - CUHOSTDEV inline PointType operator[] (int n) + FF_CUHOSTDEV inline PointType operator[] (int n) { return this->at(n); }; - CUHOSTDEV inline ConstPointType at(int n) const + FF_CUHOSTDEV inline ConstPointType at(int n) const { return ConstPointType(data + n*stride[0], stride[1]); }; - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return this->at(n); }; @@ -1088,29 +1088,29 @@ struct ConstStridedPointArray { template struct returned<0, dummy> { using type = SubArrayType; }; - CUHOSTDEV virtual ~ConstStridedPointArray() {} + FF_CUHOSTDEV virtual ~ConstStridedPointArray() {} template - CUHOSTDEV + FF_CUHOSTDEV ConstStridedPointArray(const scalar_t * data, const Stride & stride): data(data), stride(stride) {} - CUHOSTDEV + FF_CUHOSTDEV ConstStridedPointArray(scalar_t * data = nullptr): data(data), stride() { stride.copy_(1); } template - CUHOSTDEV inline + FF_CUHOSTDEV inline const typename returned<_Count::value>::type at(int n0, T... n) const { return (*this)[n0].at(n...); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType at(int n0) const { return SubArrayType(data + n0 * stride[0], stride.template view<1,nbatch+1>()); }; - CUHOSTDEV inline + FF_CUHOSTDEV inline const SubArrayType operator[] (int n0) const { return this->at(n0); @@ -1126,22 +1126,22 @@ struct ConstStridedPointArray { using PointType = StridedPoint; using ConstPointType = ConstStridedPoint; - CUHOSTDEV virtual ~ConstStridedPointArray() {} + FF_CUHOSTDEV virtual ~ConstStridedPointArray() {} template - CUHOSTDEV + FF_CUHOSTDEV ConstStridedPointArray(const scalar_t * data, const Stride & stride): data(data), stride(stride) {} - CUHOSTDEV + FF_CUHOSTDEV ConstStridedPointArray(scalar_t * data = nullptr): data(data), stride() { stride.copy_(1); } - CUHOSTDEV inline ConstPointType at(int n) const + FF_CUHOSTDEV inline ConstPointType at(int n) const { return ConstPointType(data + n*stride[0], stride[1]); }; - CUHOSTDEV inline ConstPointType operator[] (int n) const + FF_CUHOSTDEV inline ConstPointType operator[] (int n) const { return this->at(n); }; @@ -1153,6 +1153,6 @@ struct ConstStridedPointArray { FF_NAMESPACE_END(distance_mesh) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_MESH_UTILS_H diff --git a/include/fastfields/impl/kernels/distance/spline.h b/include/fastfields/impl/kernels/distance/spline.h index bf10fdd..91eb961 100755 --- a/include/fastfields/impl/kernels/distance/spline.h +++ b/include/fastfields/impl/kernels/distance/spline.h @@ -6,7 +6,7 @@ #include "../pushpull.h" #include "../utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(distance_spline) @@ -44,7 +44,7 @@ class Kernels { static constexpr offset_t ndim = static_cast(D); static constexpr offset_t nostride = static_cast(1); - CUDEV static inline + FF_CUDEV static inline void min_table( scalar_t best_time [], scalar_t best_dist [], @@ -88,7 +88,7 @@ class Kernels { } - CUDEV static inline + FF_CUDEV static inline void min_brent( scalar_t best_time [], scalar_t best_dist [], @@ -380,7 +380,7 @@ class Kernels { } - CUDEV static inline + FF_CUDEV static inline void min_gaussnewton( scalar_t best_time [], scalar_t best_dist [], @@ -468,6 +468,6 @@ class Kernels { FF_NAMESPACE_END(distance_spline) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_DISTANCE_SPLINE_H diff --git a/include/fastfields/impl/kernels/meta.h b/include/fastfields/impl/kernels/meta.h index 3ceb33e..ca161db 100644 --- a/include/fastfields/impl/kernels/meta.h +++ b/include/fastfields/impl/kernels/meta.h @@ -2,7 +2,7 @@ #define FF_META #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(meta) template struct Pack; @@ -41,7 +41,7 @@ template using NTuple = typename _NTuple:: template using Int = Tuple; FF_NAMESPACE_END(meta) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_META diff --git a/include/fastfields/impl/kernels/parallel.h b/include/fastfields/impl/kernels/parallel.h index 11c0c14..4ec455e 100755 --- a/include/fastfields/impl/kernels/parallel.h +++ b/include/fastfields/impl/kernels/parallel.h @@ -8,7 +8,7 @@ #include "fastfields/core/defines.h" #include "parallel_impl.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) constexpr int64_t GRAIN_SIZE = 32768; @@ -30,6 +30,6 @@ inline void parallel_for(int64_t begin, int64_t end, int64_t grain_size, const F internal::invoke_parallel(begin, end, grain_size, f); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PARALLEL_H diff --git a/include/fastfields/impl/kernels/parallel_impl.h b/include/fastfields/impl/kernels/parallel_impl.h index df655b9..0a32a9d 100755 --- a/include/fastfields/impl/kernels/parallel_impl.h +++ b/include/fastfields/impl/kernels/parallel_impl.h @@ -58,15 +58,15 @@ #if FF_CAN_USE_FUTURE #include "threadpool.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) inline size_t get_parallel_threads() { return get_num_threads(); } inline size_t set_parallel_threads(int nthreads) { return set_num_threads(nthreads); } inline std::string get_parallel_backend() { return "native"; } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #elif FF_CAN_USE_OPENMP // #pragma cling load("libomp"). #include -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) inline size_t get_parallel_threads() { return omp_get_max_threads(); } inline size_t set_parallel_threads(int nthreads) { @@ -74,17 +74,17 @@ inline size_t set_parallel_threads(int nthreads) return omp_get_max_threads(); } inline std::string get_parallel_backend() { return "omp"; } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #else -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) inline size_t get_parallel_threads() { return 1; } inline size_t set_parallel_threads(int nthreads) { return 1; } inline std::string get_parallel_backend() { return "none"; } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(internal) //#if 0 @@ -230,6 +230,6 @@ FF_NAMESPACE_BEGIN(internal) #endif // FF_CAN_USE_FUTURE || FF_CAN_USE_OPENMP FF_NAMESPACE_END(internal) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PARALLEL_IMPL_H diff --git a/include/fastfields/impl/kernels/posdef/cholesky.h b/include/fastfields/impl/kernels/posdef/cholesky.h index 9080508..72e331c 100755 --- a/include/fastfields/impl/kernels/posdef/cholesky.h +++ b/include/fastfields/impl/kernels/posdef/cholesky.h @@ -4,9 +4,7 @@ #include "../utils.h" #include "utils.h" -#define JFH_OnePlusTiny 1.000001 - -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(posdef) @@ -28,7 +26,7 @@ struct cholesky { typename ptr_t, typename reduce_t = internal::elem_type > - inline CUDEV static void + inline FF_CUDEV static void decompose_( ptr_t a, reduce_t unused = static_cast(0) @@ -42,7 +40,7 @@ struct cholesky { typename xptr_t, typename reduce_t = internal::return_type > - inline CUDEV static + inline FF_CUDEV static void solve_( aptr_t a, xptr_t x, @@ -67,7 +65,7 @@ struct cholesky { typename ptr_t, typename reduce_t = internal::elem_type > - inline CUDEV static + inline FF_CUDEV static void decompose_( offset_t C, ptr_t a, @@ -109,7 +107,7 @@ struct cholesky { typename reduce_t = internal::return_type > - inline CUDEV static + inline FF_CUDEV static void solve_( offset_t C, aptr_t a, @@ -139,6 +137,6 @@ struct cholesky { FF_NAMESPACE_END(posdef) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_POSDEF_CHOLESKY diff --git a/include/fastfields/impl/kernels/posdef/diag.inl b/include/fastfields/impl/kernels/posdef/diag.inl index 9946386..2be755a 100755 --- a/include/fastfields/impl/kernels/posdef/diag.inl +++ b/include/fastfields/impl/kernels/posdef/diag.inl @@ -12,7 +12,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void matvec( optr_t o, hptr_t h, @@ -31,7 +31,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_( optr_t o, hptr_t h, @@ -50,7 +50,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void submatvec_( optr_t o, hptr_t h, @@ -70,7 +70,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_( vptr_t v, hptr_t h, @@ -99,7 +99,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void invert( optr_t o, hptr_t h, @@ -123,7 +123,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void matvec( offset_t C, optr_t o, @@ -142,7 +142,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void addmatvec_( offset_t C, optr_t o, @@ -161,7 +161,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void submatvec_( offset_t C, optr_t o, @@ -181,7 +181,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void solve_impl_( offset_t C, vptr_t v, @@ -209,7 +209,7 @@ struct utils: public common_diag typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void invert( offset_t C, optr_t o, diff --git a/include/fastfields/impl/kernels/posdef/estatics.inl b/include/fastfields/impl/kernels/posdef/estatics.inl index 77af45d..36496ee 100755 --- a/include/fastfields/impl/kernels/posdef/estatics.inl +++ b/include/fastfields/impl/kernels/posdef/estatics.inl @@ -6,7 +6,7 @@ struct utils: public common_estatics { template > - static inline CUDEV void + static inline FF_CUDEV void matvec(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -28,7 +28,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -50,7 +50,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -73,7 +73,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -112,7 +112,7 @@ struct utils: public common_estatics { template > - static inline CUDEV void + static inline FF_CUDEV void matvec(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -133,7 +133,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -154,7 +154,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -176,7 +176,7 @@ struct utils: public common_estatics template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(offset_t C, vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) diff --git a/include/fastfields/impl/kernels/posdef/eye.inl b/include/fastfields/impl/kernels/posdef/eye.inl index 2688861..385932a 100755 --- a/include/fastfields/impl/kernels/posdef/eye.inl +++ b/include/fastfields/impl/kernels/posdef/eye.inl @@ -6,7 +6,7 @@ struct utils: public common_eye { template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -18,7 +18,7 @@ struct utils: public common_eye template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -31,7 +31,7 @@ struct utils: public common_eye template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -55,7 +55,7 @@ struct utils: public common_eye template > - static inline CUDEV + static inline FF_CUDEV void invert(optr_t o, hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) { @@ -68,7 +68,7 @@ struct utils: public common_eye { template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -79,7 +79,7 @@ struct utils: public common_eye template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -91,7 +91,7 @@ struct utils: public common_eye template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(offset_t C, vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -113,7 +113,7 @@ struct utils: public common_eye template > - static inline CUDEV + static inline FF_CUDEV void invert(offset_t C, optr_t o, hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) { diff --git a/include/fastfields/impl/kernels/posdef/full.inl b/include/fastfields/impl/kernels/posdef/full.inl index 5e9486b..fd6f31a 100755 --- a/include/fastfields/impl/kernels/posdef/full.inl +++ b/include/fastfields/impl/kernels/posdef/full.inl @@ -9,7 +9,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -23,7 +23,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -37,7 +37,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -52,7 +52,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t b = nullptr, reduce_t unused = static_cast(0)) @@ -75,7 +75,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -89,7 +89,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -103,7 +103,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -118,7 +118,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(offset_t C, vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t b = nullptr, reduce_t /*unused*/ = static_cast(0)) diff --git a/include/fastfields/impl/kernels/posdef/posdef.h b/include/fastfields/impl/kernels/posdef/posdef.h index 89be18e..edbaaa5 100755 --- a/include/fastfields/impl/kernels/posdef/posdef.h +++ b/include/fastfields/impl/kernels/posdef/posdef.h @@ -6,9 +6,7 @@ #include "cholesky.h" #include -#define JFH_OnePlusTiny 1.000001 - -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(posdef) @@ -87,7 +85,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void solve_( vptr_t v, hptr_t h, @@ -108,7 +106,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void solve( xptr_t x, vptr_t v, @@ -131,7 +129,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void relax_( xptr_t x, hptr_t h, @@ -153,7 +151,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_( vptr_t v, hptr_t h, @@ -170,7 +168,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_( vptr_t v, hptr_t h, @@ -188,7 +186,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_( vptr_t v, hptr_t h, @@ -207,7 +205,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve( xptr_t x, vptr_t v, @@ -226,7 +224,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve( xptr_t x, vptr_t v, @@ -246,7 +244,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_( xptr_t x, vptr_t v, @@ -266,7 +264,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_( xptr_t x, hptr_t h, @@ -285,7 +283,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_( xptr_t x, hptr_t h, @@ -305,7 +303,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_( xptr_t x, hptr_t h, @@ -320,7 +318,7 @@ struct common #endif template - static inline CUDEV void + static inline FF_CUDEV void copy_(optr_t out, iptr_t inp) { using output_t = internal::elem_type; @@ -330,7 +328,7 @@ struct common } template - static inline CUDEV void + static inline FF_CUDEV void copy_(offset_t L, optr_t out, iptr_t inp) { using output_t = internal::elem_type; @@ -344,7 +342,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void add_( optr_t out, iptr_t inp, @@ -389,7 +387,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void solve_( offset_t C, vptr_t v, @@ -419,7 +417,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void solve( offset_t C, xptr_t x, @@ -460,7 +458,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void relax_( offset_t C, xptr_t x, @@ -487,7 +485,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_(offset_t C, vptr_t v, hptr_t h, wptr_t w, double unused) { const void * b = nullptr; @@ -499,7 +497,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_(offset_t C, vptr_t v, hptr_t h, wptr_t w, float unused) { const void * b = nullptr; @@ -512,7 +510,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_(offset_t C, vptr_t v, hptr_t h, wptr_t w, half unused) { const void * b = nullptr; @@ -526,7 +524,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve(offset_t C, xptr_t x, vptr_t v, hptr_t h, wptr_t w, double unused) { const void * b = nullptr; @@ -539,7 +537,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve(offset_t C, xptr_t x, vptr_t v, hptr_t h, wptr_t w, float unused) { const void * b = nullptr; @@ -553,7 +551,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void solve_(offset_t C, xptr_t x, vptr_t v, hptr_t h, wptr_t w, half unused) { const void * b = nullptr; @@ -567,7 +565,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_(offset_t C, xptr_t x, hptr_t h, vptr_t v, wptr_t w, float unused) { const void * b = nullptr; @@ -580,7 +578,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_(offset_t C, xptr_t x, hptr_t h, vptr_t v, wptr_t w, double unused) { const void * b = nullptr; @@ -594,7 +592,7 @@ struct common typename hptr_t, typename wptr_t = const void * > - static inline CUDEV void + static inline FF_CUDEV void relax_(offset_t C, xptr_t x, hptr_t h, vptr_t v, wptr_t w, half unused) { const void * b = nullptr; @@ -610,7 +608,7 @@ struct common /// @param out[out] pointer to output vector (length C) /// @param inp[in] pointer to input vector (length C) template - static inline CUDEV void + static inline FF_CUDEV void copy_(offset_t C, optr_t out, iptr_t inp) { using output_t = internal::elem_type; @@ -629,7 +627,7 @@ struct common typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void add_( offset_t C, optr_t out, @@ -657,7 +655,7 @@ struct utils: public common, offset_t, C> typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void matvec( optr_t o, hptr_t h, @@ -677,7 +675,7 @@ struct utils: public common, offset_t, C> typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_( optr_t o, hptr_t h, @@ -697,7 +695,7 @@ struct utils: public common, offset_t, C> typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void submatvec_( optr_t o, hptr_t h, @@ -721,7 +719,7 @@ struct utils: public common, offset_t, C> typename reduce_t = internal::return_type > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_( vptr_t v, hptr_t h, @@ -749,7 +747,7 @@ struct utils: public common > - static inline CUDEV void + static inline FF_CUDEV void matvec( offset_t C, optr_t o, @@ -771,7 +769,7 @@ struct utils: public common > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_( offset_t C, optr_t o, @@ -793,7 +791,7 @@ struct utils: public common > - static inline CUDEV void + static inline FF_CUDEV void submatvec_( offset_t C, optr_t o, @@ -818,7 +816,7 @@ struct utils: public common > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_( offset_t C, vptr_t v, @@ -852,7 +850,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void matvec( optr_t /*o*/, hptr_t /*h*/, @@ -868,7 +866,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void addmatvec_( optr_t /*o*/, hptr_t /*h*/, @@ -884,7 +882,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void submatvec_( optr_t /*o*/, hptr_t /*h*/, @@ -901,7 +899,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void solve_impl_( vptr_t v, hptr_t /*h*/, @@ -931,7 +929,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void matvec( offset_t /*C*/, optr_t /*o*/, @@ -948,7 +946,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void addmatvec_( offset_t /*C*/, optr_t /*o*/, @@ -965,7 +963,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void submatvec_( offset_t /*C*/, optr_t /*o*/, @@ -983,7 +981,7 @@ struct utils: public common_none typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void solve_impl_( offset_t C, vptr_t v, @@ -1010,6 +1008,6 @@ struct utils: public common_none FF_NAMESPACE_END(posdef) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_POSDEF diff --git a/include/fastfields/impl/kernels/posdef/sym.inl b/include/fastfields/impl/kernels/posdef/sym.inl index d1179cf..f59c02c 100755 --- a/include/fastfields/impl/kernels/posdef/sym.inl +++ b/include/fastfields/impl/kernels/posdef/sym.inl @@ -18,7 +18,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void matvec_backward( hptr_t h, xptr_t x, @@ -45,7 +45,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void matvec( optr_t o, hptr_t h, @@ -71,7 +71,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void addmatvec_( optr_t o, hptr_t h, @@ -97,7 +97,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void submatvec_( optr_t o, hptr_t h, @@ -124,7 +124,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void solve_impl_( vptr_t v, hptr_t h, @@ -149,7 +149,7 @@ struct utils: public common_sym typename reduce_t = internal::return_type > - static inline CUDEV + static inline FF_CUDEV void invert( optr_t o, hptr_t h, @@ -164,7 +164,7 @@ struct utils: public common_sym typename hptr_t, typename bptr_t = const void *, typename reduce_t = internal::return_type> - static inline CUDEV + static inline FF_CUDEV void invert_( hptr_t h, bptr_t b = nullptr, @@ -272,7 +272,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void tofull(optr_t o, iptr_t i) { using scalar_t = internal::elem_type; @@ -280,7 +280,7 @@ struct utils: public common_sym # pragma unroll for (offset_t c = 0; c < C; ++c, ++i) { - o[C*c+c] = static_cast((*i) * JFH_OnePlusTiny); + o[C*c+c] = static_cast((*i) * FF_ONE_PLUS_TINY); } # pragma unroll for (offset_t c = 0; c < C; ++c) @@ -307,7 +307,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void fromfull(optr_t o, iptr_t i) { # pragma unroll @@ -322,13 +322,13 @@ struct utils: public common_sym } template - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak_rows(offset_t i, offset_t j) { return j < i ? sub2pak_rows(j, i) : i*K - (i*(i+1)) / 2 + j; } - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak(offset_t i, offset_t j) { return j < i ? sub2pak(j, i) : i == j ? i : C + sub2pak_rows(i, j-1); @@ -349,7 +349,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec_backward(offset_t C, hptr_t h, xptr_t x, yptr_t y, reduce_t /*unused*/ = static_cast(0)) { @@ -366,7 +366,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -382,7 +382,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -398,7 +398,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(offset_t C, optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -415,7 +415,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(offset_t C, vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t b = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -432,7 +432,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert(offset_t C, optr_t o, hptr_t h, bptr_t b = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -443,7 +443,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert_(offset_t C, hptr_t h, bptr_t b = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -541,14 +541,14 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void tofull(offset_t C, optr_t o, iptr_t i) { using scalar_t = internal::elem_type; scalar_t foo; for (offset_t c = 0; c < C; ++c, ++i) { - o[C*c+c] = static_cast((*i) * JFH_OnePlusTiny); + o[C*c+c] = static_cast((*i) * FF_ONE_PLUS_TINY); } for (offset_t c = 0; c < C; ++c) { @@ -571,7 +571,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void fromfull(offset_t C, optr_t o, iptr_t i) { for (offset_t c = 0; c < C; ++c, ++o) @@ -584,13 +584,13 @@ struct utils: public common_sym //protected: - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak_rows(offset_t C, offset_t i, offset_t j) { return j < i ? sub2pak_rows(C, j, i) : i*C - (i*(i+1)) / 2 + j; } - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak(offset_t C, offset_t i, offset_t j) { return j < i ? sub2pak(C, j, i) : i == j ? i : C + sub2pak_rows(C-1, i, j-1); @@ -610,7 +610,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec_backward(hptr_t h, xptr_t x, yptr_t y, reduce_t /*unused*/ = static_cast(0)) { @@ -631,7 +631,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -652,7 +652,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -673,7 +673,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -695,7 +695,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t b = nullptr, reduce_t unused = static_cast(0)) @@ -703,9 +703,9 @@ struct utils: public common_sym reduce_t x0 = static_cast(v[0]), x1 = static_cast(v[1]), x2 = static_cast(v[2]), - h00 = static_cast(h[0]) * JFH_OnePlusTiny, - h11 = static_cast(h[1]) * JFH_OnePlusTiny, - h22 = static_cast(h[2]) * JFH_OnePlusTiny, + h00 = static_cast(h[0]) * FF_ONE_PLUS_TINY, + h11 = static_cast(h[1]) * FF_ONE_PLUS_TINY, + h22 = static_cast(h[2]) * FF_ONE_PLUS_TINY, h01 = static_cast(h[3]), h02 = static_cast(h[4]), h12 = static_cast(h[5]); @@ -728,7 +728,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_le_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t b = nullptr, reduce_t unused = static_cast(0)) @@ -736,9 +736,9 @@ struct utils: public common_sym reduce_t x0 = static_cast(v[0]), x1 = static_cast(v[1]), x2 = static_cast(v[2]), - h00 = static_cast(h[0]) * JFH_OnePlusTiny, - h11 = static_cast(h[1]) * JFH_OnePlusTiny, - h22 = static_cast(h[2]) * JFH_OnePlusTiny, + h00 = static_cast(h[0]) * FF_ONE_PLUS_TINY, + h11 = static_cast(h[1]) * FF_ONE_PLUS_TINY, + h22 = static_cast(h[2]) * FF_ONE_PLUS_TINY, h01 = static_cast(h[3]), h02 = static_cast(h[4]), h12 = static_cast(h[5]); @@ -761,7 +761,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert(optr_t o, hptr_t h, bptr_t b = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -773,15 +773,15 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert_(hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) { - reduce_t h00 = static_cast(h[0]) * JFH_OnePlusTiny, - h11 = static_cast(h[1]) * JFH_OnePlusTiny, - h22 = static_cast(h[2]) * JFH_OnePlusTiny, + reduce_t h00 = static_cast(h[0]) * FF_ONE_PLUS_TINY, + h11 = static_cast(h[1]) * FF_ONE_PLUS_TINY, + h22 = static_cast(h[2]) * FF_ONE_PLUS_TINY, h01 = static_cast(h[3]), h02 = static_cast(h[4]), h12 = static_cast(h[5]); @@ -798,7 +798,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void fromfull(optr_t o, iptr_t i) { internal::set(o[0], i[0]); @@ -811,7 +811,7 @@ struct utils: public common_sym //protected: - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak_rows(offset_t i, offset_t j) { if (j < i) return sub2pak_rows(j, i); @@ -822,7 +822,7 @@ struct utils: public common_sym } } - static inline CUDEV + static inline FF_CUDEV offset_t sub2pak(offset_t i, offset_t j) { if (j < i) return sub2pak(j, i); @@ -854,7 +854,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec_backward(hptr_t h, xptr_t x, yptr_t y, reduce_t /*unused*/ = static_cast(0)) { @@ -870,7 +870,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -886,7 +886,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -902,7 +902,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -919,15 +919,15 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) { reduce_t x0 = static_cast(v[0]), x1 = static_cast(v[1]), - h00 = static_cast(h[0]) * JFH_OnePlusTiny, - h11 = static_cast(h[1]) * JFH_OnePlusTiny, + h00 = static_cast(h[0]) * FF_ONE_PLUS_TINY, + h11 = static_cast(h[1]) * FF_ONE_PLUS_TINY, h01 = static_cast(h[2]); if (w) @@ -944,7 +944,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert(optr_t o, hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -955,14 +955,14 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert_(hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) { - reduce_t h00 = static_cast(h[0]) * JFH_OnePlusTiny, - h11 = static_cast(h[1]) * JFH_OnePlusTiny, + reduce_t h00 = static_cast(h[0]) * FF_ONE_PLUS_TINY, + h11 = static_cast(h[1]) * FF_ONE_PLUS_TINY, h01 = static_cast(h[2]); reduce_t idt = static_cast(1) / (h00*h11 - h01*h01); @@ -972,7 +972,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void fromfull(optr_t o, iptr_t i) { internal::set(o[0], i[0]); @@ -994,7 +994,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec_backward(hptr_t h, xptr_t x, yptr_t y, reduce_t /*unused*/ = static_cast(0)) { @@ -1006,7 +1006,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void matvec(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -1018,7 +1018,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void addmatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -1030,7 +1030,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void submatvec_(optr_t o, hptr_t h, iptr_t i, reduce_t /*unused*/ = static_cast(0)) { @@ -1043,7 +1043,7 @@ struct utils: public common_sym template > - static inline CUDEV void + static inline FF_CUDEV void solve_impl_(vptr_t v, hptr_t h, wptr_t w = nullptr, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -1062,7 +1062,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert(optr_t o, hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -1073,7 +1073,7 @@ struct utils: public common_sym template > - static inline CUDEV + static inline FF_CUDEV void invert_(hptr_t h, bptr_t /*b*/ = nullptr, reduce_t /*unused*/ = static_cast(0)) @@ -1083,7 +1083,7 @@ struct utils: public common_sym } template - static inline CUDEV void + static inline FF_CUDEV void fromfull(optr_t o, iptr_t i) { internal::set(o[0], i[0]); diff --git a/include/fastfields/impl/kernels/posdef/utils.h b/include/fastfields/impl/kernels/posdef/utils.h index 5c289c8..98a2afd 100755 --- a/include/fastfields/impl/kernels/posdef/utils.h +++ b/include/fastfields/impl/kernels/posdef/utils.h @@ -3,7 +3,7 @@ #include "fastfields/core/cuda_switch.h" #include "../utils.h" -#define JFH_OnePlusTiny 1.000001 +#define FF_ONE_PLUS_TINY 1.000001 #define FF_UNUSED __attribute__((unused)) // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -28,7 +28,7 @@ // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(posdef) FF_NAMESPACE_BEGIN(internal) @@ -52,21 +52,21 @@ struct Pointer { scalar_t * data; static constexpr offset_t stride = static_cast(S); - // CUHOSTDEV so device kernels can construct these (empty on non-nvcc). - CUHOSTDEV Pointer(scalar_t * ptr): data(ptr) {} - CUHOSTDEV Pointer(const this_type & ptr): data(ptr.data) {} - - inline CUDEV scalar_t& operator[] (offset_t i) const { return data[i*stride]; } - // inline CUDEV const scalar_t& operator[] (offset_t i) const { return data[i*stride]; } - inline CUDEV scalar_t& operator* () const { return *data; } - inline CUDEV operator bool () const { return data != nullptr; } - - inline CUDEV this_type & operator++ () { data += stride; return *this; } - inline CUDEV this_type operator++ (int) { this_type prev = *this; data += stride; return prev; } - inline CUDEV this_type & operator-- () { data -= stride; return *this; } - inline CUDEV this_type operator-- (int) { this_type prev = *this; data -= stride; return prev; } - inline CUDEV this_type & operator += (offset_t N) { data += N * stride; return *this; } - inline CUDEV this_type & operator -= (offset_t N) { data -= N * stride; return *this; } + // FF_CUHOSTDEV so device kernels can construct these (empty on non-nvcc). + FF_CUHOSTDEV Pointer(scalar_t * ptr): data(ptr) {} + FF_CUHOSTDEV Pointer(const this_type & ptr): data(ptr.data) {} + + inline FF_CUDEV scalar_t& operator[] (offset_t i) const { return data[i*stride]; } + // inline FF_CUDEV const scalar_t& operator[] (offset_t i) const { return data[i*stride]; } + inline FF_CUDEV scalar_t& operator* () const { return *data; } + inline FF_CUDEV operator bool () const { return data != nullptr; } + + inline FF_CUDEV this_type & operator++ () { data += stride; return *this; } + inline FF_CUDEV this_type operator++ (int) { this_type prev = *this; data += stride; return prev; } + inline FF_CUDEV this_type & operator-- () { data -= stride; return *this; } + inline FF_CUDEV this_type operator-- (int) { this_type prev = *this; data -= stride; return prev; } + inline FF_CUDEV this_type & operator += (offset_t N) { data += N * stride; return *this; } + inline FF_CUDEV this_type & operator -= (offset_t N) { data -= N * stride; return *this; } }; template @@ -78,25 +78,25 @@ struct Pointer { scalar_t * data; offset_t stride; - // CUHOSTDEV so device kernels can construct these (empty on non-nvcc). - CUHOSTDEV Pointer(scalar_t * ptr): data(ptr), stride(1) {} - CUHOSTDEV Pointer(scalar_t * ptr, offset_t str): data(ptr), stride(str) {} + // FF_CUHOSTDEV so device kernels can construct these (empty on non-nvcc). + FF_CUHOSTDEV Pointer(scalar_t * ptr): data(ptr), stride(1) {} + FF_CUHOSTDEV Pointer(scalar_t * ptr, offset_t str): data(ptr), stride(str) {} template - CUHOSTDEV Pointer(const Pointer & ptr): + FF_CUHOSTDEV Pointer(const Pointer & ptr): data(ptr.data), stride(static_cast(ptr.stride)) {} - inline CUDEV scalar_t& operator[] (offset_t i) const { return data[i*stride]; } - // inline CUDEV const scalar_t& operator[] (offset_t i) const { return data[i*stride]; } - inline CUDEV scalar_t& operator* () const { return *data; } - inline CUDEV operator bool () const { return data != nullptr; } - - inline CUDEV this_type & operator++ () { data += stride; return *this; } - inline CUDEV this_type operator++ (int) { this_type prev = *this; data += stride; return prev; } - inline CUDEV this_type & operator-- () { data -= stride; return *this; } - inline CUDEV this_type operator-- (int) { this_type prev = *this; data -= stride; return prev; } - inline CUDEV this_type & operator += (offset_t N) { data += N * stride; return *this; } - inline CUDEV this_type & operator -= (offset_t N) { data -= N * stride; return *this; } + inline FF_CUDEV scalar_t& operator[] (offset_t i) const { return data[i*stride]; } + // inline FF_CUDEV const scalar_t& operator[] (offset_t i) const { return data[i*stride]; } + inline FF_CUDEV scalar_t& operator* () const { return *data; } + inline FF_CUDEV operator bool () const { return data != nullptr; } + + inline FF_CUDEV this_type & operator++ () { data += stride; return *this; } + inline FF_CUDEV this_type operator++ (int) { this_type prev = *this; data += stride; return prev; } + inline FF_CUDEV this_type & operator-- () { data -= stride; return *this; } + inline FF_CUDEV this_type operator-- (int) { this_type prev = *this; data -= stride; return prev; } + inline FF_CUDEV this_type & operator += (offset_t N) { data += N * stride; return *this; } + inline FF_CUDEV this_type & operator -= (offset_t N) { data -= N * stride; return *this; } }; template @@ -112,7 +112,7 @@ std::ostream& operator<< (std::ostream& os, const Pointer #endif template -inline CUDEV +inline FF_CUDEV Pointer operator+ (Pointer prev, offset_t N) { Pointer next = prev; @@ -121,7 +121,7 @@ Pointer operator+ (Pointer prev, o } template -inline CUDEV +inline FF_CUDEV Pointer operator- (Pointer prev, offset_t N) { Pointer next = prev; @@ -130,21 +130,21 @@ Pointer operator- (Pointer prev, o } template -inline CUDEV +inline FF_CUDEV Pointer pointer(Pointer ptr) { return ptr; } template -inline CUDEV +inline FF_CUDEV Pointer pointer(scalar_t * ptr, offset_t stride) { return Pointer(ptr, stride); } template -inline CUDEV +inline FF_CUDEV Pointer pointer(scalar_t * ptr) { return Pointer(ptr); @@ -345,14 +345,14 @@ struct _return_type { // left = right template -inline CUDEV void set(left_t & left, const right_t & right) +inline FF_CUDEV void set(left_t & left, const right_t & right) { left = static_cast(right); } // left += right template -inline CUDEV void iadd(left_t & left, const right_t & right) +inline FF_CUDEV void iadd(left_t & left, const right_t & right) { left = static_cast(static_cast(left) + static_cast(right)); @@ -360,7 +360,7 @@ inline CUDEV void iadd(left_t & left, const right_t & right) // left -= right template -inline CUDEV void isub(left_t & left, const right_t & right) +inline FF_CUDEV void isub(left_t & left, const right_t & right) { left = static_cast(static_cast(left) - static_cast(right)); @@ -368,7 +368,7 @@ inline CUDEV void isub(left_t & left, const right_t & right) // left *= right template -inline CUDEV void imul(left_t & left, const right_t & right) +inline FF_CUDEV void imul(left_t & left, const right_t & right) { left = static_cast(static_cast(left) * static_cast(right)); @@ -376,7 +376,7 @@ inline CUDEV void imul(left_t & left, const right_t & right) // left /= right template -inline CUDEV void idiv(left_t & left, const right_t & right) +inline FF_CUDEV void idiv(left_t & left, const right_t & right) { left = static_cast(static_cast(left) / static_cast(right)); @@ -384,7 +384,7 @@ inline CUDEV void idiv(left_t & left, const right_t & right) // out += left * right template -inline CUDEV void iaddcmul(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void iaddcmul(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(out) + static_cast(left) * @@ -393,7 +393,7 @@ inline CUDEV void iaddcmul(out_t & out, const left_t & left, const right_t & rig // out -= left * right template -inline CUDEV void isubcmul(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void isubcmul(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(out) - static_cast(left) * @@ -402,7 +402,7 @@ inline CUDEV void isubcmul(out_t & out, const left_t & left, const right_t & rig // out /= left + right template -inline CUDEV void idivcadd(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void idivcadd(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(out) / (static_cast(left) + @@ -411,7 +411,7 @@ inline CUDEV void idivcadd(out_t & out, const left_t & left, const right_t & rig // out = left + right template -inline CUDEV void add(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void add(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(left) + static_cast(right)); @@ -419,7 +419,7 @@ inline CUDEV void add(out_t & out, const left_t & left, const right_t & right) // out = left - right template -inline CUDEV void sub(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void sub(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(left) - static_cast(right)); @@ -427,7 +427,7 @@ inline CUDEV void sub(out_t & out, const left_t & left, const right_t & right) // out = left * right template -inline CUDEV void mul(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void mul(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(left) * static_cast(right)); @@ -435,7 +435,7 @@ inline CUDEV void mul(out_t & out, const left_t & left, const right_t & right) // out = left / right template -inline CUDEV void div(out_t & out, const left_t & left, const right_t & right) +inline FF_CUDEV void div(out_t & out, const left_t & left, const right_t & right) { out = static_cast(static_cast(left) / static_cast(right)); @@ -444,6 +444,6 @@ inline CUDEV void div(out_t & out, const left_t & left, const right_t & right) FF_NAMESPACE_END(internal) FF_NAMESPACE_END(posdef) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_POSDEF_UTILS diff --git a/include/fastfields/impl/kernels/pushpull/1d.h b/include/fastfields/impl/kernels/pushpull/1d.h index 645134f..60dcf60 100755 --- a/include/fastfields/impl/kernels/pushpull/1d.h +++ b/include/fastfields/impl/kernels/pushpull/1d.h @@ -10,7 +10,7 @@ #include "../bounds.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -25,7 +25,7 @@ struct Kernels, Bound, ABS>> { using self = Kernels, Bound, ABS>>; template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -50,7 +50,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -75,7 +75,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [1], @@ -95,7 +95,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -115,7 +115,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -135,7 +135,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -159,7 +159,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -182,7 +182,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t inp [], @@ -198,7 +198,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -235,7 +235,7 @@ struct Kernels, Bound, ABS>> { static const int8_t negate = static_cast(ABS ? 1 : -1); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -265,7 +265,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -295,7 +295,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [1], @@ -318,7 +318,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -349,7 +349,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -369,7 +369,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -412,7 +412,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -457,7 +457,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -484,7 +484,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -538,7 +538,7 @@ struct Kernels, Bound, ABS>> { using utils = PushPullUtils; template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -570,7 +570,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -602,7 +602,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [1], @@ -627,7 +627,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -661,7 +661,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -696,7 +696,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -740,7 +740,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -784,7 +784,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -814,7 +814,7 @@ struct Kernels, Bound, ABS>> { template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -870,7 +870,7 @@ struct Kernels, Bound, ABS>> { using utils = PushPullUtils; template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -904,7 +904,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -938,7 +938,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [1], @@ -965,7 +965,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -1001,7 +1001,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -1038,7 +1038,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -1085,7 +1085,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -1132,7 +1132,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -1164,7 +1164,7 @@ struct Kernels, Bound, ABS>> { template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -1223,7 +1223,7 @@ struct Kernels, Bound, ABS>> { static constexpr int N = utils::bufsize; template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -1259,7 +1259,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -1292,7 +1292,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [1], @@ -1318,7 +1318,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -1356,7 +1356,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -1396,7 +1396,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -1443,7 +1443,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -1493,7 +1493,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -1526,7 +1526,7 @@ struct Kernels, Bound, ABS>> { template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -1577,6 +1577,6 @@ struct Kernels, Bound, ABS>> { FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PUSHPULL_1D diff --git a/include/fastfields/impl/kernels/pushpull/2d.h b/include/fastfields/impl/kernels/pushpull/2d.h index a42e5e8..4818578 100755 --- a/include/fastfields/impl/kernels/pushpull/2d.h +++ b/include/fastfields/impl/kernels/pushpull/2d.h @@ -12,7 +12,7 @@ // TODO: quadratic and cubic specializations -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -30,7 +30,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicby = (BY == bound_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -58,7 +58,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -86,7 +86,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [2], @@ -109,7 +109,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -131,7 +131,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -154,7 +154,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -179,7 +179,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -203,7 +203,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t inp [], @@ -220,7 +220,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -262,7 +262,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicby = (BY == bound_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -307,7 +307,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -354,7 +354,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [2], @@ -394,7 +394,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -445,7 +445,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -494,7 +494,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -565,7 +565,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -634,7 +634,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -686,7 +686,7 @@ struct Kernels, Bound, ABS>> { template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -798,7 +798,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicsy = (IY == spline_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -841,7 +841,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -879,7 +879,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [2], @@ -910,7 +910,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -959,7 +959,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -1012,7 +1012,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -1080,7 +1080,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -1141,7 +1141,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -1185,7 +1185,7 @@ struct Kernels, Bound, ABS>> { template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -1251,6 +1251,6 @@ struct Kernels, Bound, ABS>> { FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PUSHPULL_2D diff --git a/include/fastfields/impl/kernels/pushpull/3d.h b/include/fastfields/impl/kernels/pushpull/3d.h index 4afec5d..bbc8da0 100755 --- a/include/fastfields/impl/kernels/pushpull/3d.h +++ b/include/fastfields/impl/kernels/pushpull/3d.h @@ -12,7 +12,7 @@ // TODO: quadratic and cubic specializations -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -32,7 +32,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicbz = (BZ == bound_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -62,7 +62,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -92,7 +92,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [3], @@ -117,7 +117,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -140,7 +140,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t * out, const scalar_t * inp, @@ -166,7 +166,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -192,7 +192,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -217,7 +217,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t inp [], @@ -235,7 +235,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -280,7 +280,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicbz = (BZ == bound_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -373,7 +373,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -465,7 +465,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [3], @@ -550,7 +550,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -652,7 +652,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -750,7 +750,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -898,7 +898,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -1030,7 +1030,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -1136,7 +1136,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -1277,7 +1277,7 @@ struct Kernels, Bound, ABS>> { static constexpr bool isdynamicsz = (IZ == spline_t::Dynamic); template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], const scalar_t inp [], @@ -1327,7 +1327,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], const scalar_t inp [], @@ -1376,7 +1376,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], const reduce_t loc [3], @@ -1418,7 +1418,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], const scalar_t inp [], @@ -1478,7 +1478,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], const scalar_t inp [], @@ -1548,7 +1548,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], scalar_t gout [], @@ -1631,7 +1631,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -1705,7 +1705,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -1759,7 +1759,7 @@ struct Kernels, Bound, ABS>> { } template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -1847,6 +1847,6 @@ struct Kernels, Bound, ABS>> { FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PUSHPULL_3D diff --git a/include/fastfields/impl/kernels/pushpull/nd.h b/include/fastfields/impl/kernels/pushpull/nd.h index bebc64c..9776580 100755 --- a/include/fastfields/impl/kernels/pushpull/nd.h +++ b/include/fastfields/impl/kernels/pushpull/nd.h @@ -12,7 +12,7 @@ // TODO + FIXME -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -27,7 +27,7 @@ struct Kernels,Bound<>,ABS>> { using maybe = PushPullMaybe; template - static CUDEV + static FF_CUDEV void pull( scalar_t out [], scalar_t inp [], @@ -88,7 +88,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void push(scalar_t * out, scalar_t * inp, reduce_t x, offset_t nx, offset_t sx, reduce_t y, offset_t ny, offset_t sy, @@ -140,7 +140,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void count(scalar_t * out, reduce_t x, offset_t nx, offset_t sx, reduce_t y, offset_t ny, offset_t sy, @@ -187,7 +187,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void grad(scalar_t * out, scalar_t * inp, reduce_t x, offset_t nx, offset_t sx, reduce_t y, offset_t ny, offset_t sy, @@ -255,7 +255,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void pull_backward(scalar_t * out, scalar_t * gout, scalar_t * inp, scalar_t * ginp, reduce_t x, offset_t nx, offset_t osx, offset_t isx, @@ -320,7 +320,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void push_backward(scalar_t * out, scalar_t * gout, scalar_t * inp, scalar_t * ginp, reduce_t x, offset_t nx, offset_t sx, @@ -377,7 +377,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void count_backward(scalar_t * gout, scalar_t * ginp, reduce_t x, offset_t nx, offset_t sx, reduce_t y, offset_t ny, offset_t sy, @@ -418,7 +418,7 @@ struct Kernels,Bound<>,ABS>> { } template - static CUDEV + static FF_CUDEV void grad_backward(scalar_t * out, scalar_t * gout, scalar_t * inp, scalar_t * ginp, reduce_t x, offset_t nx, offset_t osx, offset_t isx, @@ -484,6 +484,6 @@ struct Kernels,Bound<>,ABS>> { FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif FF_PUSHPULL_ND diff --git a/include/fastfields/impl/kernels/pushpull/utils.h b/include/fastfields/impl/kernels/pushpull/utils.h index ccf7359..c7313df 100755 --- a/include/fastfields/impl/kernels/pushpull/utils.h +++ b/include/fastfields/impl/kernels/pushpull/utils.h @@ -5,7 +5,7 @@ #include "../bounds.h" #include "../meta.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(pushpull) @@ -33,7 +33,7 @@ struct Kernels { static constexpr int D = Config::dim; template - CUDEV static inline + FF_CUDEV static inline void pull( scalar_t out [], // pointer to output voxel const scalar_t inp [], // pointer to input tensor @@ -48,7 +48,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void push( scalar_t out [], // pointer to output tensor const scalar_t inp [], // pointer to input voxel @@ -63,7 +63,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void count( scalar_t out [], // pointer to output tensor const reduce_t loc [D], // output location in which to push ones @@ -74,7 +74,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void grad( scalar_t out [], // pointer to output voxel const scalar_t inp [], // pointer to input tensor @@ -90,7 +90,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void hess( scalar_t out [], // pointer to output voxel const scalar_t inp [], // pointer to input tensor @@ -106,7 +106,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void pull_backward( scalar_t out [], // pointer to output tensor scalar_t gout [], // pointer to output gradient @@ -126,7 +126,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void push_backward( scalar_t out [], scalar_t gout [], @@ -145,7 +145,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void count_backward( scalar_t gout [], const scalar_t ginp [], @@ -158,7 +158,7 @@ struct Kernels { ); template - CUDEV static inline + FF_CUDEV static inline void grad_backward( scalar_t out [], scalar_t gout [], @@ -256,7 +256,7 @@ struct InFOV {}; template struct InFOV { template - static inline CUDEV bool + static inline FF_CUDEV bool infov(const scalar_t * loc, const offset_t * size, offset_t stride=1) { return true; } @@ -265,7 +265,7 @@ struct InFOV { template struct InFOV { // Limits at voxel centers template - static inline CUDEV bool + static inline FF_CUDEV bool infov(const scalar_t * loc, const offset_t * size, offset_t stride=1) { # pragma unroll for (int d=0; d < D; ++d, loc += stride) { @@ -282,7 +282,7 @@ struct InFOV { // Limits at voxel centers template struct InFOV { // Limits at voxel edges template - static inline CUDEV bool + static inline FF_CUDEV bool infov(const scalar_t * loc, const offset_t * size, offset_t stride=1) { # pragma unroll for (int d=0; d < D; ++d, loc += stride) { @@ -300,7 +300,7 @@ struct InFOV { // Limits at voxel edges template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, offset_t nx) { return true; } @@ -309,7 +309,7 @@ struct InFOV { template <> struct InFOV { // Limits at voxel centers template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, offset_t nx) { if (x < -FF_EXTRAPOLATE_TINY) return false; @@ -322,7 +322,7 @@ struct InFOV { // Limits at voxel centers template <> struct InFOV { // Limits at voxel edges template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, offset_t nx) { if (x < -0.5 - FF_EXTRAPOLATE_TINY) return false; @@ -335,7 +335,7 @@ struct InFOV { // Limits at voxel edges template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, offset_t nx, offset_t ny) { return true; } @@ -344,7 +344,7 @@ struct InFOV { template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, offset_t nx, offset_t ny) { return InFOV<0, 1>::infov(x, nx) && InFOV<0, 1>::infov(y, ny); @@ -354,7 +354,7 @@ struct InFOV { template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, offset_t nx, offset_t ny) { return InFOV<-1, 1>::infov(x, nx) && InFOV<-1, 1>::infov(y, ny); @@ -364,7 +364,7 @@ struct InFOV { template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, scalar_t z, offset_t nx, offset_t ny, offset_t nz) { return true; @@ -374,7 +374,7 @@ struct InFOV { template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, scalar_t z, offset_t nx, offset_t ny, offset_t nz) { return InFOV<0, 1>::infov(x, nx) && @@ -386,7 +386,7 @@ struct InFOV { template <> struct InFOV { template - static CUDEV bool + static FF_CUDEV bool infov(scalar_t x, scalar_t y, scalar_t z, offset_t nx, offset_t ny, offset_t nz) { return InFOV<-1, 1>::infov(x, nx) && @@ -400,7 +400,7 @@ struct InFOV { template struct PushPullMaybe { template - static inline CUDEV + static inline FF_CUDEV const T& fabs(const T& val) { return val; } }; @@ -408,7 +408,7 @@ struct PushPullMaybe { template <> struct PushPullMaybe { template - static inline CUDEV + static inline FF_CUDEV T fabs(const T& val) { return ::fabs(val); } }; @@ -425,7 +425,7 @@ struct PushPullAnyUtils { template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index(reduce_t x, offset_t size, offset_t i[], reduce_t w[], int8_t s[]) { offset_t b0, b1; @@ -444,7 +444,7 @@ struct PushPullAnyUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex(reduce_t x, offset_t size, offset_t i[], reduce_t w[], reduce_t g[], int8_t s[]) { offset_t b0, b1; @@ -467,7 +467,7 @@ struct PushPullAnyUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex(reduce_t x, offset_t size, offset_t i[], reduce_t w[], reduce_t g[], reduce_t h[], int8_t s[]) { @@ -520,7 +520,7 @@ struct PushPullUtils { static constexpr int bufsize = SplineBufSize::bufsize; template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -549,7 +549,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex( reduce_t x, offset_t size, @@ -583,7 +583,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex( reduce_t x, offset_t size, @@ -631,7 +631,7 @@ struct PushPullUtils { static constexpr spline_t S = Z; template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -654,7 +654,7 @@ struct PushPullUtils { // Weight-less overload: nearest interpolation has an implicit weight // of 1, so callers that do not need the weight buffer can omit it. template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -670,7 +670,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex( reduce_t x, offset_t size, @@ -690,7 +690,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex( reduce_t x, offset_t size, @@ -721,7 +721,7 @@ struct PushPullUtils { static constexpr spline_t S = L; template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -745,7 +745,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex( reduce_t x, offset_t size, @@ -768,7 +768,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex( reduce_t x, offset_t size, @@ -800,7 +800,7 @@ struct PushPullUtils { static constexpr spline_t S = Q; template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -829,7 +829,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex( reduce_t x, offset_t size, @@ -862,7 +862,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex( reduce_t x, offset_t size, @@ -909,7 +909,7 @@ struct PushPullUtils { static constexpr spline_t S = C; template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t index( reduce_t x, offset_t size, @@ -942,7 +942,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t gindex( reduce_t x, offset_t size, @@ -980,7 +980,7 @@ struct PushPullUtils { } template - static inline CUDEV offset_t + static inline FF_CUDEV offset_t hindex( reduce_t x, offset_t size, @@ -1025,6 +1025,6 @@ struct PushPullUtils { FF_NAMESPACE_END(pushpull) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_PUSHPULL_UTILS diff --git a/include/fastfields/impl/kernels/regularisers/field/1d.h b/include/fastfields/impl/kernels/regularisers/field/1d.h index 9c95d3b..ab2b795 100755 --- a/include/fastfields/impl/kernels/regularisers/field/1d.h +++ b/include/fastfields/impl/kernels/regularisers/field/1d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -29,10 +29,10 @@ struct Kernels> // `bound::type::Dynamic` (single instantiation, runtime dispatch). bound::dyn bound_utils_x; - inline CUDEV Kernels() {} + inline FF_CUDEV Kernels() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV Kernels(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV Kernels(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) {} typedef scalar_t & (*OpType)(scalar_t &, const reduce_t &); @@ -42,12 +42,12 @@ struct Kernels> static const offset_t kernelsize_absolute = C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_absolute(offset_t nc = C) { return C < 0 ? nc : C; } /// kernel <- [abs, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t kernel [], const reduce_t absolute [], @@ -62,7 +62,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t out [], const scalar_t inp [], @@ -80,7 +80,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute( scalar_t out [], offset_t osc, @@ -96,7 +96,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute( scalar_t out [], offset_t osc, @@ -113,12 +113,12 @@ struct Kernels> static const offset_t kernelsize_membrane = (D+1)*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane(offset_t nc = C) { return (D+1) * (C < 0 ? nc : C); } /// kernel <- [abs, w1, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -138,7 +138,7 @@ struct Kernels> } /// kernel <- [w00, w10, w01, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -160,7 +160,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t out [], const scalar_t inp [], @@ -202,7 +202,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t out [], offset_t sc, @@ -228,7 +228,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t out [], offset_t osc, @@ -254,12 +254,12 @@ struct Kernels> static const offset_t kernelsize_bending = 3*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending(offset_t nc = C) { return 3 * (C < 0 ? nc : C); } /// kernel <- [abs, w1, w2, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -281,7 +281,7 @@ struct Kernels> } /// kernel <- [w0, w1, w2, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -305,7 +305,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t out [], const scalar_t inp [], @@ -357,7 +357,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t out [], offset_t sc, @@ -387,7 +387,7 @@ struct Kernels> // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t out [], offset_t osc, @@ -425,7 +425,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_rls( scalar_t out [], const scalar_t inp [], @@ -446,7 +446,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_rls( scalar_t out [], const scalar_t wgt [], @@ -467,7 +467,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t out [], const scalar_t inp [], @@ -486,7 +486,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t out [], const scalar_t wgt [], @@ -506,11 +506,11 @@ struct Kernels> static const offset_t kernelsize_membrane_rls = kernelsize_membrane; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane_rls(offset_t nc = C) { return get_kernelsize_membrane(nc); } - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_rls( reduce_t kernel [], const reduce_t absolute [], @@ -527,7 +527,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_rls( scalar_t out [], const scalar_t inp [], @@ -597,7 +597,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_rls( scalar_t out [], const scalar_t wgt [], @@ -653,7 +653,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t out [], const scalar_t inp [], @@ -720,7 +720,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t out [], const scalar_t wgt [], @@ -772,11 +772,11 @@ struct Kernels> static const offset_t kernelsize_bending_rls = kernelsize_bending; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending_rls(offset_t nc = C) { return get_kernelsize_bending(nc); } - inline CUDEV void + inline FF_CUDEV void make_kernel_bending_rls( reduce_t kernel [], const reduce_t absolute [], @@ -810,7 +810,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_rls( scalar_t out [], const scalar_t inp [], @@ -902,7 +902,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_rls( scalar_t out [], const scalar_t wgt [], @@ -969,7 +969,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_jrls( scalar_t out [], const scalar_t inp [], @@ -1063,7 +1063,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_jrls( scalar_t out [], const scalar_t wgt [], @@ -1128,6 +1128,6 @@ struct Kernels> FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FIELD_1D diff --git a/include/fastfields/impl/kernels/regularisers/field/2d.h b/include/fastfields/impl/kernels/regularisers/field/2d.h index c918429..fb6c01c 100755 --- a/include/fastfields/impl/kernels/regularisers/field/2d.h +++ b/include/fastfields/impl/kernels/regularisers/field/2d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -31,10 +31,10 @@ struct Kernels> bound::dyn bound_utils_x; bound::dyn bound_utils_y; - inline CUDEV Kernels() {} + inline FF_CUDEV Kernels() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV Kernels(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV Kernels(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) , bound_utils_y(bnd[1]) {} typedef scalar_t & (*OpType)(scalar_t &, const reduce_t &); @@ -45,12 +45,12 @@ struct Kernels> static const offset_t kernelsize_absolute = C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_absolute(offset_t nc = C) { return C < 0 ? nc : C; } /// kernel <- [abs, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t kernel [], const reduce_t absolute [], @@ -65,7 +65,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t out [], const scalar_t inp [], @@ -83,7 +83,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute( scalar_t out [], offset_t osc, @@ -99,7 +99,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute( scalar_t out [], offset_t osc, @@ -116,12 +116,12 @@ struct Kernels> static const offset_t kernelsize_membrane = (D+1)*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane(offset_t nc = C) { return (D+1) * (C < 0 ? nc : C); } /// kernel <- [abs, w10, w01, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -142,7 +142,7 @@ struct Kernels> } /// kernel <- [w00, w10, w01, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -165,7 +165,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t out [], const scalar_t inp [], @@ -212,7 +212,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t out [], offset_t sc, @@ -240,7 +240,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t out [], offset_t osc, @@ -268,12 +268,12 @@ struct Kernels> static const offset_t kernelsize_bending = 6*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending(offset_t nc = C) { return 6 * (C < 0 ? nc : C); } /// kernel <- [abs, w10, w01, w20, w02, w11, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -298,7 +298,7 @@ struct Kernels> } /// kernel <- [w00, w10, w01, w20, w02, w11, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -327,7 +327,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t out [], const scalar_t inp [], @@ -392,7 +392,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t out [], offset_t sc, @@ -431,7 +431,7 @@ struct Kernels> // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t out [], offset_t osc, @@ -476,7 +476,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_rls( scalar_t out [], const scalar_t inp [], @@ -497,7 +497,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_rls( scalar_t out [], const scalar_t wgt [], @@ -518,7 +518,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t out [], const scalar_t inp [], @@ -537,7 +537,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t out [], const scalar_t wgt [], @@ -557,11 +557,11 @@ struct Kernels> static const offset_t kernelsize_membrane_rls = kernelsize_membrane; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane_rls(offset_t nc = C) { return get_kernelsize_membrane(nc); } - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_rls( reduce_t kernel [], const reduce_t absolute [], @@ -578,7 +578,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_rls( scalar_t out [], const scalar_t inp [], @@ -659,7 +659,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_rls( scalar_t out [], const scalar_t wgt [], @@ -721,7 +721,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t out [], const scalar_t inp [], @@ -799,7 +799,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t out [], const scalar_t wgt [], @@ -857,11 +857,11 @@ struct Kernels> static const offset_t kernelsize_bending_rls = kernelsize_bending; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending_rls(offset_t nc = C) { return get_kernelsize_bending(nc); } - inline CUDEV void + inline FF_CUDEV void make_kernel_bending_rls( reduce_t kernel [], const reduce_t absolute [], @@ -899,7 +899,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_rls( scalar_t out [], const scalar_t inp [], @@ -1048,7 +1048,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_rls( scalar_t out [], const scalar_t wgt [], @@ -1161,7 +1161,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_jrls( scalar_t out [], const scalar_t inp [], @@ -1313,7 +1313,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_jrls( scalar_t out [], const scalar_t wgt [], @@ -1424,7 +1424,7 @@ struct Kernels> FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FIELD_2D diff --git a/include/fastfields/impl/kernels/regularisers/field/3d.h b/include/fastfields/impl/kernels/regularisers/field/3d.h index 6ced3d5..9eb62b8 100755 --- a/include/fastfields/impl/kernels/regularisers/field/3d.h +++ b/include/fastfields/impl/kernels/regularisers/field/3d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -32,10 +32,10 @@ struct Kernels> bound::dyn bound_utils_y; bound::dyn bound_utils_z; - inline CUDEV Kernels() {} + inline FF_CUDEV Kernels() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV Kernels(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV Kernels(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) , bound_utils_y(bnd[1]) , bound_utils_z(bnd[2]) {} @@ -47,12 +47,12 @@ struct Kernels> static const offset_t kernelsize_absolute = C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_absolute(offset_t nc = C) { return C < 0 ? nc : C; } /// kernel <- [abs, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t kernel [], const reduce_t absolute [], @@ -67,7 +67,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t out [], const scalar_t inp [], @@ -85,7 +85,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute( scalar_t out [], offset_t osc, @@ -101,7 +101,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute( scalar_t out [], offset_t osc, @@ -118,12 +118,12 @@ struct Kernels> static const offset_t kernelsize_membrane = (D+1)*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane(offset_t nc = C) { return (D+1) * (C < 0 ? nc : C); } /// kernel <- [abs, w100, w010, w001, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -145,7 +145,7 @@ struct Kernels> } /// kernel <- [w00, w100, w010, w001, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t kernel [], const reduce_t absolute [], @@ -169,7 +169,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t out [], const scalar_t inp [], @@ -221,7 +221,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t out [], offset_t sc, @@ -252,7 +252,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t out [], offset_t osc, @@ -282,13 +282,13 @@ struct Kernels> static const offset_t kernelsize_bending = 10*C; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending(offset_t nc = C) { return 10 * (C < 0 ? nc : C); } /// kernel <- [ /// abs, w100, w010, w001, w200, w020, w002, w110, w101, w011, ...] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -318,7 +318,7 @@ struct Kernels> /// kernel <- [ /// w000, w100, w010, w001, w200, w020, w002, w110, w101, w011, ...] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t kernel [], const reduce_t absolute [], @@ -352,7 +352,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t out [], const scalar_t inp [], @@ -431,7 +431,7 @@ struct Kernels> // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t out [], offset_t sc, @@ -481,7 +481,7 @@ struct Kernels> // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t out [], offset_t osc, @@ -531,7 +531,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_rls( scalar_t out [], const scalar_t inp [], @@ -552,7 +552,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_rls( scalar_t out [], const scalar_t wgt [], @@ -573,7 +573,7 @@ struct Kernels> // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t out [], const scalar_t inp [], @@ -592,7 +592,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t out [], const scalar_t wgt [], @@ -612,11 +612,11 @@ struct Kernels> static const offset_t kernelsize_membrane_rls = kernelsize_membrane; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_membrane_rls(offset_t nc = C) { return get_kernelsize_membrane(nc); } - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_rls( reduce_t kernel [], const reduce_t absolute [], @@ -633,7 +633,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_rls( scalar_t out [], const scalar_t inp [], @@ -726,7 +726,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_rls( scalar_t out [], const scalar_t wgt [], @@ -800,7 +800,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t out [], const scalar_t inp [], @@ -890,7 +890,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t out [], const scalar_t wgt [], @@ -960,11 +960,11 @@ struct Kernels> static const offset_t kernelsize_bending_rls = kernelsize_bending; - CUDEV inline offset_t + FF_CUDEV inline offset_t get_kernelsize_bending_rls(offset_t nc = C) { return get_kernelsize_bending(nc); } - inline CUDEV void + inline FF_CUDEV void make_kernel_bending_rls( reduce_t kernel [], const reduce_t absolute [], @@ -1007,7 +1007,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_rls( scalar_t * out, const scalar_t inp [], @@ -1215,7 +1215,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_rls( scalar_t out [], const scalar_t wgt [], @@ -1380,7 +1380,7 @@ struct Kernels> // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_jrls( scalar_t out [], const scalar_t inp [], @@ -1590,7 +1590,7 @@ struct Kernels> // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_jrls( scalar_t out [], const scalar_t wgt [], @@ -1752,6 +1752,6 @@ struct Kernels> FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FIELD_3D diff --git a/include/fastfields/impl/kernels/regularisers/field/utils.h b/include/fastfields/impl/kernels/regularisers/field/utils.h index 25221f8..4c58910 100755 --- a/include/fastfields/impl/kernels/regularisers/field/utils.h +++ b/include/fastfields/impl/kernels/regularisers/field/utils.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "../../meta.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_field) @@ -52,34 +52,34 @@ using RegField = Kernels -inline CUDEV T & set(T & out, const IT & in) +inline FF_CUDEV T & set(T & out, const IT & in) { out = static_cast(in); return out; } template -inline CUDEV T & iadd(T & out, const IT & in) +inline FF_CUDEV T & iadd(T & out, const IT & in) { out = static_cast(static_cast(out) + in); return out; } template -inline CUDEV T & isub(T & out, const IT & in) +inline FF_CUDEV T & isub(T & out, const IT & in) { out = static_cast(static_cast(out) - in); return out; } template -inline CUDEV T add(const T & out, const IT & in) +inline FF_CUDEV T add(const T & out, const IT & in) { return static_cast(static_cast(out) + in); } template -inline CUDEV T sub(const T & out, const IT & in) +inline FF_CUDEV T sub(const T & out, const IT & in) { return static_cast(static_cast(out) - in); } @@ -107,7 +107,7 @@ struct Op<'-', scalar_t, reduce_t> { //---------------------------------------------------------------------- template -CUDEV inline +FF_CUDEV inline U center_offset(const U * size, const U * stride) { U offset = 0; @@ -118,7 +118,7 @@ U center_offset(const U * size, const U * stride) } template -CUDEV inline +FF_CUDEV inline bool patch1(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -129,7 +129,7 @@ bool patch1(const offset_t loc[N], offset_t n) } template -CUDEV inline +FF_CUDEV inline bool patch2(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -141,7 +141,7 @@ bool patch2(const offset_t loc[N], offset_t n) } template -CUDEV inline +FF_CUDEV inline bool patch3(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -155,6 +155,6 @@ bool patch3(const offset_t loc[N], offset_t n) FF_NAMESPACE_END(reg_field) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_UTILS diff --git a/include/fastfields/impl/kernels/regularisers/flow/1d.h b/include/fastfields/impl/kernels/regularisers/flow/1d.h index 05df091..d321cbf 100755 --- a/include/fastfields/impl/kernels/regularisers/flow/1d.h +++ b/include/fastfields/impl/kernels/regularisers/flow/1d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -21,10 +21,10 @@ struct RegFlow { // `bound::type::Dynamic` (single instantiation, runtime dispatch). bound::dyn bound_utils_x; - inline CUDEV RegFlow() {} + inline FF_CUDEV RegFlow() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV RegFlow(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV RegFlow(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) {} typedef scalar_t & (*OpType)(scalar_t &, const reduce_t &); @@ -35,7 +35,7 @@ struct RegFlow { static const int kernelsize_absolute = 1; /// kernel <- [absx] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t * kernel, reduce_t absolute, const reduce_t voxel_size[1]) { @@ -46,7 +46,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t * out, const scalar_t * inp, offset_t osc, offset_t isc, const reduce_t kernel[1]) @@ -57,7 +57,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[1]) { op(out[0], kernel[0]); @@ -66,7 +66,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[1]) { return kernel_absolute(out, osc, kernel); @@ -79,7 +79,7 @@ struct RegFlow { static const int kernelsize_membrane = 2; /// kernel <- [absx, wx1] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[3]) @@ -91,7 +91,7 @@ struct RegFlow { } /// kernel <- [wx0, wx1] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[2]) @@ -105,7 +105,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t * out, const scalar_t * inp, const offset_t loc[1], const offset_t size[1], const offset_t stride[1], @@ -139,7 +139,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t * out, offset_t sc, const offset_t stride[1], const reduce_t kernel[2]) @@ -160,7 +160,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t * out, offset_t osc, const offset_t loc[1], const offset_t size[1], @@ -182,7 +182,7 @@ struct RegFlow { static const int kernelsize_bending = 3; /// kernel <- [absx, wx100, wx200] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[1]) @@ -200,7 +200,7 @@ struct RegFlow { } /// kernel <- [wx000, wx100, wx200] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[1]) @@ -220,7 +220,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t * out, const scalar_t * inp, const offset_t loc[1], const offset_t size[1], @@ -265,7 +265,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t * out, offset_t sc, const offset_t stride[1], const reduce_t kernel[3]) @@ -289,7 +289,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t * out, offset_t osc, const offset_t loc[1], const offset_t size[1], @@ -320,7 +320,7 @@ struct RegFlow { static const int kernelsize_all = 3; /// kernel <- [absx, wx100, wx200] - inline CUDEV void + inline FF_CUDEV void make_kernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, @@ -339,7 +339,7 @@ struct RegFlow { } /// kernel <- [wx000, wx100, wx200] - inline CUDEV void make_fullkernel_all( + inline FF_CUDEV void make_fullkernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, reduce_t shears, reduce_t div, const reduce_t voxel_size[1]) @@ -359,7 +359,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_all( scalar_t * out, const scalar_t * inp, const offset_t loc[1], const offset_t size[1], @@ -404,7 +404,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_all( scalar_t * out, const offset_t sc[2], const offset_t stride[1], const reduce_t kernel[3]) @@ -427,7 +427,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_all( scalar_t * out, offset_t osc, const offset_t loc[1], const offset_t size[1], @@ -458,7 +458,7 @@ struct RegFlow { static const int kernelsize_lame = 2; /// kernel <- [absx, wx100] - CUDEV inline void + FF_CUDEV inline void make_kernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[1]) @@ -471,7 +471,7 @@ struct RegFlow { } /// kernel <- [wx000, wx100] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[1]) @@ -489,7 +489,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_lame( scalar_t * out, const scalar_t * inp, const offset_t loc[1], const offset_t size[1], @@ -527,7 +527,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_lame( scalar_t * out, const offset_t sc[2], const offset_t stride[1], const reduce_t kernel[2]) @@ -548,7 +548,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_lame( scalar_t * out, offset_t osc, const offset_t loc[1], const offset_t size[1], @@ -576,7 +576,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, offset_t osc, offset_t isc, const reduce_t kernel[1]) @@ -588,7 +588,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t * out, const scalar_t * wgt, offset_t osc, const reduce_t kernel[1]) @@ -603,7 +603,7 @@ struct RegFlow { static const int kernelsize_membrane_jrls = kernelsize_membrane; - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[1]) @@ -616,7 +616,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[1], const offset_t size[1], @@ -675,7 +675,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[1], const offset_t size[1], @@ -726,7 +726,7 @@ struct RegFlow { * * wx100 = -(0.5*div + shears) */ - CUDEV inline void + FF_CUDEV inline void make_kernel_lame_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[2]) @@ -740,7 +740,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_lame_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[1], const offset_t size[1], @@ -802,7 +802,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV + inline FF_CUDEV void diag_lame_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[1], const offset_t size[1], @@ -846,6 +846,6 @@ struct RegFlow { FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FLOW_1D diff --git a/include/fastfields/impl/kernels/regularisers/flow/2d.h b/include/fastfields/impl/kernels/regularisers/flow/2d.h index 82afc65..a9fcd92 100755 --- a/include/fastfields/impl/kernels/regularisers/flow/2d.h +++ b/include/fastfields/impl/kernels/regularisers/flow/2d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -24,10 +24,10 @@ struct RegFlow { bound::dyn bound_utils_xt; bound::dyn bound_utils_yt; - inline CUDEV RegFlow() {} + inline FF_CUDEV RegFlow() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV RegFlow(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV RegFlow(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) , bound_utils_y(bnd[1]) , bound_utils_xt(bound::transpose(bnd[0])) @@ -41,7 +41,7 @@ struct RegFlow { static const int kernelsize_absolute = 2; /// kernel <- [absx, absy] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t * kernel, reduce_t absolute, const reduce_t voxel_size[2]) { @@ -54,7 +54,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t * out, const scalar_t * inp, offset_t osc, offset_t isc, const reduce_t kernel[2]) @@ -66,7 +66,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[2]) { op(out[0], kernel[0]); @@ -76,7 +76,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[2]) { return kernel_absolute(out, osc, kernel); @@ -90,7 +90,7 @@ struct RegFlow { /// kernel <- [absx, wx10, wx01, /// absy, wy10, wy01] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[3]) @@ -107,7 +107,7 @@ struct RegFlow { /// kernel <- [wx00, wx10, wx01, /// wy00, wy10, wy01] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[2]) @@ -125,7 +125,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t * out, const scalar_t * inp, const offset_t loc[2], const offset_t size[2], const offset_t stride[2], @@ -165,7 +165,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t * out, offset_t sc, const offset_t stride[2], const reduce_t kernel[6]) @@ -189,7 +189,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t * out, offset_t osc, const offset_t loc[2], const offset_t size[2], @@ -216,7 +216,7 @@ struct RegFlow { /// kernel <- [ /// absx, wx100, wx010, wx200, wx020, wx110, /// absy, wy100, wy010, wy200, wy020, wy110] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[2]) @@ -249,7 +249,7 @@ struct RegFlow { /// kernel <- [ /// wx000, wx100, wx010, wx200, wx020, wx110, /// wy000, wy100, wy010, wy200, wy020, wy110] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[2]) @@ -284,7 +284,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t * out, const scalar_t * inp, const offset_t loc[2], const offset_t size[2], @@ -343,7 +343,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t * out, offset_t sc, const offset_t stride[2], const reduce_t kernel[12]) @@ -377,7 +377,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t * out, offset_t osc, const offset_t loc[2], const offset_t size[2], @@ -419,7 +419,7 @@ struct RegFlow { /// absx, wx100, wx010, wx200, wx020, wx110, /// absy, wy100, wy010, wy200, wy020, wy110, /// ww] - inline CUDEV void + inline FF_CUDEV void make_kernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, @@ -457,7 +457,7 @@ struct RegFlow { /// wx000, wx100, wx010, wx200, wx020, wx110, /// wy000, wy100, wy010, wy200, wy020, wy110, /// ww] - inline CUDEV void make_fullkernel_all( + inline FF_CUDEV void make_fullkernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, reduce_t shears, reduce_t div, const reduce_t voxel_size[2]) @@ -495,7 +495,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_all( scalar_t * out, const scalar_t * inp, const offset_t loc[2], const offset_t size[2], @@ -611,7 +611,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_all( scalar_t * out, const offset_t sc[2], const offset_t stride[2], const reduce_t kernel[13]) @@ -653,7 +653,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_all( scalar_t * out, offset_t osc, const offset_t loc[2], const offset_t size[2], @@ -699,7 +699,7 @@ struct RegFlow { /// absx, wx100, wx010, /// absy, wy100, wy010, /// ww] - CUDEV inline void + FF_CUDEV inline void make_kernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[2]) @@ -726,7 +726,7 @@ struct RegFlow { /// wx000, wx100, wx010, /// wy000, wy100, wy010, /// ww] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[3]) @@ -752,7 +752,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_lame( scalar_t * out, const scalar_t * inp, const offset_t loc[2], const offset_t size[2], @@ -831,7 +831,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_lame( scalar_t * out, const offset_t sc[2], const offset_t stride[2], const reduce_t kernel[7]) @@ -864,7 +864,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_lame( scalar_t * out, offset_t osc, const offset_t loc[2], const offset_t size[2], @@ -897,7 +897,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, offset_t osc, offset_t isc, const reduce_t kernel[2]) @@ -910,7 +910,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t * out, const scalar_t * wgt, offset_t osc, const reduce_t kernel[2]) @@ -926,7 +926,7 @@ struct RegFlow { static const int kernelsize_membrane_jrls = kernelsize_membrane; - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[2]) @@ -939,7 +939,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[2], const offset_t size[2], @@ -1009,7 +1009,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[2], const offset_t size[2], @@ -1081,7 +1081,7 @@ struct RegFlow { * where lx = 1/(vx[0]*vx[0]) * ly = 1/(vx[1]*vx[1]) */ - inline CUDEV void + inline FF_CUDEV void make_kernel_bending_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[3]) @@ -1107,7 +1107,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[2], const offset_t size[2], @@ -1300,7 +1300,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -1469,7 +1469,7 @@ struct RegFlow { * d2 = 0.25*div * s2 = 0.25*shears */ - CUDEV inline void + FF_CUDEV inline void make_kernel_lame_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[2]) @@ -1486,7 +1486,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_lame_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[2], const offset_t size[2], @@ -1597,7 +1597,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV + inline FF_CUDEV void diag_lame_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[2], const offset_t size[2], @@ -1646,6 +1646,6 @@ struct RegFlow { FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FLOW_2D diff --git a/include/fastfields/impl/kernels/regularisers/flow/3d.h b/include/fastfields/impl/kernels/regularisers/flow/3d.h index e287653..05cf366 100755 --- a/include/fastfields/impl/kernels/regularisers/flow/3d.h +++ b/include/fastfields/impl/kernels/regularisers/flow/3d.h @@ -5,7 +5,7 @@ #include "../../utils.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -26,10 +26,10 @@ struct RegFlow { bound::dyn bound_utils_yt; bound::dyn bound_utils_zt; - inline CUDEV RegFlow() {} + inline FF_CUDEV RegFlow() {} // Runtime boundary conditions; ignored by statically instantiated axes. - explicit inline CUDEV RegFlow(const ::FF::bound::BoundVec & bnd) + explicit inline FF_CUDEV RegFlow(const ::FF_NS::bound::BoundVec & bnd) : bound_utils_x(bnd[0]) , bound_utils_y(bnd[1]) , bound_utils_z(bnd[2]) @@ -45,7 +45,7 @@ struct RegFlow { static const int kernelsize_absolute = 3; /// kernel <- [absx, absy, absz] - CUDEV inline void + FF_CUDEV inline void make_kernel_absolute( reduce_t * kernel, reduce_t absolute, const reduce_t voxel_size[3]) { @@ -59,7 +59,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_absolute( scalar_t * out, const scalar_t * inp, offset_t osc, offset_t isc, const reduce_t kernel[3]) @@ -72,7 +72,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[3]) { op(out[0], kernel[0]); @@ -83,7 +83,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute(scalar_t * out, offset_t osc, const reduce_t kernel[3]) { return kernel_absolute(out, osc, kernel); @@ -98,7 +98,7 @@ struct RegFlow { /// kernel <- [absx, wx100, wx010, wx001, /// absy, wy100, wy010, wy001, /// absz, wz100, wz010, wz001] - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[3]) @@ -122,7 +122,7 @@ struct RegFlow { /// kernel <- [wx000, wx100, wx010, wx001, /// wy000, wy100, wy010, wy001, /// wz000, wz100, wz010, wz001] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_membrane( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[3]) @@ -146,7 +146,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane( scalar_t * out, const scalar_t * inp, const offset_t loc[3], const offset_t size[3], const offset_t stride[3], @@ -192,7 +192,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_membrane( scalar_t * out, offset_t sc, const offset_t stride[3], const reduce_t kernel[12]) @@ -220,7 +220,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane( scalar_t * out, offset_t osc, const offset_t loc[3], const offset_t size[3], @@ -251,7 +251,7 @@ struct RegFlow { /// absx, wx100, wx010, wx001, wx200, wx020, wx002, wx110, wx101, wx011, /// absy, wy100, wy010, wy001, wy200, wy020, wy002, wy110, wy101, wy011, /// absz, wz100, wz010, wz001, wz200, wz020, wz002, wz110, wz101, wz011] - CUDEV inline void + FF_CUDEV inline void make_kernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[3]) @@ -308,7 +308,7 @@ struct RegFlow { /// wx000, wx100, wx010, wx001, wx200, wx020, wx002, wx110, wx101, wx011, /// wy000, wy100, wy010, wy001, wy200, wy020, wy002, wy110, wy101, wy011, /// wz000, wz100, wz010, wz001, wz200, wz020, wz002, wz110, wz101, wz011] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_bending( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[3]) @@ -366,7 +366,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending( scalar_t * out, const scalar_t * inp, const offset_t loc[3], const offset_t size[3], @@ -440,7 +440,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_bending( scalar_t * out, offset_t sc, const offset_t stride[3], const reduce_t kernel[30]) @@ -487,7 +487,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV void + inline FF_CUDEV void diag_bending( scalar_t * out, offset_t osc, const offset_t loc[3], const offset_t size[3], @@ -544,7 +544,7 @@ struct RegFlow { /// absy, wy100, wy010, wy001, wy200, wy020, wy002, wy110, wy101, wy001, /// absz, wz100, wz010, wz001, wz200, wz020, wz002, wz110, wz101, wz001, /// ww] - inline CUDEV void + inline FF_CUDEV void make_kernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, @@ -606,7 +606,7 @@ struct RegFlow { /// wy000, wy100, wy010, wy001, wy200, wy020, wy002, wy110, wy101, wy001, /// wz000, wz100, wz010, wz001, wz200, wz020, wz002, wz110, wz101, wz001, /// ww] - inline CUDEV void make_fullkernel_all( + inline FF_CUDEV void make_fullkernel_all( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, reduce_t shears, reduce_t div, const reduce_t voxel_size[3]) @@ -667,7 +667,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_all( scalar_t * out, const scalar_t * inp, const offset_t loc[3], const offset_t size[3], @@ -849,7 +849,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_all( scalar_t * out, const offset_t sc[2], const offset_t stride[3], const reduce_t kernel[31]) @@ -910,7 +910,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_all( scalar_t * out, offset_t osc, const offset_t loc[3], const offset_t size[3], @@ -964,7 +964,7 @@ struct RegFlow { /// absy, wy100, wy010, wy001, /// absz, wz100, wz010, wz001, /// ww] - CUDEV inline void + FF_CUDEV inline void make_kernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[3]) @@ -1000,7 +1000,7 @@ struct RegFlow { /// wy000, wy100, wy010, wy001, /// wz000, wz100, wz010, wz001, /// ww] - CUDEV inline void + FF_CUDEV inline void make_fullkernel_lame( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[3]) @@ -1034,7 +1034,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_lame( scalar_t * out, const scalar_t * inp, const offset_t loc[3], const offset_t size[3], @@ -1150,7 +1150,7 @@ struct RegFlow { // --- kernel --- template - CUDEV inline void + FF_CUDEV inline void kernel_lame( scalar_t * out, const offset_t sc[2], const offset_t stride[3], const reduce_t kernel[13]) @@ -1191,7 +1191,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_lame( scalar_t * out, offset_t osc, const offset_t loc[3], const offset_t size[3], @@ -1228,7 +1228,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_absolute_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, offset_t osc, offset_t isc, const reduce_t kernel[3]) @@ -1242,7 +1242,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_absolute_jrls( scalar_t * out, const scalar_t * wgt, offset_t osc, const reduce_t kernel[3]) @@ -1259,7 +1259,7 @@ struct RegFlow { static const int kernelsize_membrane_jrls = kernelsize_membrane; - CUDEV inline void + FF_CUDEV inline void make_kernel_membrane_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, const reduce_t voxel_size[3]) @@ -1272,7 +1272,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_membrane_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -1355,7 +1355,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_membrane_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -1439,7 +1439,7 @@ struct RegFlow { * ly = 1/(vx[1]*vx[1]) * lz = 1/(vx[2]*vx[2]) */ - inline CUDEV void + inline FF_CUDEV void make_kernel_bending_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t bending, const reduce_t voxel_size[3]) @@ -1470,7 +1470,7 @@ struct RegFlow { // --- matvec --- template - CUDEV inline void + FF_CUDEV inline void matvec_bending_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -1692,7 +1692,7 @@ struct RegFlow { // --- diagonal --- template - CUDEV inline void + FF_CUDEV inline void diag_bending_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -1866,7 +1866,7 @@ struct RegFlow { * d2 = 0.25*div * s2 = 0.25*shears */ - CUDEV inline void + FF_CUDEV inline void make_kernel_lame_jrls( reduce_t * kernel, reduce_t absolute, reduce_t membrane, reduce_t shears, reduce_t div, const reduce_t voxel_size[3]) @@ -1883,7 +1883,7 @@ struct RegFlow { // --- matvec --- template - inline CUDEV + inline FF_CUDEV void matvec_lame_jrls( scalar_t * out, const scalar_t * inp, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -2043,7 +2043,7 @@ struct RegFlow { // --- diagonal --- template - inline CUDEV + inline FF_CUDEV void diag_lame_jrls( scalar_t * out, const scalar_t * wgt, const offset_t loc[3], const offset_t size[3], @@ -2099,6 +2099,6 @@ struct RegFlow { FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FLOW_3D diff --git a/include/fastfields/impl/kernels/regularisers/flow/utils.h b/include/fastfields/impl/kernels/regularisers/flow/utils.h index 8201cfb..7c4a6da 100755 --- a/include/fastfields/impl/kernels/regularisers/flow/utils.h +++ b/include/fastfields/impl/kernels/regularisers/flow/utils.h @@ -4,7 +4,7 @@ #include "../../bounds.h" #include "../../utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(reg_flow) @@ -23,34 +23,34 @@ struct RegFlow {}; //---------------------------------------------------------------------- template -inline CUDEV T & set(T & out, const IT & in) +inline FF_CUDEV T & set(T & out, const IT & in) { out = static_cast(in); return out; } template -inline CUDEV T & iadd(T & out, const IT & in) +inline FF_CUDEV T & iadd(T & out, const IT & in) { out = static_cast(static_cast(out) + in); return out; } template -inline CUDEV T & isub(T & out, const IT & in) +inline FF_CUDEV T & isub(T & out, const IT & in) { out = static_cast(static_cast(out) - in); return out; } template -inline CUDEV T add(const T & out, const IT & in) +inline FF_CUDEV T add(const T & out, const IT & in) { return static_cast(static_cast(out) + in); } template -inline CUDEV T sub(const T & out, const IT & in) +inline FF_CUDEV T sub(const T & out, const IT & in) { return static_cast(static_cast(out) - in); } @@ -78,7 +78,7 @@ struct Op<'-', scalar_t, reduce_t> { //---------------------------------------------------------------------- template -CUDEV inline +FF_CUDEV inline U center_offset(const U * size, const U * stride) { U offset = 0; @@ -89,7 +89,7 @@ U center_offset(const U * size, const U * stride) } template -CUDEV inline +FF_CUDEV inline bool patch1(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -100,7 +100,7 @@ bool patch1(const offset_t loc[N], offset_t n) } template -CUDEV inline +FF_CUDEV inline bool patch2(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -112,7 +112,7 @@ bool patch2(const offset_t loc[N], offset_t n) } template -CUDEV inline +FF_CUDEV inline bool patch3(const offset_t loc[N], offset_t n) { offset_t acc = 0; @@ -125,6 +125,6 @@ bool patch3(const offset_t loc[N], offset_t n) FF_NAMESPACE_END(reg_flow) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_REGULARISERS_FLOW_UTILS diff --git a/include/fastfields/impl/kernels/resize.h b/include/fastfields/impl/kernels/resize.h index cae9cde..3993de1 100755 --- a/include/fastfields/impl/kernels/resize.h +++ b/include/fastfields/impl/kernels/resize.h @@ -7,7 +7,7 @@ #include "batch.h" // index2sub -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(resize) @@ -37,7 +37,7 @@ template < struct Multiscale { template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -104,7 +104,7 @@ template struct Multiscale { using bound_utils = bound::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -129,7 +129,7 @@ template struct Multiscale { using bound_utils = bound::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -163,7 +163,7 @@ template struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -201,7 +201,7 @@ template struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -244,7 +244,7 @@ struct Multiscale { using spline_utils_x = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -299,7 +299,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -334,7 +334,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -387,7 +387,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -447,7 +447,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -518,7 +518,7 @@ struct Multiscale { using spline_utils_y = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -598,7 +598,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -639,7 +639,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize(scalar_t * out, const scalar_t * inp, const offset_t loc[3], const offset_t size[3], const offset_t stride[3], const reduce_t scl[3], @@ -705,7 +705,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -786,7 +786,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -885,7 +885,7 @@ struct Multiscale { using spline_utils_z = spline::utils; template - static inline CUDEV + static inline FF_CUDEV void resize( scalar_t out [], const scalar_t inp [], @@ -972,6 +972,6 @@ struct Multiscale { FF_NAMESPACE_END(resize) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_RESIZE diff --git a/include/fastfields/impl/kernels/restrict.h b/include/fastfields/impl/kernels/restrict.h index bfdff4e..9e0f5a6 100755 --- a/include/fastfields/impl/kernels/restrict.h +++ b/include/fastfields/impl/kernels/restrict.h @@ -5,7 +5,7 @@ #include "bounds.h" #include "batch.h" // index2sub -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(restrict) @@ -33,7 +33,7 @@ template < struct Multiscale { template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -95,7 +95,7 @@ struct Multiscale static const int32_t spline_order = static_cast(I); template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -134,7 +134,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -186,7 +186,7 @@ struct Multiscale { static const int32_t spline_order_y = static_cast(IY); template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -239,7 +239,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -312,7 +312,7 @@ struct Multiscale { static const int32_t spline_order_z = static_cast(IZ); template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -375,7 +375,7 @@ struct Multiscale { using spline_utils = spline::utils; template - static CUDEV + static FF_CUDEV void restrict( scalar_t out [], const scalar_t inp [], @@ -451,6 +451,6 @@ struct Multiscale { FF_NAMESPACE_END(restrict) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_RESTRICT diff --git a/include/fastfields/impl/kernels/splinc.h b/include/fastfields/impl/kernels/splinc.h index 43e4b1d..2a93158 100755 --- a/include/fastfields/impl/kernels/splinc.h +++ b/include/fastfields/impl/kernels/splinc.h @@ -31,13 +31,13 @@ #include "bounds.h" #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(splinc) FF_NAMESPACE_BEGIN(_splinc) template -inline CUDEV +inline FF_CUDEV scalar_t dft_initial(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -56,7 +56,7 @@ scalar_t dft_initial(const scalar_t * inp, reduce_t pole, } template -inline CUDEV +inline FF_CUDEV scalar_t dft_final(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -74,7 +74,7 @@ scalar_t dft_final(const scalar_t * inp, reduce_t pole, } template -inline CUDEV +inline FF_CUDEV scalar_t dct1_initial(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -115,7 +115,7 @@ scalar_t dct1_initial(const scalar_t * inp, reduce_t pole, template -inline CUDEV +inline FF_CUDEV scalar_t dct1_final(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -129,7 +129,7 @@ scalar_t dct1_final(const scalar_t * inp, reduce_t pole, template -inline CUDEV +inline FF_CUDEV scalar_t dct1_scipy_initial(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -159,7 +159,7 @@ scalar_t dct1_scipy_initial(const scalar_t * inp, reduce_t pole, template -inline CUDEV +inline FF_CUDEV reduce_t dct2_initial(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -191,7 +191,7 @@ reduce_t dct2_initial(const scalar_t * inp, reduce_t pole, } template -inline CUDEV +inline FF_CUDEV scalar_t dct2_final(const scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { @@ -203,7 +203,7 @@ scalar_t dct2_final(const scalar_t * inp, reduce_t pole, FF_NAMESPACE_END(_splinc) template -inline CUDEV int get_poles(int order, scalar_t * poles) +inline FF_CUDEV int get_poles(int order, scalar_t * poles) { switch (order) { case 0: @@ -238,7 +238,7 @@ inline CUDEV int get_poles(int order, scalar_t * poles) } template -inline CUDEV scalar_t get_gain(scalar_t * poles, int npoles) +inline FF_CUDEV scalar_t get_gain(scalar_t * poles, int npoles) { double lam = 1.; for (int i=0; i struct utils { // ZERO & DCT1 template - static inline CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct1_scipy_initial(inp, pole, size, stride); } template - static inline CUDEV scalar_t final(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t final(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct1_final(inp, pole, size, stride); } }; template <> struct utils { template - static inline CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct2_initial(inp, pole, size, stride); } template - static inline CUDEV scalar_t final(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t final(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct2_final(inp, pole, size, stride); } }; template <> struct utils { template - static inline CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct2_initial(inp, pole, size, stride); } template - static inline CUDEV scalar_t final(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t final(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dct2_final(inp, pole, size, stride); } }; template <> struct utils { template - static inline CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t initial(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dft_initial(inp, pole, size, stride); } template - static inline CUDEV scalar_t final(scalar_t * inp, reduce_t pole, + static inline FF_CUDEV scalar_t final(scalar_t * inp, reduce_t pole, offset_t size, offset_t stride) { return _splinc::dft_final(inp, pole, size, stride); } }; template -inline CUDEV void filter(scalar_t * inp, offset_t size, offset_t stride, +inline FF_CUDEV void filter(scalar_t * inp, offset_t size, offset_t stride, const reduce_t * poles) { using bound_utils = utils; @@ -334,6 +334,6 @@ inline CUDEV void filter(scalar_t * inp, offset_t size, offset_t stride, FF_NAMESPACE_END(splinc) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_SPLINC diff --git a/include/fastfields/impl/kernels/spline.h b/include/fastfields/impl/kernels/spline.h index ec95f17..81f9d4e 100755 --- a/include/fastfields/impl/kernels/spline.h +++ b/include/fastfields/impl/kernels/spline.h @@ -28,7 +28,7 @@ #include "fastfields/core/cuda_switch.h" #include "meta.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(spline) enum class type : int8_t { @@ -63,11 +63,11 @@ struct SplineVecN { static const int max_ndim = MaxNDim; int8_t s[max_ndim]; - inline CUHOSTDEV SplineVecN() + inline FF_CUHOSTDEV SplineVecN() { for (int d = 0; d < max_ndim; ++d) s[d] = static_cast(type::Linear); } // Isotropic: the same order on every axis (what the public ABI exposes). - explicit inline CUHOSTDEV SplineVecN(type v) + explicit inline FF_CUHOSTDEV SplineVecN(type v) { for (int d = 0; d < max_ndim; ++d) s[d] = static_cast(v); } // Anisotropic: one order per axis, `ndim <= max_ndim` of them meaningful. @@ -84,13 +84,13 @@ struct SplineVecN { // one -- making any such latent bug cheap rather than silently expensive. // Matches `bound::BoundVecN`'s analogous choice of `type::Zero` (that // enum's own semantic default) for the same never-read padding purpose. - inline CUHOSTDEV SplineVecN(const type * v, int ndim) + inline FF_CUHOSTDEV SplineVecN(const type * v, int ndim) { for (int d = 0; d < max_ndim; ++d) s[d] = static_cast(d < ndim ? v[d] : type::Linear); } - inline CUHOSTDEV type operator[] (int d) const + inline FF_CUHOSTDEV type operator[] (int d) const { return static_cast(s[d]); } }; @@ -161,8 +161,8 @@ template using Spline = meta::Tuple; # define FF_STATIC_SPLINE_SEVENTHORDER FF_STATIC_SPLINES #endif -#define FF_SPLINE_IF_1(NAME) ::FF::spline::type::NAME -#define FF_SPLINE_IF_0(NAME) ::FF::spline::type::Dynamic +#define FF_SPLINE_IF_1(NAME) ::FF_NS::spline::type::NAME +#define FF_SPLINE_IF_0(NAME) ::FF_NS::spline::type::Dynamic #define FF_SPLINE_CAT_(A, B) A##B #define FF_SPLINE_CAT(A, B) FF_SPLINE_CAT_(A, B) #define FF_SPLINE_SEL(FLAG, NAME) FF_SPLINE_CAT(FF_SPLINE_IF_, FLAG)(NAME) @@ -181,55 +181,55 @@ template using Spline = meta::Tuple; FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(spline) -using FF::spline::type; -using FF::spline::SplineVec; +using FF_NS::spline::type; +using FF_NS::spline::SplineVec; FF_NAMESPACE_BEGIN(_spline) // Forward declarations for the few basis functions that are referenced // by a lower/earlier-defined function (two-phase name lookup would // otherwise fail on these unqualified dependent calls). - template static inline CUDEV scalar_t fastgrad1(scalar_t x); - template static inline CUDEV scalar_t fasthess5(scalar_t x); - template static inline CUDEV scalar_t fasthess6(scalar_t x); - template static inline CUDEV scalar_t fasthess7(scalar_t x); + template static inline FF_CUDEV scalar_t fastgrad1(scalar_t x); + template static inline FF_CUDEV scalar_t fasthess5(scalar_t x); + template static inline FF_CUDEV scalar_t fasthess6(scalar_t x); + template static inline FF_CUDEV scalar_t fasthess7(scalar_t x); // --- order 0 ------------------------------------------------------- template - static inline CUDEV scalar_t weight0(scalar_t x) { + static inline FF_CUDEV scalar_t weight0(scalar_t x) { x = fabs(x); return x < 0.5 ? static_cast(1) : static_cast(0); } template - static inline CUDEV scalar_t fastweight0(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight0(scalar_t x) { x = fabs(x); return static_cast(1); } template - static inline CUDEV scalar_t grad0(scalar_t x) { + static inline FF_CUDEV scalar_t grad0(scalar_t x) { return static_cast(0); } template - static inline CUDEV scalar_t fastgrad0(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad0(scalar_t x) { return static_cast(0); } template - static inline CUDEV scalar_t hess0(scalar_t x) { + static inline FF_CUDEV scalar_t hess0(scalar_t x) { return static_cast(0); } template - static inline CUDEV scalar_t fasthess0(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess0(scalar_t x) { return static_cast(0); } template - static inline CUDEV void bounds0(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds0(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(round(x)); upp = low; } @@ -237,39 +237,39 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 1 ------------------------------------------------------- template - static inline CUDEV scalar_t weight1(scalar_t x) { + static inline FF_CUDEV scalar_t weight1(scalar_t x) { x = fabs(x); return x < 1 ? static_cast(1) - x : static_cast(0); } template - static inline CUDEV scalar_t fastweight1(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight1(scalar_t x) { return static_cast(1) - x; } template - static inline CUDEV scalar_t grad1(scalar_t x) { + static inline FF_CUDEV scalar_t grad1(scalar_t x) { if (fabs(x) >= 1) return static_cast(0); return fastgrad1(x); } template - static inline CUDEV scalar_t fastgrad1(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad1(scalar_t x) { return static_cast(-1); } template - static inline CUDEV scalar_t hess1(scalar_t x) { + static inline FF_CUDEV scalar_t hess1(scalar_t x) { return static_cast(0); } template - static inline CUDEV scalar_t fasthess1(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess1(scalar_t x) { return static_cast(0); } template - static inline CUDEV void bounds1(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds1(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x)); upp = low + 1; } @@ -277,7 +277,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 2 ------------------------------------------------------- template - static inline CUDEV scalar_t weight2(scalar_t x) { + static inline FF_CUDEV scalar_t weight2(scalar_t x) { x = fabs(x); if ( x < 0.5 ) { @@ -295,7 +295,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight2(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight2(scalar_t x) { if ( x < 0.5 ) { return 0.75 - x * x; @@ -308,7 +308,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad2(scalar_t x) { + static inline FF_CUDEV scalar_t grad2(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < 0.5 ) @@ -328,7 +328,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad2(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad2(scalar_t x) { if ( x < 0.5 ) { x = -2. * x; @@ -341,7 +341,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t hess2(scalar_t x) { + static inline FF_CUDEV scalar_t hess2(scalar_t x) { x = fabs(x); if ( x < 0.5 ) { @@ -358,7 +358,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess2(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess2(scalar_t x) { if ( x < 0.5 ) { return static_cast(-2.); @@ -370,7 +370,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV void bounds2(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds2(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-.5)); upp = low + 2; } @@ -378,7 +378,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 3 ------------------------------------------------------- template - static inline CUDEV scalar_t weight3(scalar_t x) { + static inline FF_CUDEV scalar_t weight3(scalar_t x) { x = fabs(x); if ( x < 1. ) { @@ -396,7 +396,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight3(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight3(scalar_t x) { if ( x < 1. ) { return ( x * x * (x - 2.) * 3. + 4. ) / 6.; @@ -409,7 +409,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad3(scalar_t x) { + static inline FF_CUDEV scalar_t grad3(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < 1. ) @@ -430,7 +430,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad3(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad3(scalar_t x) { if ( x < 1. ) { x = x * ( x * 1.5 - 2. ); @@ -444,7 +444,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t hess3(scalar_t x) { + static inline FF_CUDEV scalar_t hess3(scalar_t x) { x = fabs(x); if ( x < 1. ) { @@ -461,7 +461,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess3(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess3(scalar_t x) { if ( x < 1. ) { return x * 3. - 2.; @@ -474,7 +474,7 @@ FF_NAMESPACE_BEGIN(_spline) template - static inline CUDEV void bounds3(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds3(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-1.)); upp = low + 3; } @@ -482,7 +482,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 4 ------------------------------------------------------- template - static inline CUDEV scalar_t weight4(scalar_t x) { + static inline FF_CUDEV scalar_t weight4(scalar_t x) { x = fabs(x); if ( x < 0.5 ) { @@ -506,7 +506,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight4(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight4(scalar_t x) { if ( x < 0.5 ) { x *= x; @@ -525,7 +525,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad4(scalar_t x) { + static inline FF_CUDEV scalar_t grad4(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < 0.5 ) @@ -550,7 +550,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad4(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad4(scalar_t x) { if ( x < 0.5 ) { x = x * ( x * x - 1.25 ); @@ -568,7 +568,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t hess4(scalar_t x) { + static inline FF_CUDEV scalar_t hess4(scalar_t x) { x = fabs(x); if ( x < 0.5 ) { @@ -590,7 +590,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess4(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess4(scalar_t x) { if ( x < 0.5 ) { return ( x * x ) * 3. - 1.25; @@ -607,7 +607,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV void bounds4(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds4(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-1.5)); upp = low + 4; } @@ -615,7 +615,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 5 ------------------------------------------------------- template - static inline CUDEV scalar_t weight5(scalar_t x) { + static inline FF_CUDEV scalar_t weight5(scalar_t x) { x = fabs(x); if ( x < 1. ) { @@ -638,7 +638,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight5(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight5(scalar_t x) { if ( x < 1. ) { scalar_t f = x * x; @@ -658,7 +658,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad5(scalar_t x) { + static inline FF_CUDEV scalar_t grad5(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < 1. ) @@ -684,7 +684,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad5(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad5(scalar_t x) { if ( x < 1. ) { x = x * ( x * ( x * ( x * ( -5. / 12. ) + 1. ) ) - 1. ); @@ -703,7 +703,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t hess5(scalar_t x) { + static inline FF_CUDEV scalar_t hess5(scalar_t x) { x = fabs(x); if ( x >= 3. ) return static_cast(0); @@ -712,7 +712,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess5(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess5(scalar_t x) { if ( x < 1. ) return - (x * x) * (x * (5./3.) - 3.) - 1.; else if ( x < 2. ) @@ -722,7 +722,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV void bounds5(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds5(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-2.)); upp = low + 5; } @@ -730,7 +730,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 6 ------------------------------------------------------- template - static inline CUDEV scalar_t weight6(scalar_t x) { + static inline FF_CUDEV scalar_t weight6(scalar_t x) { x = fabs(x); if ( x < 0.5 ) { @@ -761,7 +761,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight6(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight6(scalar_t x) { if ( x < 0.5 ) { x *= x; @@ -789,7 +789,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad6(scalar_t x) { + static inline FF_CUDEV scalar_t grad6(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < .5 ) @@ -823,7 +823,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad6(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad6(scalar_t x) { if ( x < .5 ) { scalar_t x2 = x * x; @@ -851,7 +851,7 @@ FF_NAMESPACE_BEGIN(_spline) template - static inline CUDEV scalar_t hess6(scalar_t x) { + static inline FF_CUDEV scalar_t hess6(scalar_t x) { x = fabs(x); if ( x >= 3.5 ) return static_cast(0); @@ -860,7 +860,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess6(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess6(scalar_t x) { if ( x < 0.5 ) { x *= x; return - x * (x * (5./6) - 7./4.) - 77./96.; @@ -874,7 +874,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV void bounds6(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds6(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-2.5)); upp = low + 6; } @@ -882,7 +882,7 @@ FF_NAMESPACE_BEGIN(_spline) // --- order 7 ------------------------------------------------------- template - static inline CUDEV scalar_t weight7(scalar_t x) { + static inline FF_CUDEV scalar_t weight7(scalar_t x) { x = fabs(x); if ( x < 1. ) { @@ -913,7 +913,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastweight7(scalar_t x) { + static inline FF_CUDEV scalar_t fastweight7(scalar_t x) { if ( x < 1. ) { scalar_t f = x * x; @@ -941,7 +941,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t grad7(scalar_t x) { + static inline FF_CUDEV scalar_t grad7(scalar_t x) { bool neg = x < 0; if (neg) x = -x; if ( x < 1. ) @@ -976,7 +976,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fastgrad7(scalar_t x) { + static inline FF_CUDEV scalar_t fastgrad7(scalar_t x) { if ( x < 1. ) { scalar_t x2 = x * x; @@ -1004,7 +1004,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t hess7(scalar_t x) { + static inline FF_CUDEV scalar_t hess7(scalar_t x) { x = fabs(x); if ( x >= 4. ) return static_cast(0); @@ -1013,7 +1013,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV scalar_t fasthess7(scalar_t x) { + static inline FF_CUDEV scalar_t fasthess7(scalar_t x) { if ( x < 1. ) { scalar_t x2 = x * x; return x2 * (x2 * (x * (7./24.) - 5./6.) + 4./3.) - 2./3.; @@ -1026,7 +1026,7 @@ FF_NAMESPACE_BEGIN(_spline) } template - static inline CUDEV void bounds7(scalar_t x, offset_t & low, offset_t & upp) { + static inline FF_CUDEV void bounds7(scalar_t x, offset_t & low, offset_t & upp) { low = static_cast(floor(x-3.)); upp = low + 7; } @@ -1041,7 +1041,7 @@ template using _value_fn_t = typename _value_fn::type; template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t weight_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::weight0; @@ -1057,7 +1057,7 @@ weight_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t weight(type spline_type, scalar_t x) { return weight_fn(spline_type)(x); // switch (spline_type) { @@ -1074,7 +1074,7 @@ weight(type spline_type, scalar_t x) { } template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t fastweight_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::fastweight0; @@ -1090,7 +1090,7 @@ fastweight_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t fastweight(type spline_type, scalar_t x) { return fastweight_fn(spline_type)(x); // switch (spline_type) { @@ -1107,7 +1107,7 @@ fastweight(type spline_type, scalar_t x) { } template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t grad_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::grad0; @@ -1123,7 +1123,7 @@ grad_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t grad(type spline_type, scalar_t x) { return grad_fn(spline_type)(x); // switch (spline_type) { @@ -1140,7 +1140,7 @@ grad(type spline_type, scalar_t x) { } template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t fastgrad_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::fastgrad0; @@ -1156,7 +1156,7 @@ fastgrad_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t fastgrad(type spline_type, scalar_t x) { return fastgrad_fn(spline_type)(x); // switch (spline_type) { @@ -1173,7 +1173,7 @@ fastgrad(type spline_type, scalar_t x) { } template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t hess_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::hess0; @@ -1189,7 +1189,7 @@ hess_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t hess(type spline_type, scalar_t x) { return hess_fn(spline_type)(x); // switch (spline_type) { @@ -1206,7 +1206,7 @@ hess(type spline_type, scalar_t x) { } template -static inline CUDEV _value_fn_t +static inline FF_CUDEV _value_fn_t fasthess_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::fasthess0; @@ -1222,7 +1222,7 @@ fasthess_fn(type spline_type) { } template -static inline CUDEV scalar_t +static inline FF_CUDEV scalar_t fasthess(type spline_type, scalar_t x) { return fasthess_fn(spline_type)(x); // switch (spline_type) { @@ -1245,7 +1245,7 @@ template using _bounds_fn_t = typename _bounds_fn::type; template -static inline CUDEV _bounds_fn_t +static inline FF_CUDEV _bounds_fn_t bounds_fn(type spline_type) { switch (spline_type) { case type::Nearest: return _spline::bounds0; @@ -1261,7 +1261,7 @@ bounds_fn(type spline_type) { } template -static inline CUDEV void +static inline FF_CUDEV void bounds(type spline_type, scalar_t x, offset_t & low, offset_t & upp) { return bounds_fn(spline_type)(x, low, upp); @@ -1281,39 +1281,39 @@ bounds(type spline_type, scalar_t x, offset_t & low, offset_t & upp) template struct utils {}; -#define INTERPOL_UTILS(NAME, ORDER) \ +#define FF_INTERPOL_UTILS(NAME, ORDER) \ template <> struct utils { \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ weight(scalar_t x) { return _spline::weight##ORDER(x); } \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ fastweight(scalar_t x) { return _spline::fastweight##ORDER(x); } \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ grad(scalar_t x) { return _spline::grad##ORDER(x); } \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ fastgrad(scalar_t x) { return _spline::fastgrad##ORDER(x); } \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ hess(scalar_t x) { return _spline::hess##ORDER(x); } \ template \ - static inline CUDEV scalar_t \ + static inline FF_CUDEV scalar_t \ fasthess(scalar_t x) { return _spline::fasthess##ORDER(x); } \ template \ - static inline CUDEV void \ + static inline FF_CUDEV void \ bounds(scalar_t x, offset_t & low, offset_t & upp) { return _spline::bounds##ORDER(x, low, upp); } \ }; -INTERPOL_UTILS(Nearest, 0) -INTERPOL_UTILS(Linear, 1) -INTERPOL_UTILS(Quadratic, 2) -INTERPOL_UTILS(Cubic, 3) -INTERPOL_UTILS(FourthOrder, 4) -INTERPOL_UTILS(FifthOrder, 5) -INTERPOL_UTILS(SixthOrder, 6) -INTERPOL_UTILS(SeventhOrder, 7) +FF_INTERPOL_UTILS(Nearest, 0) +FF_INTERPOL_UTILS(Linear, 1) +FF_INTERPOL_UTILS(Quadratic, 2) +FF_INTERPOL_UTILS(Cubic, 3) +FF_INTERPOL_UTILS(FourthOrder, 4) +FF_INTERPOL_UTILS(FifthOrder, 5) +FF_INTERPOL_UTILS(SixthOrder, 6) +FF_INTERPOL_UTILS(SeventhOrder, 7) // NOTE: there is deliberately no `utils` specialisation. // There used to be one whose methods all returned 0 -- a placeholder that made @@ -1342,52 +1342,52 @@ INTERPOL_UTILS(SeventhOrder, 7) template struct dyn { - inline CUDEV dyn() {} - explicit inline CUDEV dyn(type) {} // runtime value: not needed, ignored + inline FF_CUDEV dyn() {} + explicit inline FF_CUDEV dyn(type) {} // runtime value: not needed, ignored - inline CUDEV type value() const { return S; } + inline FF_CUDEV type value() const { return S; } template - inline CUDEV scalar_t weight(scalar_t x) const + inline FF_CUDEV scalar_t weight(scalar_t x) const { return utils::template weight(x); } template - inline CUDEV scalar_t fastweight(scalar_t x) const + inline FF_CUDEV scalar_t fastweight(scalar_t x) const { return utils::template fastweight(x); } template - inline CUDEV scalar_t grad(scalar_t x) const + inline FF_CUDEV scalar_t grad(scalar_t x) const { return utils::template grad(x); } template - inline CUDEV scalar_t fastgrad(scalar_t x) const + inline FF_CUDEV scalar_t fastgrad(scalar_t x) const { return utils::template fastgrad(x); } template - inline CUDEV scalar_t hess(scalar_t x) const + inline FF_CUDEV scalar_t hess(scalar_t x) const { return utils::template hess(x); } template - inline CUDEV scalar_t fasthess(scalar_t x) const + inline FF_CUDEV scalar_t fasthess(scalar_t x) const { return utils::template fasthess(x); } template - inline CUDEV void bounds(scalar_t x, offset_t & low, offset_t & upp) const + inline FF_CUDEV void bounds(scalar_t x, offset_t & low, offset_t & upp) const { return utils::template bounds(x, low, upp); } // Number of nodes in the support: `bounds` always yields exactly this many. // Static here, so the surrounding loops keep their compile-time trip count. - inline CUDEV int nodes() const { return static_cast(S) + 1; } + inline FF_CUDEV int nodes() const { return static_cast(S) + 1; } }; template <> struct dyn { type spl; - inline CUDEV dyn() : spl(type::Linear) {} - explicit inline CUDEV dyn(type s) : spl(s) {} + inline FF_CUDEV dyn() : spl(type::Linear) {} + explicit inline FF_CUDEV dyn(type s) : spl(s) {} - inline CUDEV type value() const { return spl; } + inline FF_CUDEV type value() const { return spl; } // Direct switches, *not* the `weight_fn` / `bounds_fn` function-pointer // helpers above: an indirect call cannot be inlined and is expensive on the @@ -1395,7 +1395,7 @@ template <> struct dyn // the same trade-off `bound::dyn` makes. #define FF_SPLINE_DYN_FWD(NAME) \ template \ - inline CUDEV scalar_t NAME(scalar_t x) const \ + inline FF_CUDEV scalar_t NAME(scalar_t x) const \ { \ switch (spl) { \ case type::Nearest: return _spline::NAME##0(x); \ @@ -1418,7 +1418,7 @@ template <> struct dyn #undef FF_SPLINE_DYN_FWD template - inline CUDEV void bounds(scalar_t x, offset_t & low, offset_t & upp) const + inline FF_CUDEV void bounds(scalar_t x, offset_t & low, offset_t & upp) const { switch (spl) { case type::Nearest: return _spline::bounds0(x, low, upp); @@ -1435,13 +1435,13 @@ template <> struct dyn // Runtime support size. Callers must use this (not a compile-time bound) to // size their loops; the buffers themselves are sized by `SplineBufSize`, // which reserves the worst case (8) for Dynamic. - inline CUDEV int nodes() const + inline FF_CUDEV int nodes() const { return static_cast(static_cast(spl)) + 1; } }; FF_NAMESPACE_END(spline) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_SPLINE diff --git a/include/fastfields/impl/kernels/tetrahedron.h b/include/fastfields/impl/kernels/tetrahedron.h index 07d8399..553494b 100755 --- a/include/fastfields/impl/kernels/tetrahedron.h +++ b/include/fastfields/impl/kernels/tetrahedron.h @@ -4,13 +4,13 @@ #include "utils.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) FF_NAMESPACE_BEGIN(tetra) // sort vertices by increasing z value template -CUDEV +FF_CUDEV void sort4(scalar_t & v0x, scalar_t & v0y, scalar_t & v0z, scalar_t * f0, scalar_t & v1x, scalar_t & v1y, scalar_t & v1z, scalar_t * f1, scalar_t & v2x, scalar_t & v2y, scalar_t & v2z, scalar_t * f2, @@ -43,7 +43,7 @@ void sort4(scalar_t & v0x, scalar_t & v0y, scalar_t & v0z, scalar_t * f0, } template -CUDEV +FF_CUDEV void sort3(scalar_t & v0x, scalar_t & v0y, scalar_t & v1x, scalar_t & v1y, scalar_t & v2x, scalar_t & v2y) @@ -63,7 +63,7 @@ void sort3(scalar_t & v0x, scalar_t & v0y, } template -CUDEV +FF_CUDEV scalar_t barycoord1(center_t px, center_t py, center_t pz, scalar_t v0x, scalar_t v0y, scalar_t v0z, scalar_t v1x, scalar_t v1y, scalar_t v1z, @@ -101,7 +101,7 @@ scalar_t barycoord1(center_t px, center_t py, center_t pz, } template -CUDEV +FF_CUDEV void barycoord(scalar_t & l0, scalar_t & l1, scalar_t & l2, scalar_t & l3, center_t px, center_t py, center_t pz, scalar_t v0x, scalar_t v0y, scalar_t v0z, @@ -120,7 +120,7 @@ void barycoord(scalar_t & l0, scalar_t & l1, scalar_t & l2, scalar_t & l3, } template -CUDEV +FF_CUDEV void pull1(scalar_t * output, scalar_t v0x, scalar_t v0y, scalar_t v0z, scalar_t v1x, scalar_t v1y, scalar_t v1z, @@ -237,7 +237,7 @@ void pull1(scalar_t * output, FF_NAMESPACE_END(tetra) FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_TETRAHEDRON diff --git a/include/fastfields/impl/kernels/threadpool.h b/include/fastfields/impl/kernels/threadpool.h index 4b59419..db07c3b 100755 --- a/include/fastfields/impl/kernels/threadpool.h +++ b/include/fastfields/impl/kernels/threadpool.h @@ -15,7 +15,7 @@ #include #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) class ThreadPool; std::shared_ptr get_global_pool(); @@ -227,7 +227,7 @@ class ThreadPool } }; -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #include "threadpool.inl" diff --git a/include/fastfields/impl/kernels/threadpool.inl b/include/fastfields/impl/kernels/threadpool.inl index f69b9df..a782b06 100755 --- a/include/fastfields/impl/kernels/threadpool.inl +++ b/include/fastfields/impl/kernels/threadpool.inl @@ -5,7 +5,7 @@ #include #include "fastfields/core/defines.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(internal) // Some of this is copied from pytorch/aten @@ -74,5 +74,5 @@ inline std::shared_ptr get_global_pool() { return internal::global_pool(); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_THREADPOOL_INL diff --git a/include/fastfields/impl/kernels/utils.h b/include/fastfields/impl/kernels/utils.h index 952160d..0705507 100755 --- a/include/fastfields/impl/kernels/utils.h +++ b/include/fastfields/impl/kernels/utils.h @@ -13,7 +13,7 @@ # include // std::numeric_limits #endif // __CUDA_ARCH__ -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // static check for floating types @@ -30,14 +30,14 @@ struct is_floating_point { static constexpr bool value = true; }; template -inline CUDEV +inline FF_CUDEV void swap(T& a, T& b) { T c(a); a=b; b=c; } template -inline CUDEV +inline FF_CUDEV T square(T a) { return a*a; @@ -46,26 +46,26 @@ T square(T a) #ifdef __CUDACC__ template -inline CUDEV +inline FF_CUDEV T sqrt(T a) {} template <> -inline CUDEV +inline FF_CUDEV float sqrt(float a) { return ::sqrtf(a); } template <> -inline CUDEV +inline FF_CUDEV double sqrt(double a) { return ::sqrt(a); } template <> -inline CUDEV +inline FF_CUDEV half sqrt(half a) { // hsqrt is not visible at global scope in every CUDA/arch combination; @@ -76,7 +76,7 @@ half sqrt(half a) #else template -inline CUDEV +inline FF_CUDEV T sqrt(T a) { return std::sqrt(a); @@ -86,7 +86,7 @@ T sqrt(T a) template -inline CUDEV +inline FF_CUDEV T pow(T a) { T p = a; # pragma unroll @@ -96,7 +96,7 @@ T pow(T a) { } template -inline CUDEV +inline FF_CUDEV T pow(T a, int N) { T p = a; # pragma unroll @@ -106,28 +106,28 @@ T pow(T a, int N) { } template -inline CUDEV +inline FF_CUDEV T min(T a, T b) { return (a < b ? a : b); } template -inline CUDEV +inline FF_CUDEV T max(T a, T b) { return (a > b ? a : b); } template -inline CUDEV +inline FF_CUDEV T abs(T a) { return static_cast(a < 0 ? -a : a); } template -inline CUDEV +inline FF_CUDEV signed char sign(T a) { return static_cast(a == 0 ? 0 : a < 0 ? -1 : 1); @@ -135,7 +135,7 @@ signed char sign(T a) #ifdef __CUDACC__ template <> -inline CUDEV +inline FF_CUDEV half min<>(half a, half b) { // Compare via float: half has multiple implicit conversions to built-in @@ -145,7 +145,7 @@ half min<>(half a, half b) return (af < bf ? a : b); } template <> -inline CUDEV +inline FF_CUDEV half max<>(half a, half b) { float af = static_cast(a); @@ -160,7 +160,7 @@ template ::value > struct _mod { - inline CUDEV static + inline FF_CUDEV static T f(T x, U d) { signed char sx = sign(x); @@ -175,7 +175,7 @@ struct _mod template struct _mod { - inline CUDEV static + inline FF_CUDEV static T f(T x, U d) { return x % d; @@ -183,14 +183,14 @@ struct _mod }; template -inline CUDEV +inline FF_CUDEV T mod(T x, U d) { return _mod::f(x, d); } template -inline CUDEV +inline FF_CUDEV OT typed_prod(const IT * x, size_t size) { if (size == 0) @@ -202,7 +202,7 @@ OT typed_prod(const IT * x, size_t size) } template -inline CUDEV +inline FF_CUDEV OT typed_prod(const IT * x) { if (size == 0) @@ -215,21 +215,21 @@ OT typed_prod(const IT * x) } template -inline CUDEV +inline FF_CUDEV T prod(const T * x, size_t size) { return typed_prod(x, size); } template -inline CUDEV +inline FF_CUDEV T prod(const T * x) { return typed_prod(x); } template -inline CUDEV +inline FF_CUDEV void fillfrom(U out[N], const V * inp) { # pragma unroll @@ -238,7 +238,7 @@ void fillfrom(U out[N], const V * inp) } template -inline CUDEV +inline FF_CUDEV void fillfrom(U out[N], const V * inp, W stride) { # pragma unroll @@ -247,7 +247,7 @@ void fillfrom(U out[N], const V * inp, W stride) } template -inline CUDEV +inline FF_CUDEV void fillfrom(int N, U out[], const V * inp) { for (int n=0; n < N; ++ n) @@ -255,7 +255,7 @@ void fillfrom(int N, U out[], const V * inp) } template -inline CUDEV +inline FF_CUDEV void fillfrom(int N, U out[], const V * inp, W stride) { for (int n=0; n < N; ++n, inp += stride) @@ -263,7 +263,7 @@ void fillfrom(int N, U out[], const V * inp, W stride) } template -inline CUDEV +inline FF_CUDEV void fill(U * out, V inp) { auto val = static_cast(inp); @@ -273,7 +273,7 @@ void fill(U * out, V inp) } template -inline CUDEV +inline FF_CUDEV void fill(U * out, V inp, W stride) { auto val = static_cast(inp); @@ -287,34 +287,34 @@ void fill(U * out, V inp, W stride) template struct StaticValue { - CUHOSTDEV constexpr StaticValue(const T & value = Value) {} - CUHOSTDEV constexpr StaticValue(const StaticValue & value) {} - CUHOSTDEV constexpr operator T() const { return Value; } + FF_CUHOSTDEV constexpr StaticValue(const T & value = Value) {} + FF_CUHOSTDEV constexpr StaticValue(const StaticValue & value) {} + FF_CUHOSTDEV constexpr operator T() const { return Value; } }; template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator+(const StaticValue & v) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator-(const StaticValue & v) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator!(const StaticValue & v) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator~(const StaticValue & v) { return StaticValue(); @@ -323,112 +323,112 @@ operator~(const StaticValue & v) // static vs static template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator+(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator-(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator*(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator/(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator%(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator&(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator|(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator^(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue>W),(V>>W)> +FF_CUHOSTDEV constexpr StaticValue>W),(V>>W)> operator>>(const StaticValue & v, const StaticValue & w) { return StaticValue>W),(V>>W)>(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator<<(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator==(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator!=(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator<=(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValue=W> +FF_CUHOSTDEV constexpr StaticValue=W> operator>=(const StaticValue & v, const StaticValue & w) { return StaticValue=W>(); } template -CUHOSTDEV constexpr StaticValue +FF_CUHOSTDEV constexpr StaticValue operator<(const StaticValue & v, const StaticValue & w) { return StaticValue(); } template -CUHOSTDEV constexpr StaticValueW)> +FF_CUHOSTDEV constexpr StaticValueW)> operator>(const StaticValue & v, const StaticValue & w) { return StaticValueW)>(); @@ -437,112 +437,112 @@ operator>(const StaticValue & v, const StaticValue & w) // static vs dynamic template -CUHOSTDEV inline decltype(V+U()) +FF_CUHOSTDEV inline decltype(V+U()) operator+(const StaticValue & v, const U & w) { return v + w; } template -CUHOSTDEV inline decltype(V-U()) +FF_CUHOSTDEV inline decltype(V-U()) operator-(const StaticValue & v, const U & w) { return v - w; } template -CUHOSTDEV inline decltype(V*U()) +FF_CUHOSTDEV inline decltype(V*U()) operator*(const StaticValue & v, const U & w) { return v * w; } template -CUHOSTDEV inline decltype(V/U()) +FF_CUHOSTDEV inline decltype(V/U()) operator/(const StaticValue & v, const U & w) { return v / w; } template -CUHOSTDEV inline decltype(V%U()) +FF_CUHOSTDEV inline decltype(V%U()) operator%(const StaticValue & v, const U & w) { return v % w; } template -CUHOSTDEV inline decltype(V&U()) +FF_CUHOSTDEV inline decltype(V&U()) operator&(const StaticValue & v, const U & w) { return v & w; } template -CUHOSTDEV inline decltype(V|U()) +FF_CUHOSTDEV inline decltype(V|U()) operator|(const StaticValue & v, const U & w) { return v | w; } template -CUHOSTDEV inline decltype(V^U()) +FF_CUHOSTDEV inline decltype(V^U()) operator^(const StaticValue & v, const U & w) { return v ^ w; } template -CUHOSTDEV inline decltype(V>>U()) +FF_CUHOSTDEV inline decltype(V>>U()) operator>>(const StaticValue & v, const U & w) { return v >> w; } template -CUHOSTDEV inline decltype(V< & v, const U & w) { return v << w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator==(const StaticValue & v, const U & w) { return v == w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator!=(const StaticValue & v, const U & w) { return v != w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator>=(const StaticValue & v, const U & w) { return v >= w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator<=(const StaticValue & v, const U & w) { return v <= w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator>(const StaticValue & v, const U & w) { return v > w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator<(const StaticValue & v, const U & w) { return v < w; @@ -551,112 +551,112 @@ operator<(const StaticValue & v, const U & w) // dynamic vs static template -CUHOSTDEV inline decltype(T()+W) +FF_CUHOSTDEV inline decltype(T()+W) operator+(const T & v, const StaticValue & w) { return v + w; } template -CUHOSTDEV inline decltype(T()-W) +FF_CUHOSTDEV inline decltype(T()-W) operator-(const T & v, const StaticValue & w) { return v - w; } template -CUHOSTDEV inline decltype(T()/W) +FF_CUHOSTDEV inline decltype(T()/W) operator/(const T & v, const StaticValue & w) { return v / w; } template -CUHOSTDEV inline decltype(T()*W) +FF_CUHOSTDEV inline decltype(T()*W) operator*(const T & v, const StaticValue & w) { return v * w; } template -CUHOSTDEV inline decltype(T()%W) +FF_CUHOSTDEV inline decltype(T()%W) operator%(const T & v, const StaticValue & w) { return v % w; } template -CUHOSTDEV inline decltype(T()&W) +FF_CUHOSTDEV inline decltype(T()&W) operator&(const T & v, const StaticValue & w) { return v & w; } template -CUHOSTDEV inline decltype(T()|W) +FF_CUHOSTDEV inline decltype(T()|W) operator|(const T & v, const StaticValue & w) { return v | w; } template -CUHOSTDEV inline decltype(T()^W) +FF_CUHOSTDEV inline decltype(T()^W) operator^(const T & v, const StaticValue & w) { return v ^ w; } template -CUHOSTDEV inline decltype(T()>>W) +FF_CUHOSTDEV inline decltype(T()>>W) operator>>(const T & v, const StaticValue & w) { return v >> w; } template -CUHOSTDEV inline decltype(T()< & w) { return v << w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator==(const T & v, const StaticValue & w) { return v == w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator!=(const T & v, const StaticValue & w) { return v != w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator>=(const T & v, const StaticValue & w) { return v >= w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator<=(const T & v, const StaticValue & w) { return v <= w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator>(const T & v, const StaticValue & w) { return v > w; } template -CUHOSTDEV inline bool +FF_CUHOSTDEV inline bool operator<(const T & v, const StaticValue & w) { return v < w; @@ -665,7 +665,7 @@ operator<(const T & v, const StaticValue & w) // - 32 bit index math check template -CUHOST inline bool canUse32BitIndexMath( +FF_CUHOST inline bool canUse32BitIndexMath( ndim_t ndim, const size_t * size, const stride_t * stride @@ -696,6 +696,6 @@ CUHOST inline bool canUse32BitIndexMath( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) #endif // FF_UTILS diff --git a/include/fastfields/impl/kernels/vector/abstract_ptr.h b/include/fastfields/impl/kernels/vector/abstract_ptr.h index 44afbb1..c46e24d 100755 --- a/include/fastfields/impl/kernels/vector/abstract_ptr.h +++ b/include/fastfields/impl/kernels/vector/abstract_ptr.h @@ -46,34 +46,34 @@ class Accessors { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Accessors() {} //--- conversion --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV operator bool () const { return this->data() != nullptr; } - CUHOSTDEV + FF_CUHOSTDEV operator T* () const { return this->data(); } //--- virtual ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV virtual pointer data() const = 0; - CUHOSTDEV + FF_CUHOSTDEV virtual offset_type stride() const = 0; //--- accessors ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline reference operator*() const { return *(this->data()); } - CUHOSTDEV + FF_CUHOSTDEV inline reference operator[](offset_type i) const { return this->data()[i*this->stride()]; @@ -81,73 +81,73 @@ class Accessors { //--- comparisons -------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline bool operator== (const T * other) const { return this->data() == other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator== (const Accessors & other) const { return this->data() == other.data(); } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator!= (const T * other) const { return this->data() != other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator!= (const Accessors & other) const { return this->data() != other.data(); } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator< (const T * other) const { return this->data() < other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator< (const Accessors & other) const { return this->data() < other.data(); } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator<= (const T * other) const { return this->data() <= other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator<= (const Accessors & other) const { return this->data() <= other.data(); } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator> (const T * other) const { return this->data() > other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator> (const Accessors & other) const { return this->data() > other.data(); } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator>= (const T * other) const { return this->data() >= other; } - CUHOSTDEV + FF_CUHOSTDEV inline bool operator>= (const Accessors & other) const { return this->data() >= other.data(); @@ -155,14 +155,14 @@ class Accessors { //--- arithmetic --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline final_type & operator++() { this->data() += this->stride(); return *this; } - CUHOSTDEV + FF_CUHOSTDEV inline final_type operator++(int) { final_type copy = *this; @@ -170,14 +170,14 @@ class Accessors { return copy; } - CUHOSTDEV + FF_CUHOSTDEV inline final_type & operator+=(offset_type offset) { this->data() += offset * this->stride(); return *this; } - CUHOSTDEV + FF_CUHOSTDEV inline final_type & operator-=(offset_type offset) { this->data() -= offset * this->stride(); @@ -186,7 +186,7 @@ class Accessors { // --- external operators --- - CUHOSTDEV + FF_CUHOSTDEV friend inline final_type operator+(const final_type & ptr, offset_type offset) { auto copy = ptr; @@ -194,7 +194,7 @@ class Accessors { return copy; } - CUHOSTDEV + FF_CUHOSTDEV friend inline final_type operator-(const final_type & ptr, offset_type offset) { auto copy = ptr; @@ -238,12 +238,12 @@ class Iterators: public Accessors > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Iterators() {} //--- implicit conversion ------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV operator WeakRef () const { return WeakRef(this->data(), this->stride()); @@ -256,7 +256,7 @@ class Iterators: public Accessors > { //--- concrete ----------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline offset_type stride() const { return S; @@ -264,26 +264,26 @@ class Iterators: public Accessors > { //--- iterators ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline iterator begin() const { return iterator(this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline reverse_iterator rend() const { return reverse_iterator( this->data() - this->stride(), -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator cbegin() const { return this->begin(); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator crend() const { return this->rend(); @@ -321,11 +321,11 @@ class Impl: public Iterators > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Impl() {} //--- constructor -------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV Impl(offset_type stride = S) { if (S != stride) throw std::runtime_error("stride not consistent"); @@ -333,7 +333,7 @@ class Impl: public Iterators > { //--- concrete ----------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline offset_type stride() const { return S; @@ -367,18 +367,18 @@ class Impl: public Iterators_stride; @@ -425,7 +425,7 @@ public internal::abstractptr::Impl > //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~AbstractPointer() {} //--- constructor -------------------------------------------------- diff --git a/include/fastfields/impl/kernels/vector/abstract_sized.h b/include/fastfields/impl/kernels/vector/abstract_sized.h index a156e72..7f64617 100755 --- a/include/fastfields/impl/kernels/vector/abstract_sized.h +++ b/include/fastfields/impl/kernels/vector/abstract_sized.h @@ -46,7 +46,7 @@ class Base: public AbstractPointer > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Base() {} //--- constructor -------------------------------------------------- @@ -55,12 +55,12 @@ class Base: public AbstractPointer > { //--- virtual ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual size_type size() const = 0; //--- accessors ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline reference at(offset_type i) { if (i < 0 || i >= this->size()) @@ -68,7 +68,7 @@ class Base: public AbstractPointer > { return (*this)[i]; } - CUHOSTDEV + FF_CUHOSTDEV inline const_reference at(offset_type i) const { if (i < 0 || i >= this->size()) @@ -78,14 +78,14 @@ class Base: public AbstractPointer > { //--- iterators ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline iterator end() const { return iterator( this->data() + this->size() * this->stride(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline reverse_iterator rbegin() const { return reverse_iterator( @@ -93,13 +93,13 @@ class Base: public AbstractPointer > { -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator cend() const { return static_cast(this->end()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator crbegin() const { return static_cast(this->rbegin()); @@ -137,14 +137,14 @@ class Impl: public Base > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Impl() {} //--- constructor -------------------------------------------------- using Base >::Base; - CUHOSTDEV + FF_CUHOSTDEV Impl(size_type size = N, offset_type stride = S): parent_type::Base(stride) { @@ -153,7 +153,7 @@ class Impl: public Base > { //--- make concrete ------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline size_type size() const { return N; } }; @@ -186,21 +186,21 @@ class Impl: public Base >::Base; - CUHOSTDEV + FF_CUHOSTDEV Impl(size_type size = 0, offset_type stride = S): parent_type::Base(stride), _size(size) {} //--- make concrete ------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline size_type size() const { return this->_size; } protected: @@ -243,7 +243,7 @@ public internal::abstractsized::Impl > //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~AbstractSizedPointer() {} //--- constructor -------------------------------------------------- diff --git a/include/fastfields/impl/kernels/vector/abstract_vector.h b/include/fastfields/impl/kernels/vector/abstract_vector.h index 9e06a24..5c8cec7 100755 --- a/include/fastfields/impl/kernels/vector/abstract_vector.h +++ b/include/fastfields/impl/kernels/vector/abstract_vector.h @@ -36,97 +36,97 @@ class Accessors { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Accessors() {} //--- conversion --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV operator bool () const { return this->data() != nullptr; } - CUHOSTDEV + FF_CUHOSTDEV operator const_reference () const { return this->data(); } - CUHOSTDEV + FF_CUHOSTDEV operator reference () { return this->data(); } //--- virtual ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV virtual pointer data() = 0; - CUHOSTDEV + FF_CUHOSTDEV virtual const_pointer data() const = 0; - CUHOSTDEV + FF_CUHOSTDEV virtual offset_type stride() const = 0; - CUHOSTDEV + FF_CUHOSTDEV virtual size_type size() const = 0; //--- accessors ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline const T & x() const noexcept { return (*this)[0]; } - CUHOSTDEV + FF_CUHOSTDEV inline const T & y() const noexcept { return (*this)[1]; } - CUHOSTDEV + FF_CUHOSTDEV inline const T & z() const noexcept { return (*this)[2]; } - CUHOSTDEV + FF_CUHOSTDEV inline T & x() noexcept { return (*this)[0]; } - CUHOSTDEV + FF_CUHOSTDEV inline T & y() noexcept { return (*this)[1]; } - CUHOSTDEV + FF_CUHOSTDEV inline T & z() noexcept { return (*this)[2]; } - CUHOSTDEV + FF_CUHOSTDEV inline reference operator*() noexcept { return *(this->data()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reference operator*() const noexcept { return *(this->data()); } - CUHOSTDEV + FF_CUHOSTDEV inline reference operator[](offset_type i) noexcept { return this->data()[i*this->stride()]; } - CUHOSTDEV + FF_CUHOSTDEV inline const_reference operator[](offset_type i) const noexcept { return this->data()[i*this->stride()]; } - CUHOSTDEV + FF_CUHOSTDEV inline reference at(offset_type i) { if (i < 0 || i >= this->size()) @@ -134,7 +134,7 @@ class Accessors { return (*this)[i]; } - CUHOSTDEV + FF_CUHOSTDEV inline const_reference at(offset_type i) const { if (i < 0 || i >= this->size()) @@ -144,7 +144,7 @@ class Accessors { //--- setters ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline void fill(const T & value) { for (long d = 0; d < this->size(); ++d) @@ -152,7 +152,7 @@ class Accessors { } template - CUHOSTDEV + FF_CUHOSTDEV inline void copy(const U * other) // assume stride 1 and length N { for (size_type d = 0; d < this->size(); ++d) @@ -160,7 +160,7 @@ class Accessors { } template - CUHOSTDEV + FF_CUHOSTDEV inline void copy(Iterator begin, const Iterator & end) { for (size_type d=0; begin != end; ++begin, ++d) @@ -168,7 +168,7 @@ class Accessors { } template - CUHOSTDEV + FF_CUHOSTDEV inline void copy(const AbstractSizedPointer & other) { size_type nb_elem = (this->size() >= other.size() ? this->size() : other.size()); @@ -177,7 +177,7 @@ class Accessors { } template - CUHOSTDEV + FF_CUHOSTDEV inline void copy(const AbstractVector & other) { size_type nb_elem = (this->size() >= other.size() ? this->size() : other.size()); @@ -188,7 +188,7 @@ class Accessors { //--- comparisons -------------------------------------------------- template - CUHOSTDEV + FF_CUHOSTDEV inline bool operator == (const U & other) { for (size_type d=0; dsize(); ++d) @@ -197,7 +197,7 @@ class Accessors { } template - CUHOSTDEV + FF_CUHOSTDEV inline bool operator != (const U & other) { return !((*this) == other); @@ -238,12 +238,12 @@ class Iterators: public Accessors > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Iterators() {} //--- concrete ----------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline offset_type stride() const { return S; @@ -251,59 +251,59 @@ class Iterators: public Accessors > { //--- iterators ---------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline iterator begin() { return iterator(this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator begin() const { return iterator(this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline reverse_iterator rend() { return reverse_iterator( this->data() - this->stride(), -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator rend() const { return reverse_iterator( this->data() - this->stride(), -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator cbegin() const { return this->begin(); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator crend() const { return this->rend(); } - CUHOSTDEV + FF_CUHOSTDEV inline iterator end() { return iterator( this->data() + this->size() * this->stride(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator end() const { return iterator( this->data() + this->size() * this->stride(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline reverse_iterator rbegin() { return reverse_iterator( @@ -311,7 +311,7 @@ class Iterators: public Accessors > { -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator rbegin() const { return reverse_iterator( @@ -319,13 +319,13 @@ class Iterators: public Accessors > { -this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_iterator cend() const { return this->end(); } - CUHOSTDEV + FF_CUHOSTDEV inline const_reverse_iterator crbegin() const { return this->rbegin(); @@ -363,12 +363,12 @@ class SwitchStride: public Iterators > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~SwitchStride() {} //--- constructor -------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV SwitchStride(offset_type stride = S) { if (S != stride) throw std::runtime_error("stride not consistent"); @@ -376,7 +376,7 @@ class SwitchStride: public Iterators > { //--- concrete ----------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline offset_type stride() const { return S; @@ -410,18 +410,18 @@ class SwitchStride: public Iterators_stride; @@ -462,12 +462,12 @@ class SwitchSize: public SwitchStride > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~SwitchSize() {} //--- constructor -------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV SwitchSize(size_type size = N, offset_type stride = S): parent_type(stride) { @@ -476,7 +476,7 @@ class SwitchSize: public SwitchStride > { //--- concrete ----------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline size_type size() const { return N; @@ -510,19 +510,19 @@ class SwitchSize: public SwitchStride_size; @@ -574,7 +574,7 @@ public internal::abstractvec::SwitchSize > //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~AbstractVector() {} //--- constructor --------------------------------------------------- @@ -583,61 +583,61 @@ public internal::abstractvec::SwitchSize > //--- conversions -------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline weak_type weak() { return weak_type(this->size(), this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_weak_type weak() const { return const_weak_type(this->size(), this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_weak_type cweak() const { return this->ref(); } - CUHOSTDEV + FF_CUHOSTDEV inline ref_type ref() { return ref_type(this->size(), this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_ref_type ref() const { return const_ref_type(this->size(), this->data(), this->stride()); } - CUHOSTDEV + FF_CUHOSTDEV inline const_ref_type cref() const { return this->ref(); } - CUHOSTDEV + FF_CUHOSTDEV inline operator weak_type () { return this->weak(); } - CUHOSTDEV + FF_CUHOSTDEV inline operator const_weak_type () const { return this->cweak(); } - CUHOSTDEV + FF_CUHOSTDEV inline operator ref_type () { return this->ref(); } - CUHOSTDEV + FF_CUHOSTDEV inline operator const_ref_type () const { return this->cref(); @@ -646,7 +646,7 @@ public internal::abstractvec::SwitchSize > //--- unbind ------------------------------------------------------- template - CUHOSTDEV + FF_CUHOSTDEV void unbind(U&... x) const { return this->ref().unbind(); @@ -658,28 +658,28 @@ public internal::abstractvec::SwitchSize > // ===================================================================== template -CUHOSTDEV +FF_CUHOSTDEV inline T& get(AbstractVector & v) noexcept { return v[I]; }; template -CUHOSTDEV +FF_CUHOSTDEV inline T&& get(AbstractVector && v) noexcept { return static_cast(v[I]); }; template -CUHOSTDEV +FF_CUHOSTDEV inline const T& get(const AbstractVector & v) noexcept { return v[I]; }; template -CUHOSTDEV +FF_CUHOSTDEV inline const T&& get(const AbstractVector && v) noexcept { return static_cast(v[I]); diff --git a/include/fastfields/impl/kernels/vector/concrete_vector.h b/include/fastfields/impl/kernels/vector/concrete_vector.h index d5a5f43..0dffc8c 100755 --- a/include/fastfields/impl/kernels/vector/concrete_vector.h +++ b/include/fastfields/impl/kernels/vector/concrete_vector.h @@ -42,42 +42,42 @@ class Vector: public AbstractVector > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~Vector() {} //--- constructors ------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV Vector() {} - CUHOSTDEV + FF_CUHOSTDEV Vector(const T & value) { this->fill(value); } - CUHOSTDEV + FF_CUHOSTDEV Vector(std::initializer_list list) { this->copy(list.begin(), list.end()); } template - CUHOSTDEV + FF_CUHOSTDEV Vector(const U * other) // assume stride 1 and length N { this->copy(other); } template - CUHOSTDEV + FF_CUHOSTDEV Vector(Iterator begin, const Iterator & end) { this->copy(begin, end); } template - CUHOSTDEV + FF_CUHOSTDEV Vector(const AbstractSizedPointer & other) { this->copy(other); @@ -85,13 +85,13 @@ class Vector: public AbstractVector > { //--- other -------------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV inline const T * data() const { return this->_data; } - CUHOSTDEV + FF_CUHOSTDEV inline T * data() { return this->_data; @@ -136,7 +136,7 @@ public AbstractVector > //--- destructor --------------------------------------------------- - CUHOST + FF_CUHOST virtual ~Vector() { delete this->_data; @@ -144,19 +144,19 @@ public AbstractVector > //--- constructors ------------------------------------------------- - CUHOST + FF_CUHOST Vector(size_type size): parent_type(size), _data(new T[size]) {} - CUHOST + FF_CUHOST Vector(size_type size, const T & value): parent_type(size), _data(new T[size]) { this->fill(value); } - CUHOST + FF_CUHOST template Vector(size_type size, const U * other): parent_type(size), _data(new T[size]) @@ -164,7 +164,7 @@ public AbstractVector > this->copy(other); } - CUHOST + FF_CUHOST template Vector(size_type size, Iterator begin, const Iterator & end): parent_type(size), _data(new T[size]) @@ -172,14 +172,14 @@ public AbstractVector > this->copy(begin, end); } - CUHOST + FF_CUHOST Vector(std::initializer_list other): parent_type(other.size()), _data(new T[other.size()]) { this->copy(other.begin(), other.end()); } - CUHOST + FF_CUHOST template Vector(const AbstractVector & other): parent_type(other.size()), _data(new T[other.size()]) @@ -189,13 +189,13 @@ public AbstractVector > //--- other -------------------------------------------------------- - CUHOST + FF_CUHOST inline const T * data() const { return this->_data; } - CUHOST + FF_CUHOST inline T * data() { return this->_data; diff --git a/include/fastfields/impl/kernels/vector/weak_ref.h b/include/fastfields/impl/kernels/vector/weak_ref.h index 8e81cde..27b5b05 100755 --- a/include/fastfields/impl/kernels/vector/weak_ref.h +++ b/include/fastfields/impl/kernels/vector/weak_ref.h @@ -32,22 +32,22 @@ class WeakRef: public AbstractPointer > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~WeakRef() {} //--- constructors ------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV WeakRef(offset_type stride): parent_type(stride) {} - CUHOSTDEV + FF_CUHOSTDEV WeakRef(T * other = nullptr, offset_type stride=S): parent_type(stride), _data(other) {} - CUHOSTDEV + FF_CUHOSTDEV template WeakRef(const AbstractPointer & other): parent_type(other.stride()), _data(other.data()) @@ -55,7 +55,7 @@ class WeakRef: public AbstractPointer > { //--- virtual ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline T * data() const { return _data; @@ -70,21 +70,21 @@ template using DynamicWeakRef = WeakRef; template -CUHOSTDEV +FF_CUHOSTDEV WeakRef weak_ref(T * ptr) { return WeakRef(ptr); } template -CUHOSTDEV +FF_CUHOSTDEV DynamicWeakRef weak_ref(T * ptr, long stride) { return DynamicWeakRef(ptr, stride); } template -CUHOSTDEV +FF_CUHOSTDEV WeakRef weak_ref(const AbstractPointer & ptr) { return WeakRef(ptr); diff --git a/include/fastfields/impl/kernels/vector/weak_sized.h b/include/fastfields/impl/kernels/vector/weak_sized.h index 739c0ce..bc6b78b 100755 --- a/include/fastfields/impl/kernels/vector/weak_sized.h +++ b/include/fastfields/impl/kernels/vector/weak_sized.h @@ -35,27 +35,27 @@ public AbstractSizedPointer > //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~WeakSizedRef() {} //--- constructors ------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV WeakSizedRef(size_type size, offset_type stride): parent_type(size, stride), _data(nullptr) {} - CUHOSTDEV + FF_CUHOSTDEV WeakSizedRef(size_type size, T * other = nullptr, offset_type stride = S): parent_type(size, stride), _data(other) {} - CUHOSTDEV + FF_CUHOSTDEV WeakSizedRef(T * other, offset_type stride = S): parent_type(N, stride), _data(other) {} - CUHOSTDEV + FF_CUHOSTDEV template WeakSizedRef(const AbstractSizedPointer & other): parent_type(other.size(), other.stride()), _data(other.data()) @@ -63,7 +63,7 @@ public AbstractSizedPointer > //--- virtual ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline T * data() const { return _data; @@ -72,7 +72,7 @@ public AbstractSizedPointer > //--- unbind ------------------------------------------------------- template - CUHOSTDEV + FF_CUHOSTDEV void unbind(U& x, V&... y) const { if (this->size() == 0) return; @@ -83,14 +83,14 @@ public AbstractSizedPointer > } template - CUHOSTDEV + FF_CUHOSTDEV void unbind(U& x) const { if (this->size() == 0) return; x = (*this)[0]; } - CUHOSTDEV + FF_CUHOSTDEV void unbind() const {} @@ -99,35 +99,35 @@ public AbstractSizedPointer > }; template -CUHOSTDEV +FF_CUHOSTDEV WeakSizedRef weak_ref(unsigned long N, T * ptr) { return WeakSizedRef(N, ptr); } template -CUHOSTDEV +FF_CUHOSTDEV WeakSizedRef weak_ref(T ptr[N]) { return WeakSizedRef(N, ptr); } template -CUHOSTDEV +FF_CUHOSTDEV WeakSizedRef weak_ref(unsigned long N, T * ptr, long stride) { return WeakSizedRef(N, ptr, stride); } template -CUHOSTDEV +FF_CUHOSTDEV WeakSizedRef weak_ref(T ptr[N], long stride) { return WeakSizedRef(N, ptr, stride); } template -CUHOSTDEV +FF_CUHOSTDEV WeakSizedRef weak_ref(const AbstractSizedPointer & ptr) { return WeakSizedRef(ptr); diff --git a/include/fastfields/impl/kernels/vector/weak_vector.h b/include/fastfields/impl/kernels/vector/weak_vector.h index 03069f0..9f628a9 100755 --- a/include/fastfields/impl/kernels/vector/weak_vector.h +++ b/include/fastfields/impl/kernels/vector/weak_vector.h @@ -40,41 +40,41 @@ class WeakVector: public AbstractVector > { //--- destructor --------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV virtual ~WeakVector() {} //--- constructors ------------------------------------------------- - CUHOSTDEV + FF_CUHOSTDEV WeakVector(size_type size, offset_type stride): parent_type(size, stride), _data(nullptr) {} - CUHOSTDEV + FF_CUHOSTDEV WeakVector(size_type size, T * other = nullptr, offset_type stride = S): parent_type(size, stride), _data(other) {} - CUHOSTDEV + FF_CUHOSTDEV WeakVector(T * other, offset_type stride = S): parent_type(N, stride), _data(other) {} template - CUHOSTDEV + FF_CUHOSTDEV WeakVector(const AbstractSizedPointer & other): parent_type(other.size(), other.stride()), _data(other.data()) {} //--- virtual ------------------------------------------------------ - CUHOSTDEV + FF_CUHOSTDEV inline const T * data() const { return this->_data; } - CUHOSTDEV + FF_CUHOSTDEV inline T * data() { return this->_data; @@ -85,35 +85,35 @@ class WeakVector: public AbstractVector > { }; template -CUHOSTDEV +FF_CUHOSTDEV WeakVector weak_vec(unsigned long N, T * ptr) { return WeakVector(N, ptr); } template -CUHOSTDEV +FF_CUHOSTDEV WeakVector weak_vec(T ptr[N]) { return WeakVector(N, ptr); } template -CUHOSTDEV +FF_CUHOSTDEV WeakVector weak_vec(unsigned long N, T * ptr, long stride) { return WeakVector(N, ptr, stride); } template -CUHOSTDEV +FF_CUHOSTDEV WeakVector weak_vec(T ptr[N], long stride) { return WeakVector(N, ptr, stride); } template -CUHOSTDEV +FF_CUHOSTDEV WeakVector weak_vec(const AbstractSizedPointer & ptr) { return WeakVector(ptr); diff --git a/src/lib-cpu/distance.cpp b/src/lib-cpu/distance.cpp index 56c4421..0636621 100644 --- a/src/lib-cpu/distance.cpp +++ b/src/lib-cpu/distance.cpp @@ -11,7 +11,7 @@ #include "fastfields/impl/cpu/distance_spline.h" #include "fastfields/impl/cpu/distance_mesh.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -720,4 +720,4 @@ void dt_mesh( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/posdef.cpp b/src/lib-cpu/posdef.cpp index 40d9bf6..02b9882 100644 --- a/src/lib-cpu/posdef.cpp +++ b/src/lib-cpu/posdef.cpp @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/posdef.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduce/accumulation type used by the compact-symmetric kernels. @@ -566,4 +566,4 @@ void sym_invert_( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/pushpull.cpp b/src/lib-cpu/pushpull.cpp index f3b267c..161752f 100644 --- a/src/lib-cpu/pushpull.cpp +++ b/src/lib-cpu/pushpull.cpp @@ -1,11 +1,11 @@ #include "fastfields/api/cpu/pushpull.h" #include -// FF_VOIDPTR / CHECK_* / DISPATCH_PP and the reduce_t typedef, shared with +// FF_VOIDPTR / CHECK_* / FF_DISPATCH_PP and the reduce_t typedef, shared with // pushpull_backward.cpp so the two translation units cannot drift apart on // which (order, bound) pairs are statically instantiated. #include "fastfields/api/cpu/pushpull_dispatch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -174,7 +174,7 @@ void pull( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_pull, + FF_DISPATCH_PP(_pull, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -228,7 +228,7 @@ void push( const int ex = static_cast(extrapolate); // size_splinc = out (the splatted volume); size_grid = grid. - DISPATCH_PP(_push, + FF_DISPATCH_PP(_push, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -275,7 +275,7 @@ void count( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_count, + FF_DISPATCH_PP(_count, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(grid), @@ -329,7 +329,7 @@ void grad( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_grad, + FF_DISPATCH_PP(_grad, bvec, svec, static_cast(nbatch), n1, ex, abs, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -338,4 +338,4 @@ void grad( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/pushpull_backward.cpp b/src/lib-cpu/pushpull_backward.cpp index 15cf070..1585075 100644 --- a/src/lib-cpu/pushpull_backward.cpp +++ b/src/lib-cpu/pushpull_backward.cpp @@ -9,7 +9,7 @@ #include #include "fastfields/api/cpu/pushpull_dispatch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -230,7 +230,7 @@ void pull_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_pull_backward, + FF_DISPATCH_PP(_pull_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -297,7 +297,7 @@ void push_backward( const int ex = static_cast(extrapolate); // size_splinc is the *pushed volume*, i.e. ginp's shape. - DISPATCH_PP(_push_backward, + FF_DISPATCH_PP(_push_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -350,7 +350,7 @@ void count_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_count_backward, + FF_DISPATCH_PP(_count_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(gout), FF_CVOIDPTR(ginp), FF_CVOIDPTR(grid), @@ -416,7 +416,7 @@ void grad_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_grad_backward, + FF_DISPATCH_PP(_grad_backward, bvec, svec, static_cast(nbatch), n1, ex, abs, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -426,4 +426,4 @@ void grad_backward( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/reg_field.cpp b/src/lib-cpu/reg_field.cpp index 38abb23..ef2b97c 100644 --- a/src/lib-cpu/reg_field.cpp +++ b/src/lib-cpu/reg_field.cpp @@ -12,7 +12,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/reg_field.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) typedef double reduce_t; @@ -1333,4 +1333,4 @@ void field_relax_rls( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/reg_flow.cpp b/src/lib-cpu/reg_flow.cpp index dd1e02f..eff02e9 100644 --- a/src/lib-cpu/reg_flow.cpp +++ b/src/lib-cpu/reg_flow.cpp @@ -12,7 +12,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/reg_flow.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduction / accumulation type (matches jitfields' float64 default) @@ -1378,4 +1378,4 @@ void flow_relax_rls( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/resize.cpp b/src/lib-cpu/resize.cpp index 6df361b..4b4d80d 100644 --- a/src/lib-cpu/resize.cpp +++ b/src/lib-cpu/resize.cpp @@ -8,7 +8,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/resize.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -181,4 +181,4 @@ void resample( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/restrict.cpp b/src/lib-cpu/restrict.cpp index 32ee41c..b78e947 100644 --- a/src/lib-cpu/restrict.cpp +++ b/src/lib-cpu/restrict.cpp @@ -8,7 +8,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/restrict.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -181,4 +181,4 @@ void restriction( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/solve_field.cpp b/src/lib-cpu/solve_field.cpp index d1311f5..13e0912 100644 --- a/src/lib-cpu/solve_field.cpp +++ b/src/lib-cpu/solve_field.cpp @@ -11,7 +11,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/solve_field.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) typedef double reduce_t; @@ -270,4 +270,4 @@ void field_cg( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cpu/splinc.cpp b/src/lib-cpu/splinc.cpp index ba6334a..a418562 100644 --- a/src/lib-cpu/splinc.cpp +++ b/src/lib-cpu/splinc.cpp @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cpu/splinc.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -19,7 +19,7 @@ FF_NAMESPACE_BEGIN(FF_DEVICE) // Host-side poles / npoles (mirrors kernels/splinc.h get_poles). // // `std::sqrt` is qualified to stay identical to src/lib-cuda's copy, where the -// qualification is mandatory: unqualified `sqrt` there resolves to the CUDEV +// qualification is mandatory: unqualified `sqrt` there resolves to the FF_CUDEV // (__device__) `ff::cuda::sqrt` and nvcc refuses to call it from this __host__ // function. Here `ff::cpu::sqrt` is an ordinary host function, so either // spelling compiles -- keep them the same anyway. @@ -150,4 +150,4 @@ void spline_coeff( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/distance.cpp b/src/lib-cuda/distance.cpp index d80dc41..21a26ea 100644 --- a/src/lib-cuda/distance.cpp +++ b/src/lib-cuda/distance.cpp @@ -11,7 +11,7 @@ #include "fastfields/impl/cuda/distance_spline.h" #include "fastfields/impl/cuda/distance_mesh.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -714,4 +714,4 @@ void dt_mesh( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/posdef.cpp b/src/lib-cuda/posdef.cpp index 7ddf107..77fa26a 100644 --- a/src/lib-cuda/posdef.cpp +++ b/src/lib-cuda/posdef.cpp @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/posdef.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduce/accumulation type used by the compact-symmetric kernels. @@ -566,4 +566,4 @@ void sym_invert_( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/pushpull.cpp b/src/lib-cuda/pushpull.cpp index b3f4916..48f2509 100644 --- a/src/lib-cuda/pushpull.cpp +++ b/src/lib-cuda/pushpull.cpp @@ -1,11 +1,11 @@ #include "fastfields/api/cuda/pushpull.h" #include -// FF_VOIDPTR / CHECK_* / DISPATCH_PP and the reduce_t typedef, shared with +// FF_VOIDPTR / CHECK_* / FF_DISPATCH_PP and the reduce_t typedef, shared with // pushpull_backward.cpp so the two translation units cannot drift apart on // which (order, bound) pairs are statically instantiated. #include "fastfields/api/cuda/pushpull_dispatch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -178,7 +178,7 @@ void pull( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_pull, + FF_DISPATCH_PP(_pull, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -232,7 +232,7 @@ void push( const int ex = static_cast(extrapolate); // size_splinc = out (the splatted volume); size_grid = grid. - DISPATCH_PP(_push, + FF_DISPATCH_PP(_push, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -279,7 +279,7 @@ void count( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_count, + FF_DISPATCH_PP(_count, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_CVOIDPTR(grid), @@ -333,7 +333,7 @@ void grad( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_grad, + FF_DISPATCH_PP(_grad, bvec, svec, static_cast(nbatch), n1, ex, abs, FF_VOIDPTR(out), FF_CVOIDPTR(inp), FF_CVOIDPTR(grid), @@ -342,4 +342,4 @@ void grad( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/pushpull_backward.cpp b/src/lib-cuda/pushpull_backward.cpp index ea8f17d..63ee477 100644 --- a/src/lib-cuda/pushpull_backward.cpp +++ b/src/lib-cuda/pushpull_backward.cpp @@ -13,7 +13,7 @@ #include #include "fastfields/api/cuda/pushpull_dispatch.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -238,7 +238,7 @@ void pull_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_pull_backward, + FF_DISPATCH_PP(_pull_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -305,7 +305,7 @@ void push_backward( const int ex = static_cast(extrapolate); // size_splinc is the *pushed volume*, i.e. ginp's shape. - DISPATCH_PP(_push_backward, + FF_DISPATCH_PP(_push_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -358,7 +358,7 @@ void count_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_count_backward, + FF_DISPATCH_PP(_count_backward, bvec, svec, static_cast(nbatch), n1, ex, FF_VOIDPTR(gout), FF_CVOIDPTR(ginp), FF_CVOIDPTR(grid), @@ -424,7 +424,7 @@ void grad_backward( const spline::SplineVec svec(spl); const int ex = static_cast(extrapolate); - DISPATCH_PP(_grad_backward, + FF_DISPATCH_PP(_grad_backward, bvec, svec, static_cast(nbatch), n1, ex, abs, FF_VOIDPTR(out), FF_VOIDPTR(gout), @@ -434,4 +434,4 @@ void grad_backward( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/reg_field.cpp b/src/lib-cuda/reg_field.cpp index 3c84e1d..aad45d9 100644 --- a/src/lib-cuda/reg_field.cpp +++ b/src/lib-cuda/reg_field.cpp @@ -13,7 +13,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/reg_field.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) typedef double reduce_t; @@ -958,4 +958,4 @@ void field_precond_( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/reg_field_rls.cpp b/src/lib-cuda/reg_field_rls.cpp index ecef8a0..8345574 100644 --- a/src/lib-cuda/reg_field_rls.cpp +++ b/src/lib-cuda/reg_field_rls.cpp @@ -13,7 +13,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/reg_field.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) typedef double reduce_t; @@ -478,4 +478,4 @@ void field_relax_rls( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/reg_flow.cpp b/src/lib-cuda/reg_flow.cpp index e5d8436..fa7a2b2 100644 --- a/src/lib-cuda/reg_flow.cpp +++ b/src/lib-cuda/reg_flow.cpp @@ -12,7 +12,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/reg_flow.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduction / accumulation type (matches jitfields' float64 default) @@ -1032,4 +1032,4 @@ void flow_precond_( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/reg_flow_rls.cpp b/src/lib-cuda/reg_flow_rls.cpp index 5aa6b69..8da7e4a 100644 --- a/src/lib-cuda/reg_flow_rls.cpp +++ b/src/lib-cuda/reg_flow_rls.cpp @@ -12,7 +12,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/reg_flow.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) // reduction / accumulation type (matches jitfields' float64 default) @@ -416,4 +416,4 @@ void flow_relax_rls( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/resize.cpp b/src/lib-cuda/resize.cpp index af931c1..a9cae09 100644 --- a/src/lib-cuda/resize.cpp +++ b/src/lib-cuda/resize.cpp @@ -8,7 +8,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/resize.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -152,4 +152,4 @@ void resample( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/restrict.cpp b/src/lib-cuda/restrict.cpp index fd0b373..a748982 100644 --- a/src/lib-cuda/restrict.cpp +++ b/src/lib-cuda/restrict.cpp @@ -8,7 +8,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/restrict.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -152,4 +152,4 @@ void restriction( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib-cuda/splinc.cpp b/src/lib-cuda/splinc.cpp index 5346d9d..47529c0 100644 --- a/src/lib-cuda/splinc.cpp +++ b/src/lib-cuda/splinc.cpp @@ -9,7 +9,7 @@ #include "fastfields/impl/kernels/utils.h" #include "fastfields/impl/cuda/splinc.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) FF_NAMESPACE_BEGIN(FF_DEVICE) /*********************************************************************** @@ -22,7 +22,7 @@ FF_NAMESPACE_BEGIN(FF_DEVICE) // choice. This function is host-only, but it is defined inside `ff::cuda`, so // unqualified `sqrt` finds `ff::cuda::sqrt` from impl/kernels/utils.h by // ordinary lookup in the enclosing namespace -- and under __CUDACC__ that -// overload is CUDEV (__device__), which a __host__ function may not call. +// overload is FF_CUDEV (__device__), which a __host__ function may not call. // nvcc rejected this file outright until the calls were qualified; nothing // noticed, because the module was missing from src/lib-cuda/Makefile's MODULES // and so had never been compiled (fastfields-lib#80). src/lib-cpu's copy is @@ -155,4 +155,4 @@ void spline_coeff( } FF_NAMESPACE_END(FF_DEVICE) -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/distance.cpp b/src/lib/distance.cpp index e053376..8fca611 100644 --- a/src/lib/distance.cpp +++ b/src/lib/distance.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/distance.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void dt_euclidean( DLTensor & inp_out , @@ -143,4 +143,4 @@ void dt_mesh( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/posdef.cpp b/src/lib/posdef.cpp index 8bea319..9dd2934 100644 --- a/src/lib/posdef.cpp +++ b/src/lib/posdef.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/posdef.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void sym_matvec( DLTensor & out , @@ -160,4 +160,4 @@ void sym_invert_( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/pushpull.cpp b/src/lib/pushpull.cpp index 5d8e0bd..fcd8666 100644 --- a/src/lib/pushpull.cpp +++ b/src/lib/pushpull.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/pushpull.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void pull( DLTensor & out, @@ -196,4 +196,4 @@ void grad_backward( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/reg_field.cpp b/src/lib/reg_field.cpp index 52d23ce..911e972 100644 --- a/src/lib/reg_field.cpp +++ b/src/lib/reg_field.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/reg_field.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void field_matvec( DLTensor & out , @@ -411,4 +411,4 @@ void field_relax_rls( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/reg_flow.cpp b/src/lib/reg_flow.cpp index 0e89b46..c678ed7 100644 --- a/src/lib/reg_flow.cpp +++ b/src/lib/reg_flow.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/reg_flow.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void flow_matvec( DLTensor & out , @@ -441,4 +441,4 @@ void flow_relax_rls( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/resize.cpp b/src/lib/resize.cpp index f4f3a26..6875ba3 100644 --- a/src/lib/resize.cpp +++ b/src/lib/resize.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/resize.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void resample( DLTensor & out , @@ -32,4 +32,4 @@ void resample( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/restrict.cpp b/src/lib/restrict.cpp index 3273fb5..5b93601 100644 --- a/src/lib/restrict.cpp +++ b/src/lib/restrict.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/restrict.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void restriction( DLTensor & out , @@ -32,4 +32,4 @@ void restriction( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/solve_field.cpp b/src/lib/solve_field.cpp index 7aa3c50..d6d5127 100644 --- a/src/lib/solve_field.cpp +++ b/src/lib/solve_field.cpp @@ -3,7 +3,7 @@ #include "fastfields/api/checks.h" #include "fastfields/api/cpu/solve_field.h" -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) // The CG driver is CPU-only for now: unlike the other modules there is no // `FF_CUDA::field_cg` to forward to, because the solver's dot products need a @@ -39,4 +39,4 @@ void field_cg( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/src/lib/splinc.cpp b/src/lib/splinc.cpp index ebeb769..d796d77 100644 --- a/src/lib/splinc.cpp +++ b/src/lib/splinc.cpp @@ -7,7 +7,7 @@ #include "fastfields/api/cuda/splinc.h" #endif -FF_NAMESPACE_BEGIN(FF) +FF_NAMESPACE_BEGIN(FF_NS) void spline_coeff( DLTensor & inp_out , @@ -32,4 +32,4 @@ void spline_coeff( throw std::invalid_argument("unsupported device"); } -FF_NAMESPACE_END(FF) +FF_NAMESPACE_END(FF_NS) diff --git a/tests/impl-cuda/compile_probe_mesh.cu b/tests/impl-cuda/compile_probe_mesh.cu index 483add2..1a1fb34 100644 --- a/tests/impl-cuda/compile_probe_mesh.cu +++ b/tests/impl-cuda/compile_probe_mesh.cu @@ -51,7 +51,7 @@ // which reports a spurious // error: calling a __device__ function("ff::cuda::prod") from a // __host__ function("sdt") is not allowed -// because `prod` is CUDEV. `distance_euclidean.h`'s `dt` reproduces that +// because `prod` is FF_CUDEV. `distance_euclidean.h`'s `dt` reproduces that // identically yet compiles fine in the real build, so it is an artifact of the // probe technique, not a defect. See fastfields-cuda-impl#40. diff --git a/tools/rename-macros.py b/tools/rename-macros.py new file mode 100644 index 0000000..fd00c5f --- /dev/null +++ b/tools/rename-macros.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +rename-macros.py -- give every macro on the installed public surface an FF_ +prefix. + +THE RULE THIS APPLIES +-------------------------------------------------------------------------- +Every macro that survives preprocessing of a header under `include/` -- i.e. +every `#define` that is not `#undef`'d before the end of the header that +defined it -- must be spelled `FF_*`. Installing a header means every +translation unit downstream of it inherits its macros, and an unprefixed one +is a name this project has silently taken from everybody who includes us. +184 macros here already follow the convention; these are the stragglers. + +Macros defined and consumed inside a single `.cpp` are NOT covered: they never +leave the translation unit, so they cannot collide with anyone. (A macro that +gets *moved into* a header is covered from the moment it moves -- which is why +de-duplication and prefixing are one job, done in that order. That half landed +in the preceding commit.) + +Where an `inline` function will do the job, prefer it and delete the macro +outright: a function in `ff::` is collision-safe with no prefix at all. The +preceding commit did that for `IS_CPU` / `IS_CUDA`; this one does it for +`uchar_t`. + +DELIBERATELY EXEMPT +-------------------------------------------------------------------------- +`include/fastfields/core/cuda_switch.h` keeps two families unprefixed, because +prefixing them would destroy the thing they exist to do: + + * `#define int8_t signed char` ... `#define uint64_t unsigned long`, under + `#ifdef __CUDACC_RTC__`. NVRTC ships no standard library, so these hand + definitions ARE `` in that mode. They must keep the standard + spellings or nothing downstream compiles. + * `#define __device__` / `#define __host__`, under `#ifndef __CUDACC__`. + These erase nvcc's qualifiers when a host compiler sees CUDA-annotated + code. Renaming them would leave the real names undefined. + +`include/fastfields/core/dlpack.h` is vendored upstream code (`DLPACK_*`) and +is never rewritten by this script. + +USAGE +-------------------------------------------------------------------------- + python3 tools/rename-macros.py # apply + python3 tools/rename-macros.py --check # report residue, change nothing + +Idempotent, and whole-identifier: `CUHOST` cannot eat `CUHOSTDEV`, and `FF` +cannot eat `FF_DEVICE`. Every rename is compiler-verified -- a missed site is +an undeclared identifier, not a silent behaviour change. + +If this lands on a base that has moved, do NOT resolve conflicts by hand: +reset, re-run the script on the new base, and commit that. It is deterministic, +which is the whole reason it is committed rather than described. +""" + +import os +import re +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +SOURCE_DIRS = ("include", "src", "tests") +SOURCE_EXTS = (".h", ".hpp", ".inl", ".cpp", ".cu", ".cuh") + +# Vendored third-party code: never rewritten. +SKIP = {"include/fastfields/core/dlpack.h"} + +RENAMES = [ + # ---- the namespace spine ------------------------------------------- + # `FF` is a two-letter, all-caps macro in an installed header -- the worst + # offender on the list. See the commit message for why it is renamed + # rather than left alone or #undef'd (the #undef option is not merely + # awkward here, it is silently wrong). + ("FF", "FF_NS"), + # ---- core/cuda_switch.h: the CUDA qualifier abbreviations ----------- + ("CUGLOB", "FF_CUGLOB"), + ("CUHOSTDEV", "FF_CUHOSTDEV"), + ("CUHOST", "FF_CUHOST"), + ("CUDEV", "FF_CUDEV"), + # ---- api/{cpu,cuda}/pushpull_dispatch.h ---------------------------- + # Renamed only. The dispatch pyramid's *design* is a separate change in + # someone else's hands; this is spelling. DISPATCH_PP becomes + # FF_DISPATCH_PP rather than FF_PP_DISPATCH because impl/cuda/pushpull.h + # already uses the latter name (it #undef's it, so there is no actual + # collision -- but reusing the spelling would be gratuitously confusing). + ("DISPATCH_PP", "FF_DISPATCH_PP"), + ("PP_BOUND", "FF_PP_BOUND"), + ("PP_DTYPE", "FF_PP_DTYPE"), + ("PP_ORDER", "FF_PP_ORDER"), + # ---- impl/kernels/ -------------------------------------------------- + ("ATOMIC_INTEGER_IMPL", "FF_ATOMIC_INTEGER_IMPL"), + ("GPU_ATOMIC_INTEGER", "FF_GPU_ATOMIC_INTEGER"), + ("INTERPOL_UTILS", "FF_INTERPOL_UTILS"), + ("DIST_USE_LOOP", "FF_DIST_USE_LOOP"), + # `JFH_` is a leftover prefix from the jitfields headers this code came + # from; it is a prefix, but not this project's prefix. + ("JFH_OnePlusTiny", "FF_ONE_PLUS_TINY"), +] + +# `uchar_t` is deleted rather than renamed. A lowercase macro that +# impersonates a typedef is worse than a shouty one: a downstream +# `typedef unsigned char uchar_t;` does not merely collide, it fails to +# compile with a diagnostic pointing at the wrong file. Six uses, two headers. +TYPE_MACROS = {"uchar_t": "unsigned char"} + +# JFH_OnePlusTiny is #defined identically in three sibling headers. Only +# posdef/utils.h keeps the definition -- the other two both include it. +DEDUP_DEFINE = { + "FF_ONE_PLUS_TINY": ( + "include/fastfields/impl/kernels/posdef/utils.h", + ( + "include/fastfields/impl/kernels/posdef/cholesky.h", + "include/fastfields/impl/kernels/posdef/posdef.h", + ), + ) +} + + +def sources(): + for d in SOURCE_DIRS: + for dirpath, _, filenames in os.walk(os.path.join(ROOT, d)): + for name in sorted(filenames): + if not name.endswith(SOURCE_EXTS): + continue + rel = os.path.relpath(os.path.join(dirpath, name), ROOT) + if rel not in SKIP: + yield rel + + +def rename(text, old, new): + return re.sub( + r"(?