Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,7 @@ impl/kernels ─ impl/cpu ─ api/cpu + src/lib-cpu ┐
Namespace `ff::<FF_DEVICE>::<module>` via the `FF_NAMESPACE_BEGIN` macros —
do **not** hard-code `ff::cpu`.
- **`include/fastfields/impl/cpu/`**, **`impl/cuda/`** — the loops over
elements (thread pool / OpenMP; `__global__` kernels + `CUHOST` launchers).
elements (thread pool / OpenMP; `__global__` kernels + `FF_CUHOST` launchers).
Header-only, templated, dynamic sizes. `ff::cpu::…` / `ff::cuda::…`.
- **`include/fastfields/api/cpu/`**, **`api/cuda/`** + **`src/lib-cpu/`**,
**`src/lib-cuda/`** — the dtype-dispatch boundary. Public symbols take
Expand Down
2 changes: 1 addition & 1 deletion MIGRATION.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -135,7 +135,7 @@ CPU path against a brute-force / reference implementation, as `test_distance.cpp
spurious extra arg (no matching overload); triggered by `restrict::loop`.
10. **cpu-impl/{resize,restrict,splinc}.h** — wrong include prefix `"lib/…"` →
`"kernels/…"`; impl namespace was plain `ff::<module>` but the kernels live in
`ff::cpu::` (`FF_DEVICE`) so it must be `FF_NAMESPACE_BEGIN(FF)/(FF_DEVICE)/(<module>)`
`ff::cpu::` (`FF_DEVICE`) so it must be `FF_NAMESPACE_BEGIN(FF_NS)/(FF_DEVICE)/(<module>)`
like distance; `index2offset_nd<ndim>()` runtime-ndim → dynamic overload;
`jf::has_atomic_add` → `has_atomic_add`.

Expand Down
4 changes: 2 additions & 2 deletions include/fastfields/api/checks.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,7 +4,7 @@
#include "fastfields/core/dlpack.h"
#include "fastfields/core/defines.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

/**
* Assert that a set of DLTensors all live on the same device.
Expand DownExpand Up@@ -55,6 +55,6 @@ inline void require_same_device(const DLTensor & ref, const DLTensor & t, const
require_same_device(ref, rest...);
}

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_CHECKS
56 changes: 28 additions & 28 deletions include/fastfields/api/cpu/pushpull_dispatch.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
#include "fastfields/impl/kernels/utils.h"
#include "fastfields/impl/cpu/pushpull.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)
FF_NAMESPACE_BEGIN(FF_DEVICE)

// reduce/accumulation type used by the sampling kernels. Match jitfields
Expand All@@ -34,7 +34,7 @@ typedef double reduce_t;
* DISPATCH *
***********************************************************************/

#define PP_DTYPE(D, I, B, FN, args...) \
#define FF_PP_DTYPE(D, I, B, FN, args...) \
switch (code) { \
case kDLFloat: switch (bits) { \
case 32: return (use_32bits ? FN<D,I,B,float, int32_t>(args) \
Expand All@@ -45,25 +45,25 @@ typedef double reduce_t;
}; default: break; \
}

// The template argument fed to PP_DTYPE is FF_BOUND_<NAME> / FF_SPLINE_<NAME>
// The template argument fed to FF_PP_DTYPE is FF_BOUND_<NAME> / FF_SPLINE_<NAME>
// (kernels/bounds.h, kernels/spline.h) -- the condition/order itself when it
// is statically compiled, `Dynamic` otherwise per BOUNDFLAGS/SPLINEFLAGS. The
// switch labels stay exhaustive on the *runtime* value either way; only the
// instantiated template argument collapses onto the shared Dynamic path.
#define PP_BOUND(D, I, FN, args...) \
#define FF_PP_BOUND(D, I, FN, args...) \
switch (bnd) { \
case bound_t::Zero: PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \
case bound_t::Replicate: PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \
case bound_t::DCT1: PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \
case bound_t::DCT2: PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \
case bound_t::DST1: PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \
case bound_t::DST2: PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \
case bound_t::DFT: PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \
case bound_t::NoCheck: PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \
case bound_t::Zero: FF_PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \
case bound_t::Replicate: FF_PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \
case bound_t::DCT1: FF_PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \
case bound_t::DCT2: FF_PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \
case bound_t::DST1: FF_PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \
case bound_t::DST2: FF_PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \
case bound_t::DFT: FF_PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \
case bound_t::NoCheck: FF_PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \
default: throw std::invalid_argument("Unsupported boundary condition");\
}

// There used to be a second, hand-duplicated PP_ORDER (behind `-DFF_TEST_SPARSE`)
// There used to be a second, hand-duplicated FF_PP_ORDER (behind `-DFF_TEST_SPARSE`)
// that hard-coded a *covering* subset of the order x bound matrix -- literally
// rejecting (throwing) most bound/order combinations at runtime -- purely to
// keep the test build's compile time down. That is now redundant with, and
Expand All@@ -72,36 +72,36 @@ typedef double reduce_t;
// instantiation (the actual compile-cost win FF_TEST_SPARSE was chasing),
// while every combination stays fully *functional* (just via the Dynamic
// runtime path instead of a dedicated static one) rather than throwing.
// There is therefore only one PP_ORDER/PP_BOUND now; which combinations are
// There is therefore only one FF_PP_ORDER/FF_PP_BOUND now; which combinations are
// statically instantiated and which share Dynamic is entirely a BOUNDFLAGS/
// SPLINEFLAGS *build-time* choice (Makefile: a sparser default for the `test`
// target, the full static matrix for the library), not a code-level branch.
#define PP_ORDER(D, FN, args...) \
#define FF_PP_ORDER(D, FN, args...) \
switch (spl) { \
case spline_t::Nearest: PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \
case spline_t::Linear: PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \
case spline_t::Quadratic: PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \
case spline_t::Cubic: PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \
case spline_t::FourthOrder: PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \
case spline_t::FifthOrder: PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \
case spline_t::SixthOrder: PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \
case spline_t::SeventhOrder: PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \
case spline_t::Nearest: FF_PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \
case spline_t::Linear: FF_PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \
case spline_t::Quadratic: FF_PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \
case spline_t::Cubic: FF_PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \
case spline_t::FourthOrder: FF_PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \
case spline_t::FifthOrder: FF_PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \
case spline_t::SixthOrder: FF_PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \
case spline_t::SeventhOrder: FF_PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \
default: throw std::invalid_argument("Unsupported spline order"); \
}

#define DISPATCH_PP(FN, args...) \
#define FF_DISPATCH_PP(FN, args...) \
{ \
switch (ndim) { \
case 1: PP_ORDER(1, FN, args); break; \
case 2: PP_ORDER(2, FN, args); break; \
case 3: PP_ORDER(3, FN, args); break; \
case 1: FF_PP_ORDER(1, FN, args); break; \
case 2: FF_PP_ORDER(2, FN, args); break; \
case 3: FF_PP_ORDER(3, FN, args); break; \
default: throw std::invalid_argument("Only 1D, 2D and 3D are supported"); \
}; \
throw std::invalid_argument("Unsupported data type"); \
}


FF_NAMESPACE_END(FF_DEVICE)
FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_CPU_PUSHPULL_DISPATCH
52 changes: 26 additions & 26 deletions include/fastfields/api/cuda/pushpull_dispatch.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
#include "fastfields/impl/kernels/utils.h"
#include "fastfields/impl/cuda/pushpull.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)
FF_NAMESPACE_BEGIN(FF_DEVICE)

// reduce/accumulation type used by the sampling kernels. Match jitfields
Expand All@@ -34,7 +34,7 @@ typedef double reduce_t;
* DISPATCH *
***********************************************************************/

#define PP_DTYPE(D, I, B, FN, args...) \
#define FF_PP_DTYPE(D, I, B, FN, args...) \
switch (code) { \
case kDLFloat: switch (bits) { \
case 32: return (use_32bits ? FN<D,I,B,float, int32_t>(args) \
Expand All@@ -45,52 +45,52 @@ typedef double reduce_t;
}; default: break; \
}

// The template argument fed to PP_DTYPE is FF_BOUND_<NAME> / FF_SPLINE_<NAME>
// The template argument fed to FF_PP_DTYPE is FF_BOUND_<NAME> / FF_SPLINE_<NAME>
// (kernels/bounds.h, kernels/spline.h) -- the condition/order itself when it
// is statically compiled, `Dynamic` otherwise per BOUNDFLAGS/SPLINEFLAGS. The
// switch labels stay exhaustive on the *runtime* value either way; only the
// instantiated template argument collapses onto the shared Dynamic path.
// This is what actually keeps ptxas's memory bounded -- see BOUNDFLAGS/
// SPLINEFLAGS in the Makefile.
#define PP_BOUND(D, I, FN, args...) \
#define FF_PP_BOUND(D, I, FN, args...) \
switch (bnd) { \
case bound_t::Zero: PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \
case bound_t::Replicate: PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \
case bound_t::DCT1: PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \
case bound_t::DCT2: PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \
case bound_t::DST1: PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \
case bound_t::DST2: PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \
case bound_t::DFT: PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \
case bound_t::NoCheck: PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \
case bound_t::Zero: FF_PP_DTYPE(D,I,FF_BOUND_ZERO, FN,args); break; \
case bound_t::Replicate: FF_PP_DTYPE(D,I,FF_BOUND_REPLICATE,FN,args); break; \
case bound_t::DCT1: FF_PP_DTYPE(D,I,FF_BOUND_DCT1, FN,args); break; \
case bound_t::DCT2: FF_PP_DTYPE(D,I,FF_BOUND_DCT2, FN,args); break; \
case bound_t::DST1: FF_PP_DTYPE(D,I,FF_BOUND_DST1, FN,args); break; \
case bound_t::DST2: FF_PP_DTYPE(D,I,FF_BOUND_DST2, FN,args); break; \
case bound_t::DFT: FF_PP_DTYPE(D,I,FF_BOUND_DFT, FN,args); break; \
case bound_t::NoCheck: FF_PP_DTYPE(D,I,FF_BOUND_NOCHECK, FN,args); break; \
default: throw std::invalid_argument("Unsupported boundary condition");\
}

#define PP_ORDER(D, FN, args...) \
#define FF_PP_ORDER(D, FN, args...) \
switch (spl) { \
case spline_t::Nearest: PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \
case spline_t::Linear: PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \
case spline_t::Quadratic: PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \
case spline_t::Cubic: PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \
case spline_t::FourthOrder: PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \
case spline_t::FifthOrder: PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \
case spline_t::SixthOrder: PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \
case spline_t::SeventhOrder: PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \
case spline_t::Nearest: FF_PP_BOUND(D,FF_SPLINE_NEAREST, FN,args); break; \
case spline_t::Linear: FF_PP_BOUND(D,FF_SPLINE_LINEAR, FN,args); break; \
case spline_t::Quadratic: FF_PP_BOUND(D,FF_SPLINE_QUADRATIC, FN,args); break; \
case spline_t::Cubic: FF_PP_BOUND(D,FF_SPLINE_CUBIC, FN,args); break; \
case spline_t::FourthOrder: FF_PP_BOUND(D,FF_SPLINE_FOURTHORDER, FN,args); break; \
case spline_t::FifthOrder: FF_PP_BOUND(D,FF_SPLINE_FIFTHORDER, FN,args); break; \
case spline_t::SixthOrder: FF_PP_BOUND(D,FF_SPLINE_SIXTHORDER, FN,args); break; \
case spline_t::SeventhOrder: FF_PP_BOUND(D,FF_SPLINE_SEVENTHORDER,FN,args); break; \
default: throw std::invalid_argument("Unsupported spline order"); \
}

#define DISPATCH_PP(FN, args...) \
#define FF_DISPATCH_PP(FN, args...) \
{ \
switch (ndim) { \
case 1: PP_ORDER(1, FN, args); break; \
case 2: PP_ORDER(2, FN, args); break; \
case 3: PP_ORDER(3, FN, args); break; \
case 1: FF_PP_ORDER(1, FN, args); break; \
case 2: FF_PP_ORDER(2, FN, args); break; \
case 3: FF_PP_ORDER(3, FN, args); break; \
default: throw std::invalid_argument("Only 1D, 2D and 3D are supported"); \
}; \
throw std::invalid_argument("Unsupported data type"); \
}


FF_NAMESPACE_END(FF_DEVICE)
FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_CUDA_PUSHPULL_DISPATCH
4 changes: 2 additions & 2 deletions include/fastfields/api/cuda/stream.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
#include <cstdint>
#include "fastfields/core/cuda_switch.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)
FF_NAMESPACE_BEGIN(FF_DEVICE)

// intptr_t -> cudaStream_t (0 == the default stream).
Expand All@@ -28,6 +28,6 @@ static inline cudaStream_t _reg_stream(intptr_t stream)
}

FF_NAMESPACE_END(FF_DEVICE)
FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_CUDA_STREAM
4 changes: 2 additions & 2 deletions include/fastfields/api/distance.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,7 +4,7 @@
#include <cstdint>
#include "fastfields/core/defines.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

#ifndef FF_LIB_BOUND_SPLINE_T
#define FF_LIB_BOUND_SPLINE_T
Expand DownExpand Up@@ -175,6 +175,6 @@ void dt_mesh(
intptr_t stream = 0
);

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_DISTANCE
4 changes: 2 additions & 2 deletions include/fastfields/api/posdef.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,7 +4,7 @@
#include <cstdint>
#include "fastfields/core/defines.h"

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

/**
* Compact symmetric ("Sym") positive-definite matrix operations.
Expand DownExpand Up@@ -79,6 +79,6 @@ void sym_invert_(
intptr_t stream = 0
);

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_POSDEF
8 changes: 4 additions & 4 deletions include/fastfields/api/pushpull.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@

#ifndef FF_LIB_BOUND_SPLINE_T
#define FF_LIB_BOUND_SPLINE_T
FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

FF_NAMESPACE_BEGIN(bound_t)
using T = int8_t;
Expand DownExpand Up@@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6;
static constexpr T SeventhOrder = 7;
FF_NAMESPACE_END(spline_t)

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)
#endif // FF_LIB_BOUND_SPLINE_T

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

/**
* @brief Sample ("pull") a spline-encoded volume at arbitrary coordinates.
Expand DownExpand Up@@ -227,6 +227,6 @@ void grad_backward(
intptr_t stream = 0
);

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_PUSHPULL
8 changes: 4 additions & 4 deletions include/fastfields/api/reg_field.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@

#ifndef FF_LIB_BOUND_SPLINE_T
#define FF_LIB_BOUND_SPLINE_T
FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

FF_NAMESPACE_BEGIN(bound_t)
using T = int8_t;
Expand DownExpand Up@@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6;
static constexpr T SeventhOrder = 7;
FF_NAMESPACE_END(spline_t)

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)
#endif // FF_LIB_BOUND_SPLINE_T

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

/**
* @brief Apply a spatial regulariser operator to a multi-channel field.
Expand DownExpand Up@@ -375,6 +375,6 @@ void field_relax_rls(
intptr_t stream = 0
);

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_REG_FIELD
8 changes: 4 additions & 4 deletions include/fastfields/api/reg_flow.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@

#ifndef FF_LIB_BOUND_SPLINE_T
#define FF_LIB_BOUND_SPLINE_T
FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

FF_NAMESPACE_BEGIN(bound_t)
using T = int8_t;
Expand DownExpand Up@@ -34,10 +34,10 @@ static constexpr T SixthOrder = 6;
static constexpr T SeventhOrder = 7;
FF_NAMESPACE_END(spline_t)

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)
#endif // FF_LIB_BOUND_SPLINE_T

FF_NAMESPACE_BEGIN(FF)
FF_NAMESPACE_BEGIN(FF_NS)

/**
* @brief Apply a spatial regulariser operator to a vector flow field.
Expand DownExpand Up@@ -396,6 +396,6 @@ void flow_relax_rls(
intptr_t stream = 0
);

FF_NAMESPACE_END(FF)
FF_NAMESPACE_END(FF_NS)

#endif // FF_LIB_REG_FLOW
Loading
Loading