From 87685af676eb7727b02aca1aa0d1873f1c576800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C3=ABl=20Balbastre?= Date: Thu, 20 Aug 2026 11:17:58 +0000 Subject: [PATCH] build: make the 32-bit index axis a per-backend option (FF_INDEX32) Every templated kernel below the dispatch layer is templated on `offset_t`, which has exactly two values chosen per call by `canUse32BitIndexMath`: `int32_t` when every operand's largest element offset fits in 32 bits, `int64_t` otherwise. That axis costs exactly x2 instantiations of everything underneath -- on CUDA, x2 device code, x2 SASS and x2 ptxas memory. Measured on reg_flow, the module that peaks at 12.98 GB of a 16 GB runner: dropping it is -50.3% instantiations and -44.6% peak compiler RSS (fastfields-lib#94). Nobody has ever benchmarked what the narrow path buys here -- it is an ATen inheritance and the argument for it (register pressure) is a GPU argument, while there is no GPU in CI. So this does not change the default. It makes the axis a build-time policy, per backend, so the position is reversible once somebody with hardware can answer the question. FF_INDEX32=1 (default, both backends) today's behaviour, unchanged FF_INDEX32=0 the narrow arm names int64_t too, so both arms are one instantiation, the axis collapses, and the canUse32BitIndexMath probe folds away Spelling follows BOUNDFLAGS/SPLINEFLAGS exactly: ONE macro, with the default chosen separately by each library's own Makefile via a new INDEXFLAGS variable. That per-library default IS the per-backend mechanism -- FF_STATIC_BOUNDS is one macro too, and only differs between backends because the two Makefiles say so. An FF_INDEX32_CPU/FF_INDEX32_CUDA pair would instead force core/dispatch.h to branch on __CUDACC__, putting the policy in the source in the one directory that is backend-agnostic by contract, and every third consumer would have to know both names. INDEXFLAGS stays outside CXXFLAGS for the same reason the other two do: CUDA CI overrides CXXFLAGS to force -O1 and must not silently drop a policy while doing it. The type the narrow arm names is a typedef, `ff::off32_t`, not a macro -- CLAUDE.md's own guidance (a name in ff:: is collision-safe with no prefix at all), and it keeps the 140-line source change a pure token swap of the same width, so not one line-continuation backslash moves. CI gains the two legs that make this an option rather than a claim: a `test-cpu (clang-index64)` leg that runs the whole suite with the axis off, and `build-cuda` as a two-leg matrix over the two positions -- the off leg is the cheaper of the two and turns the per-module FFMEM table into a direct measurement of what the axis costs on the backend where it costs the most. --- .github/workflows/ci.yml | 46 +++++++++- CLAUDE.md | 19 +++- MIGRATION.md | 33 +++++++ .../fastfields/api/cpu/pushpull_dispatch.h | 4 +- .../fastfields/api/cuda/pushpull_dispatch.h | 4 +- include/fastfields/core/dispatch.h | 87 ++++++++++++++++++- src/lib-cpu/Makefile | 27 +++++- src/lib-cpu/distance.cpp | 13 +-- src/lib-cpu/posdef.cpp | 8 +- src/lib-cpu/reg_field.cpp | 52 +++++------ src/lib-cpu/reg_flow.cpp | 52 +++++------ src/lib-cpu/resize.cpp | 4 +- src/lib-cpu/restrict.cpp | 4 +- src/lib-cpu/solve_field.cpp | 4 +- src/lib-cpu/splinc.cpp | 4 +- src/lib-cuda/Makefile | 36 +++++++- src/lib-cuda/distance.cpp | 13 +-- src/lib-cuda/posdef.cpp | 8 +- src/lib-cuda/reg_field.cpp | 40 ++++----- src/lib-cuda/reg_field_rls.cpp | 12 +-- src/lib-cuda/reg_flow.cpp | 40 ++++----- src/lib-cuda/reg_flow_rls.cpp | 12 +-- src/lib-cuda/resize.cpp | 4 +- src/lib-cuda/restrict.cpp | 4 +- src/lib-cuda/splinc.cpp | 4 +- 25 files changed, 382 insertions(+), 152 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be6c107..81298d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,6 +282,23 @@ jobs: boundflags: "-DFF_STATIC_BOUNDS=0 -DFF_STATIC_BOUND_DCT2=1 -DFF_STATIC_BOUND_DST2=1" splineflags: "-DFF_STATIC_SPLINES=0 -DFF_STATIC_SPLINE_NEAREST=1 -DFF_STATIC_SPLINE_LINEAR=1 -DFF_STATIC_SPLINE_QUADRATIC=1 -DFF_STATIC_SPLINE_CUBIC=1" jobs: "2" + # The third axis, in its non-default position: FF_INDEX32=0 collapses + # the int32_t/int64_t offset dispatch onto one instantiation (see + # INDEXFLAGS in src/lib-cpu/Makefile). This leg is why the knob is a + # knob: an option that has only ever been built in its default + # position is a claim, not a configuration. All-Dynamic bounds and + # splines because they are the cheapest policy that still runs the + # whole suite -- what this leg is asking about is the index axis, and + # cross-policy agreement is the other legs' job. Same 59,886 checks: + # the offset width is an internal representation choice and no test + # result may depend on it. + - name: clang-index64 + cxx: clang++ + apt: clang + boundflags: "-DFF_STATIC_BOUNDS=0" + splineflags: "-DFF_STATIC_SPLINES=0" + indexflags: "-DFF_INDEX32=0" + jobs: "2" # g++ at -O1, not the suite's -O3, and deliberately so: g++ was # measured needing ~6.3 GB RSS on reg_field.cpp at -O3, so two such # objects in parallel would sit right at the edge of a 16 GB runner. @@ -336,7 +353,8 @@ jobs: CXX="ccache ${{ matrix.cxx }}" \ ${{ matrix.cxxflags && format('CXXFLAGS="{0}"', matrix.cxxflags) || '' }} \ BOUNDFLAGS="${{ matrix.boundflags }}" \ - SPLINEFLAGS="${{ matrix.splineflags }}" + SPLINEFLAGS="${{ matrix.splineflags }}" \ + ${{ matrix.indexflags && format('INDEXFLAGS="{0}"', matrix.indexflags) || '' }} - name: Show ccache stats if: always() run: ccache -s @@ -575,12 +593,32 @@ jobs: # where posdef/resize/restrict/splinc were absent from MODULES and eleven # FF_CUDA:: symbols the hub calls unconditionally were undefined, with this # job green throughout. Do not reduce this job back to `make cuda`. + # + # Two legs, over the 32-bit index axis (INDEXFLAGS / FF_INDEX32, see + # src/lib-cuda/Makefile). `index32` is the shipping default and is the gate; + # `index64` builds the same library with the offset dispatch collapsed onto + # int64_t. The second leg exists because CUDA is where that axis is + # expensive -- it is x2 device code, x2 SASS and x2 ptxas memory -- and + # because a build option nobody has ever compiled in its non-default + # position is not an option. It is also strictly the *cheaper* of the two + # legs (roughly half the instantiations), and it makes the FFMEM table below + # a direct measurement of what the axis costs on this backend rather than an + # extrapolation from the host compiler. Both legs run the full compile + hub + # link + ldd check; neither is allowed to fail. build-cuda: - name: build-cuda (compile + link) + name: build-cuda (${{ matrix.name }}) needs: changes if: needs.changes.outputs.cuda == 'true' runs-on: ubuntu-latest timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - name: index32 + indexflags: "" + - name: index64 + indexflags: "-DFF_INDEX32=0" steps: - uses: actions/checkout@v5 - name: Install CUDA toolkit and clang @@ -607,6 +645,7 @@ jobs: run: | set -o pipefail make -C . cuda -j2 CXX=clang++ CXXFLAGS="-std=c++14 -O1" \ + ${{ matrix.indexflags && format('INDEXFLAGS="{0}"', matrix.indexflags) || '' }} \ NVCC="/usr/bin/time -f 'FFMEM %M kB %e s %C' nvcc" \ 2>&1 | tee /tmp/build-cuda.log - name: Link the hub against the CUDA backend @@ -634,6 +673,7 @@ jobs: # Cross-policy correctness is the test-cpu matrix's job. run: | make -C . lib -j2 USE_CUDA=1 CXX=clang++ \ + ${{ matrix.indexflags && format('INDEXFLAGS="{0}"', matrix.indexflags) || '' }} \ BOUNDFLAGS="-DFF_STATIC_BOUNDS=0" \ SPLINEFLAGS="-DFF_STATIC_SPLINES=0" - name: Confirm the hub resolves every backend symbol at load time @@ -692,7 +732,7 @@ jobs: | sed -E 's#^FFMEM ([0-9]+) kB +([0-9.]+) s .*-o ([^ ]+).*#\1 kB \2 s \3#' \ | sort -rn | head -40) echo "$table" - { echo '### Peak nvcc RSS per module'; echo; echo '```'; \ + { echo "### Peak nvcc RSS per module (${{ matrix.name }})"; echo; echo '```'; \ echo "$table"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" # No table at all means the compile died before any nvcc finished diff --git a/CLAUDE.md b/CLAUDE.md index 2da0ea4..fba78cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,9 +111,10 @@ the CPU path is the tested source of truth and CUDA is **compile+link only**. ## CI `.github/workflows/ci.yml`, path-filtered. `codespell` always; `test-cpu` (a -3-leg `BOUNDFLAGS`/`SPLINEFLAGS` matrix + a g++ leg), `sanitize` (ASan+UBSan) -and `tsan` on kernels/cpu/hub changes; `test-hub` on hub changes; `build-cuda` -and `compile-probe-cuda` on kernels/cuda changes. +3-leg `BOUNDFLAGS`/`SPLINEFLAGS` matrix + an `INDEXFLAGS` leg + a g++ leg), +`sanitize` (ASan+UBSan) and `tsan` on kernels/cpu/hub changes; `test-hub` on +hub changes; `build-cuda` (two legs, one per `FF_INDEX32` position) and +`compile-probe-cuda` on kernels/cuda changes. **The `tsan` leg is the only one that runs anything in parallel.** With the shipping `GRAIN_SIZE` (32768) every workload in `tests/lib-cpu/` is below the @@ -161,6 +162,18 @@ pushpull's fully-static order×bound compile is nightly `BOUNDFLAGS` / `SPLINEFLAGS`. These live **outside** `CXXFLAGS` on purpose so that a `CXXFLAGS=` override (as CUDA CI does, to force `-O1`) cannot silently drop the policy. +- **So is the 32-bit index axis**, via `INDEXFLAGS` / `FF_INDEX32` + (`core/dispatch.h`) — the third member of that family and the most expensive + of the three: every templated kernel is templated on `offset_t`, whose two + values are chosen per call by `canUse32BitIndexMath`, so the narrow path is + exactly ×2 instantiations of everything below the dispatch layer. + `INDEXFLAGS="-DFF_INDEX32=0"` collapses both arms onto `int64_t`. Same + outside-`CXXFLAGS` rule, and **the default (on) is set separately in + `src/lib-cpu/Makefile` and `src/lib-cuda/Makefile`** — that per-library + default is what makes it a per-backend option, so do not hoist it into + `make/common.mk`. The narrow path is an inherited ATen register-pressure + optimisation that has never been benchmarked here (no GPU in CI); the + default does not move without one. - **CUDA memory limits are measured, not guessed** — and the numbers that used to be recorded here were wrong. CUDA CI forces `-O1` and `-j2` because `ptxas` was OOM-killed at ~16 GB on `reg_field.cpp`, and `src/lib-cuda`'s diff --git a/MIGRATION.md b/MIGRATION.md index ea04782..a94a07a 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -197,6 +197,39 @@ are identical either way. Applied to the regularisers only so far — `resize`/`restrict`/`splinc`/`pushpull` still use the static `bound::utils`. See fastfields-lib#43. +## The 32-bit index axis (`FF_INDEX32`) + +The third build-time axis, and the widest. Every templated kernel below the +dispatch layer is templated on `offset_t`, and `offset_t` has exactly two +values, chosen per call by `canUse32BitIndexMath`: `int32_t` when every +operand's largest element offset fits in 32 bits, `int64_t` otherwise. So the +narrow path costs exactly **x2 instantiations of everything** underneath — +x2 device code, x2 SASS and x2 ptxas memory on the CUDA side. `core/autocast.h` +(`copy_if_needed` / `free_if_needed`) exists solely to feed that second +instantiation with narrowed shape and stride arrays. + +`FF_INDEX32=0` (`core/dispatch.h`) makes the narrow arm of every index +dispatch name `int64_t` too: the two arms become one instantiation, the axis +collapses, and the `canUse32BitIndexMath` probe folds away with it. Results are +identical; code size, compile cost and per-voxel speed move. + +Which position each backend takes is a **per-library** default — +`INDEXFLAGS` in `src/lib-cpu/Makefile` and `src/lib-cuda/Makefile`, exactly as +`BOUNDFLAGS`/`SPLINEFLAGS` are — so the CPU library can drop the axis while the +CUDA library keeps it, or the reverse. **Both default to on**, i.e. unchanged +behaviour, and CI builds both positions on both backends. + +The open question is whether the narrow path is worth its cost. It is an ATen +inheritance (register pressure) and has never been benchmarked in this project, +because there is no GPU in CI. Note the shape of the trade before flipping +anything: the backend that keeps the axis is CUDA, and CUDA is also the one +with no build headroom left (`reg_flow` at 12.98 GB of a 16 GB runner). On the +narrow path CUDA additionally pays a `cudaMallocHost`/`cudaFreeHost` per array +per call, while 365 of 365 relevant `impl/cuda` upload sites use the +*synchronous* `copyToDevice` — so the pinning has no async copy to enable and +may partly offset the register-pressure win. Measured numbers and the exact +benchmark that would settle it: fastfields-lib#94 and the follow-up issue. + ## Porting pattern (per module) Use `distance.{h,cpp}` at each level as the template. diff --git a/include/fastfields/api/cpu/pushpull_dispatch.h b/include/fastfields/api/cpu/pushpull_dispatch.h index a8790c0..0c973fd 100644 --- a/include/fastfields/api/cpu/pushpull_dispatch.h +++ b/include/fastfields/api/cpu/pushpull_dispatch.h @@ -37,9 +37,9 @@ typedef double reduce_t; #define FF_PP_DTYPE(D, I, B, FN, args...) \ switch (code) { \ case kDLFloat: switch (bits) { \ - case 32: return (use_32bits ? FN(args) \ + case 32: return (use_32bits ? FN(args) \ : FN(args)); \ - case 64: return (use_32bits ? FN(args) \ + case 64: return (use_32bits ? FN(args) \ : FN(args)); \ default: break; \ }; default: break; \ diff --git a/include/fastfields/api/cuda/pushpull_dispatch.h b/include/fastfields/api/cuda/pushpull_dispatch.h index a8117b1..a8fa74e 100644 --- a/include/fastfields/api/cuda/pushpull_dispatch.h +++ b/include/fastfields/api/cuda/pushpull_dispatch.h @@ -37,9 +37,9 @@ typedef double reduce_t; #define FF_PP_DTYPE(D, I, B, FN, args...) \ switch (code) { \ case kDLFloat: switch (bits) { \ - case 32: return (use_32bits ? FN(args) \ + case 32: return (use_32bits ? FN(args) \ : FN(args)); \ - case 64: return (use_32bits ? FN(args) \ + case 64: return (use_32bits ? FN(args) \ : FN(args)); \ default: break; \ }; default: break; \ diff --git a/include/fastfields/core/dispatch.h b/include/fastfields/core/dispatch.h index c164876..5875ee1 100644 --- a/include/fastfields/core/dispatch.h +++ b/include/fastfields/core/dispatch.h @@ -64,9 +64,92 @@ // at the call site instead of hiding in one file's private prologue. #define FF_CVOIDPTR_OR_NULL(x) (x.data ? FF_CVOIDPTR(x) : nullptr) +/*********************************************************************** + * THE 32-BIT INDEX AXIS * + ***********************************************************************/ + +/** + * `FF_INDEX32` -- compile-time policy for the 32-bit index (`offset_t`) axis, + * the exact analogue of `FF_STATIC_BOUNDS` (impl/kernels/bounds.h) and + * `FF_STATIC_SPLINES` (impl/kernels/spline.h) one axis further out. + * + * Every templated kernel below the dispatch layer is templated on `offset_t`, + * and `offset_t` has exactly two values chosen per call by + * `canUse32BitIndexMath`: `int32_t` when every operand's largest element + * offset fits in 32 bits, `int64_t` otherwise. The narrow one exists to cut + * register pressure -- an optimisation inherited from ATen -- and it costs + * exactly x2 instantiations of everything underneath, hence (on CUDA) x2 + * device code and x2 ptxas memory. Measured on `reg_flow`, the module that + * peaks at 12.98 GB of a 16 GB runner: dropping the axis is -50.3% + * instantiations and -44.6% peak RSS (fastfields-lib#94). + * + * FF_INDEX32=1 (default, and today's behaviour on both backends) + * both arms exist; `canUse32BitIndexMath` picks per call. + * FF_INDEX32=0 the narrow arm names `int64_t` too, so the two arms are + * the same instantiation and the axis collapses. The + * `canUse32BitIndexMath` call folds away with it. Results + * are identical either way; only code size, compile cost and + * per-voxel speed move. + * + * It is ONE switch rather than an `FF_INDEX32_CPU` / `FF_INDEX32_CUDA` pair on + * purpose. `core/` is backend-agnostic by contract -- `src/lib-cpu` (host + * compiler) and `src/lib-cuda` (nvcc) compile this same header -- so a + * per-backend *name* would force the header to branch on `__CUDACC__`, which + * puts the policy in the source instead of in the build and would have to be + * repeated by any third consumer. BOUNDFLAGS/SPLINEFLAGS already establish the + * alternative and this follows it exactly: one macro, and the *per-library + * Makefile* chooses the default. That is what makes the option per-backend, + * and it is why the CPU library can drop the axis while the CUDA library keeps + * it (or the reverse) without either one knowing about the other. See + * INDEXFLAGS in src/lib-cpu/Makefile and src/lib-cuda/Makefile. + * + * NB the axis is genuinely unbenchmarked here: there is no GPU in CI, so the + * register-pressure win the narrow path is meant to buy has never been + * measured in this project. The default therefore stays where it has always + * been (on); this is a knob, not a decision. + */ +#ifndef FF_INDEX32 +# define FF_INDEX32 1 +#endif + +#if (FF_INDEX32 != 0) && (FF_INDEX32 != 1) +# error "FF_INDEX32 must be 0 or 1 (see include/fastfields/core/dispatch.h)" +#endif + // Can this tensor's shape/stride arithmetic be narrowed to 32-bit offsets? -// See core/autocast.h. -#define FF_CANUSE32BITS(x) (canUse32BitIndexMath(x.ndim, x.shape, x.strides)) +// See core/autocast.h. With the axis off there is nothing to narrow to, so +// this is a compile-time `false` and the O(ndim) probe disappears from every +// dispatch site. +#if FF_INDEX32 +# define FF_CANUSE32BITS(x) (canUse32BitIndexMath(x.ndim, x.shape, x.strides)) +#else +# define FF_CANUSE32BITS(x) (false) +#endif + +FF_NAMESPACE_BEGIN(FF_NS) + +/** + * The offset type the *narrow* arm of every index dispatch names, i.e. the + * `offset_t` template argument on the `use_32bits ? f<..,off32_t>(a) + * : f<..,int64_t>(a)` sites. + * + * A typedef and not a macro, per the `FF_`-prefix rule in CLAUDE.md: a name in + * `ff::` is collision-safe with no prefix at all, and the dispatch sources are + * all inside `ff::cpu` / `ff::cuda`, so it resolves unqualified exactly where + * `int32_t` used to be spelled. + * + * When `FF_INDEX32` is 0 this is `int64_t` -- deliberately the same type as + * the wide arm, which is precisely how the axis collapses: both arms then name + * one instantiation, `use_32bits` is a compile-time `false`, and the ternary + * has nothing left to choose between. + */ +#if FF_INDEX32 +typedef int32_t off32_t; +#else +typedef int64_t off32_t; +#endif + +FF_NAMESPACE_END(FF_NS) /*********************************************************************** * CHECKS * diff --git a/src/lib-cpu/Makefile b/src/lib-cpu/Makefile index cd4f4a7..519af5a 100644 --- a/src/lib-cpu/Makefile +++ b/src/lib-cpu/Makefile @@ -31,6 +31,29 @@ BOUNDFLAGS ?= # Kept out of CXXFLAGS for the same reason as BOUNDFLAGS. SPLINEFLAGS ?= +# 32-bit index compile policy (see core/dispatch.h) -- the same idea as +# BOUNDFLAGS/SPLINEFLAGS, one axis further out again, and the most expensive of +# the three: EVERY templated kernel below this layer is templated on `offset_t`, +# whose two values (int32_t / int64_t) are chosen per call by +# canUse32BitIndexMath. So the narrow path costs exactly x2 instantiations of +# everything, measured at -50.3% instantiations / -44.6% peak compiler RSS on +# reg_flow when dropped (fastfields-lib#94). +# +# make INDEXFLAGS="-DFF_INDEX32=0" # one offset type (int64_t) everywhere +# +# Results are identical either way; code size, compile cost and per-voxel speed +# move. The narrow path is an inherited register-pressure optimisation that has +# never been benchmarked in this project, and it matters least here: the CPU +# backend is not the one whose build is memory-constrained, and it has 64-bit +# integer units. It stays ON by default all the same -- flipping it is a +# performance decision and wants a benchmark, not a build tidy-up. +# +# This is a SEPARATE default from src/lib-cuda/Makefile's, which is the whole +# point of putting it here rather than in make/common.mk: the two backends can +# take the axis independently. Kept out of CXXFLAGS for the same reason as +# BOUNDFLAGS/SPLINEFLAGS -- a CXXFLAGS= override must not silently drop policy. +INDEXFLAGS ?= -DFF_INDEX32=1 + # `test` target only: a sparser *default* BOUNDFLAGS/SPLINEFLAGS than the # library's (empty = fully static). pushpull's own dispatch used to carry a # second, hand-duplicated switch behind `-DFF_TEST_SPARSE` purely to keep a @@ -88,7 +111,7 @@ $(TARGET): $(OBJECTS) | $(LIBDIR) # header (this is a header-only codebase) rebuilds the affected library object # instead of leaving a stale binary. $(OBJDIR)/%.$(MOSUF): %.cpp | $(OBJDIR) - $(CXX) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INCLUDES) $(PICFLAG) -MMD -MP -c -o $@ $< + $(CXX) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INDEXFLAGS) $(INCLUDES) $(PICFLAG) -MMD -MP -c -o $@ $< ######################################################################## # Tests @@ -123,7 +146,7 @@ TESTDRVOBJ = $(patsubst $(TESTSRCDIR)/%.cpp,$(TESTOBJDIR)/%.$(MOSUF),$(TESTSRC)) .SECONDARY: $(TESTMODOBJ) $(TESTDRVOBJ) # -MMD -MP emit header dependency files (*.d) so header edits trigger rebuilds. -TESTCPPFLAGS = $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) -DFF_TEST_SPARSE $(INCLUDES) -MMD -MP +TESTCPPFLAGS = $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INDEXFLAGS) -DFF_TEST_SPARSE $(INCLUDES) -MMD -MP $(TESTOBJDIR): $(MKDIR) $(TESTOBJDIR) diff --git a/src/lib-cpu/distance.cpp b/src/lib-cpu/distance.cpp index 0636621..cadded7 100644 --- a/src/lib-cpu/distance.cpp +++ b/src/lib-cpu/distance.cpp @@ -26,12 +26,12 @@ FF_NAMESPACE_BEGIN(FF_DEVICE) case kDLFloat: switch (inp_out.dtype.bits) { \ case 32: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ case 64: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ default: break; \ @@ -133,12 +133,12 @@ void dt_l1( case kDLFloat: switch (loc.dtype.bits) { \ case 32: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ case 64: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ default: break; \ @@ -520,6 +520,9 @@ void dt_spline_gaussnewton( * MESH * ***********************************************************************/ +// NB the `int32_t` / `uint32_t` below are the FACE-INDEX dtype read from +// `faces.dtype`, not the offset type: the offset is `O`, supplied by +// DISPATCH_MESH_OFFSET. They are not part of the FF_INDEX32 axis. #define DISPATCH_MESH_SCALAR(D, S, O, func, args...) \ switch (code_index) { \ case kDLInt: switch (faces.dtype.bits) { \ @@ -542,7 +545,7 @@ void dt_spline_gaussnewton( } #define DISPATCH_MESH_OFFSET(D, S, func, args...) \ - if (use_32bits) DISPATCH_MESH_SCALAR(D, S, int32_t, func, args) \ + if (use_32bits) DISPATCH_MESH_SCALAR(D, S, off32_t, func, args) \ else DISPATCH_MESH_SCALAR(D, S, int64_t, func, args) \ #define DISPATCH_MESH_DIM(D, func, args...) \ diff --git a/src/lib-cpu/posdef.cpp b/src/lib-cpu/posdef.cpp index 02b9882..4d6184c 100644 --- a/src/lib-cpu/posdef.cpp +++ b/src/lib-cpu/posdef.cpp @@ -40,10 +40,10 @@ static inline int64_t channels_from_packed(int64_t CC) switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ case 64: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ default: break; \ }; \ @@ -69,10 +69,10 @@ static inline int64_t channels_from_packed(int64_t CC) switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ case 64: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ default: break; \ }; \ diff --git a/src/lib-cpu/reg_field.cpp b/src/lib-cpu/reg_field.cpp index ef2b97c..0918943 100644 --- a/src/lib-cpu/reg_field.cpp +++ b/src/lib-cpu/reg_field.cpp @@ -487,10 +487,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec(MV_ARGS) \ + ? _field_matvec(MV_ARGS) \ : _field_matvec(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec(MV_ARGS) \ + ? _field_matvec(MV_ARGS) \ : _field_matvec(MV_ARGS); \ default: break; \ } break; \ @@ -502,10 +502,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -517,10 +517,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -532,10 +532,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -547,10 +547,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -562,10 +562,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -577,10 +577,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -592,10 +592,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -607,10 +607,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -622,10 +622,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_relax(RX_ARGS) \ + ? _field_relax(RX_ARGS) \ : _field_relax(RX_ARGS); \ case 64: return use_32bits \ - ? _field_relax(RX_ARGS) \ + ? _field_relax(RX_ARGS) \ : _field_relax(RX_ARGS); \ default: break; \ } break; \ @@ -637,10 +637,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_rls(RLS_MV_ARGS) \ + ? _field_matvec_rls(RLS_MV_ARGS) \ : _field_matvec_rls(RLS_MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_rls(RLS_MV_ARGS) \ + ? _field_matvec_rls(RLS_MV_ARGS) \ : _field_matvec_rls(RLS_MV_ARGS); \ default: break; \ } break; \ @@ -652,10 +652,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag_rls(RLS_DG_ARGS) \ + ? _field_diag_rls(RLS_DG_ARGS) \ : _field_diag_rls(RLS_DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag_rls(RLS_DG_ARGS) \ + ? _field_diag_rls(RLS_DG_ARGS) \ : _field_diag_rls(RLS_DG_ARGS); \ default: break; \ } break; \ @@ -667,10 +667,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_relax_rls(RLS_RX_ARGS) \ + ? _field_relax_rls(RLS_RX_ARGS) \ : _field_relax_rls(RLS_RX_ARGS); \ case 64: return use_32bits \ - ? _field_relax_rls(RLS_RX_ARGS) \ + ? _field_relax_rls(RLS_RX_ARGS) \ : _field_relax_rls(RLS_RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cpu/reg_flow.cpp b/src/lib-cpu/reg_flow.cpp index eff02e9..841c664 100644 --- a/src/lib-cpu/reg_flow.cpp +++ b/src/lib-cpu/reg_flow.cpp @@ -469,10 +469,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec(MV_ARGS) \ + ? _flow_matvec(MV_ARGS) \ : _flow_matvec(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec(MV_ARGS) \ + ? _flow_matvec(MV_ARGS) \ : _flow_matvec(MV_ARGS); \ default: break; \ } break; \ @@ -484,10 +484,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -499,10 +499,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -514,10 +514,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -529,10 +529,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -544,10 +544,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -559,10 +559,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -574,10 +574,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -589,10 +589,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -604,10 +604,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_relax(RX_ARGS) \ + ? _flow_relax(RX_ARGS) \ : _flow_relax(RX_ARGS); \ case 64: return use_32bits \ - ? _flow_relax(RX_ARGS) \ + ? _flow_relax(RX_ARGS) \ : _flow_relax(RX_ARGS); \ default: break; \ } break; \ @@ -619,10 +619,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_rls(RLS_MV_ARGS) \ + ? _flow_matvec_rls(RLS_MV_ARGS) \ : _flow_matvec_rls(RLS_MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_rls(RLS_MV_ARGS) \ + ? _flow_matvec_rls(RLS_MV_ARGS) \ : _flow_matvec_rls(RLS_MV_ARGS); \ default: break; \ } break; \ @@ -634,10 +634,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag_rls(RLS_DG_ARGS) \ + ? _flow_diag_rls(RLS_DG_ARGS) \ : _flow_diag_rls(RLS_DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag_rls(RLS_DG_ARGS) \ + ? _flow_diag_rls(RLS_DG_ARGS) \ : _flow_diag_rls(RLS_DG_ARGS); \ default: break; \ } break; \ @@ -649,10 +649,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_relax_rls(RLS_RX_ARGS) \ + ? _flow_relax_rls(RLS_RX_ARGS) \ : _flow_relax_rls(RLS_RX_ARGS); \ case 64: return use_32bits \ - ? _flow_relax_rls(RLS_RX_ARGS) \ + ? _flow_relax_rls(RLS_RX_ARGS) \ : _flow_relax_rls(RLS_RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cpu/resize.cpp b/src/lib-cpu/resize.cpp index 4b4d80d..7fbb7d3 100644 --- a/src/lib-cpu/resize.cpp +++ b/src/lib-cpu/resize.cpp @@ -63,10 +63,10 @@ inline void _resample( switch (code) { \ case kDLFloat: switch (inp.dtype.bits) { \ case 32: return ( \ - use_32bits ? _resample(args) \ + use_32bits ? _resample(args) \ : _resample(args)); \ case 64: return ( \ - use_32bits ? _resample(args) \ + use_32bits ? _resample(args) \ : _resample(args)); \ default: break; \ }; \ diff --git a/src/lib-cpu/restrict.cpp b/src/lib-cpu/restrict.cpp index b78e947..749d475 100644 --- a/src/lib-cpu/restrict.cpp +++ b/src/lib-cpu/restrict.cpp @@ -63,10 +63,10 @@ inline void _restriction( switch (code) { \ case kDLFloat: switch (inp.dtype.bits) { \ case 32: return ( \ - use_32bits ? _restriction(args) \ + use_32bits ? _restriction(args) \ : _restriction(args)); \ case 64: return ( \ - use_32bits ? _restriction(args) \ + use_32bits ? _restriction(args) \ : _restriction(args)); \ default: break; \ }; \ diff --git a/src/lib-cpu/solve_field.cpp b/src/lib-cpu/solve_field.cpp index 13e0912..3a533fd 100644 --- a/src/lib-cpu/solve_field.cpp +++ b/src/lib-cpu/solve_field.cpp @@ -125,9 +125,9 @@ inline void _axpby_( #define SOLVE_DT_SWITCH(CALL) \ switch (static_cast(x.dtype.code)) { \ case kDLFloat: switch (x.dtype.bits) { \ - case 32: return use_32bits ? CALL(float, int32_t) \ + case 32: return use_32bits ? CALL(float, off32_t) \ : CALL(float, int64_t); \ - case 64: return use_32bits ? CALL(double, int32_t) \ + case 64: return use_32bits ? CALL(double, off32_t) \ : CALL(double, int64_t); \ default: break; \ } break; \ diff --git a/src/lib-cpu/splinc.cpp b/src/lib-cpu/splinc.cpp index a418562..641e104 100644 --- a/src/lib-cpu/splinc.cpp +++ b/src/lib-cpu/splinc.cpp @@ -108,10 +108,10 @@ inline void _splinc( switch (code) { \ case kDLFloat: switch (inp_out.dtype.bits) { \ case 32: \ - if (use_32bits) DISPATCH_SPLINC_NPOLES(float, int32_t, args) \ + if (use_32bits) DISPATCH_SPLINC_NPOLES(float, off32_t, args) \ else DISPATCH_SPLINC_NPOLES(float, int64_t, args) \ case 64: \ - if (use_32bits) DISPATCH_SPLINC_NPOLES(double, int32_t, args) \ + if (use_32bits) DISPATCH_SPLINC_NPOLES(double, off32_t, args) \ else DISPATCH_SPLINC_NPOLES(double, int64_t, args) \ default: break; \ }; \ diff --git a/src/lib-cuda/Makefile b/src/lib-cuda/Makefile index de2608c..57d11f9 100644 --- a/src/lib-cuda/Makefile +++ b/src/lib-cuda/Makefile @@ -62,6 +62,38 @@ SPLINEFLAGS ?= -DFF_STATIC_SPLINES=0 \ -DFF_STATIC_SPLINE_NEAREST=1 -DFF_STATIC_SPLINE_LINEAR=1 \ -DFF_STATIC_SPLINE_QUADRATIC=1 -DFF_STATIC_SPLINE_CUBIC=1 +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# 32-bit index compile policy (see core/dispatch.h) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# The same idea as BOUNDFLAGS/SPLINEFLAGS, one axis further out again, and the +# most expensive of the three. EVERY templated kernel below this layer is +# templated on `offset_t`, whose two values (int32_t / int64_t) are chosen per +# call by canUse32BitIndexMath, so the narrow path costs exactly x2 +# instantiations of everything -- which here means x2 device code, x2 SASS and +# x2 ptxas memory: +# +# make INDEXFLAGS="-DFF_INDEX32=0" # one offset type (int64_t) everywhere +# +# Measured on reg_flow, the module in the table below that peaks at 12.98 GB of +# a 16 GB runner: dropping the axis is -50.3% instantiations and -44.6% peak +# compiler RSS (fastfields-lib#94, measured with the host compiler; the CUDA +# figures are printed per module by the `build-cuda` CI job, which builds both +# positions). +# +# It is nevertheless ON by default, which is a deliberate choice and not +# inertia. Narrow indices are a register-pressure optimisation inherited from +# ATen and there is more reason to expect them to pay on a GPU than on a CPU +# -- but nobody has ever measured it here, because there is no GPU in CI. The +# honest reading of the table below is therefore uncomfortable: this backend is +# the one with no headroom left, and it is also the one keeping the axis. If a +# benchmark on real hardware ever shows the narrow path buys nothing (or less +# than the per-call cudaMallocHost in core/autocast.h costs it), flipping this +# default halves the heaviest compile in the project. Until then the knob +# exists, both positions are built by CI, and the default does not move. +# +# Kept out of CXXFLAGS for the same reason as BOUNDFLAGS/SPLINEFLAGS. +INDEXFLAGS ?= -DFF_INDEX32=1 + # The regularisers are split into a core module and an `_rls` module (the # reweighted-least-squares ops) -- a CUDA-only split that lib-cpu does not need. # It is a memory/parallelism measure, not a correctness one: the @@ -181,7 +213,7 @@ $(TARGET): $(OBJECTS) | $(LIBDIR) # the stale .d is removed -- the tradeoff for having any dependency tracking # here at all. lib-cpu, compiled by the host compiler directly, keeps -MP. $(OBJDIR)/%.$(MOSUF): %.cpp | $(OBJDIR) - $(NVCC) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INCLUDES) \ + $(NVCC) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INDEXFLAGS) $(INCLUDES) \ -x cu -Xcompiler -fPIC -MMD -MF $(@:.$(MOSUF)=.d) -c -o $@ $< # Compile-only probe for the CUDA impl layer: there is no GPU in CI, so the @@ -190,7 +222,7 @@ PROBESRC = $(wildcard $(ROOTDIR)/tests/impl-cuda/*.cu) test-probe: | $(TESTDIR) @ for p in $(PROBESRC); do \ echo "compiling $$p"; \ - $(NVCC) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INCLUDES) \ + $(NVCC) $(CXXFLAGS) $(BOUNDFLAGS) $(SPLINEFLAGS) $(INDEXFLAGS) $(INCLUDES) \ -c -o $(TESTDIR)/$$(basename $$p .cu).$(MOSUF) $$p || exit 1; \ done diff --git a/src/lib-cuda/distance.cpp b/src/lib-cuda/distance.cpp index 21a26ea..aff83e2 100644 --- a/src/lib-cuda/distance.cpp +++ b/src/lib-cuda/distance.cpp @@ -26,12 +26,12 @@ FF_NAMESPACE_BEGIN(FF_DEVICE) case kDLFloat: switch (inp_out.dtype.bits) { \ case 32: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ case 64: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ default: break; \ @@ -137,12 +137,12 @@ void dt_l1( case kDLFloat: switch (loc.dtype.bits) { \ case 32: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ case 64: return ( \ use_32bits \ - ? func(args) \ + ? func(args) \ : func(args) \ ); \ default: break; \ @@ -524,6 +524,9 @@ void dt_spline_gaussnewton( * MESH * ***********************************************************************/ +// NB the `int32_t` / `uint32_t` below are the FACE-INDEX dtype read from +// `faces.dtype`, not the offset type: the offset is `O`, supplied by +// DISPATCH_MESH_OFFSET. They are not part of the FF_INDEX32 axis. #define DISPATCH_MESH_SCALAR(D, S, O, func, args...) \ switch (code_index) { \ case kDLInt: switch (loc.dtype.bits) { \ @@ -546,7 +549,7 @@ void dt_spline_gaussnewton( } #define DISPATCH_MESH_OFFSET(D, S, func, args...) \ - if (use_32bits) DISPATCH_MESH_SCALAR(D, S, int32_t, func, args) \ + if (use_32bits) DISPATCH_MESH_SCALAR(D, S, off32_t, func, args) \ else DISPATCH_MESH_SCALAR(D, S, int64_t, func, args) \ #define DISPATCH_MESH_DIM(D, func, args...) \ diff --git a/src/lib-cuda/posdef.cpp b/src/lib-cuda/posdef.cpp index 77fa26a..52f4f03 100644 --- a/src/lib-cuda/posdef.cpp +++ b/src/lib-cuda/posdef.cpp @@ -40,10 +40,10 @@ static inline int64_t channels_from_packed(int64_t CC) switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ case 64: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ default: break; \ }; \ @@ -69,10 +69,10 @@ static inline int64_t channels_from_packed(int64_t CC) switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ case 64: return ( \ - use_32bits ? func(args) \ + use_32bits ? func(args) \ : func(args)); \ default: break; \ }; \ diff --git a/src/lib-cuda/reg_field.cpp b/src/lib-cuda/reg_field.cpp index aad45d9..fd1e2bb 100644 --- a/src/lib-cuda/reg_field.cpp +++ b/src/lib-cuda/reg_field.cpp @@ -282,10 +282,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec(MV_ARGS) \ + ? _field_matvec(MV_ARGS) \ : _field_matvec(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec(MV_ARGS) \ + ? _field_matvec(MV_ARGS) \ : _field_matvec(MV_ARGS); \ default: break; \ } break; \ @@ -297,10 +297,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -312,10 +312,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_acc(MV_ARGS) \ + ? _field_matvec_acc(MV_ARGS) \ : _field_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -327,10 +327,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -342,10 +342,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -357,10 +357,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _field_kernel(KN_ARGS) \ + ? _field_kernel(KN_ARGS) \ : _field_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -372,10 +372,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -387,10 +387,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -402,10 +402,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag(DG_ARGS) \ + ? _field_diag(DG_ARGS) \ : _field_diag(DG_ARGS); \ default: break; \ } break; \ @@ -417,10 +417,10 @@ inline void _field_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_relax(RX_ARGS) \ + ? _field_relax(RX_ARGS) \ : _field_relax(RX_ARGS); \ case 64: return use_32bits \ - ? _field_relax(RX_ARGS) \ + ? _field_relax(RX_ARGS) \ : _field_relax(RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cuda/reg_field_rls.cpp b/src/lib-cuda/reg_field_rls.cpp index 8345574..c7816c7 100644 --- a/src/lib-cuda/reg_field_rls.cpp +++ b/src/lib-cuda/reg_field_rls.cpp @@ -251,10 +251,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_matvec_rls(RLS_MV_ARGS) \ + ? _field_matvec_rls(RLS_MV_ARGS) \ : _field_matvec_rls(RLS_MV_ARGS); \ case 64: return use_32bits \ - ? _field_matvec_rls(RLS_MV_ARGS) \ + ? _field_matvec_rls(RLS_MV_ARGS) \ : _field_matvec_rls(RLS_MV_ARGS); \ default: break; \ } break; \ @@ -266,10 +266,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_diag_rls(RLS_DG_ARGS) \ + ? _field_diag_rls(RLS_DG_ARGS) \ : _field_diag_rls(RLS_DG_ARGS); \ case 64: return use_32bits \ - ? _field_diag_rls(RLS_DG_ARGS) \ + ? _field_diag_rls(RLS_DG_ARGS) \ : _field_diag_rls(RLS_DG_ARGS); \ default: break; \ } break; \ @@ -281,10 +281,10 @@ inline void _field_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _field_relax_rls(RLS_RX_ARGS) \ + ? _field_relax_rls(RLS_RX_ARGS) \ : _field_relax_rls(RLS_RX_ARGS); \ case 64: return use_32bits \ - ? _field_relax_rls(RLS_RX_ARGS) \ + ? _field_relax_rls(RLS_RX_ARGS) \ : _field_relax_rls(RLS_RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cuda/reg_flow.cpp b/src/lib-cuda/reg_flow.cpp index fa7a2b2..b48818c 100644 --- a/src/lib-cuda/reg_flow.cpp +++ b/src/lib-cuda/reg_flow.cpp @@ -304,10 +304,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec(MV_ARGS) \ + ? _flow_matvec(MV_ARGS) \ : _flow_matvec(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec(MV_ARGS) \ + ? _flow_matvec(MV_ARGS) \ : _flow_matvec(MV_ARGS); \ default: break; \ } break; \ @@ -319,10 +319,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -334,10 +334,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_acc(MV_ARGS) \ + ? _flow_matvec_acc(MV_ARGS) \ : _flow_matvec_acc(MV_ARGS); \ default: break; \ } break; \ @@ -349,10 +349,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -364,10 +364,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -379,10 +379,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag(DG_ARGS) \ + ? _flow_diag(DG_ARGS) \ : _flow_diag(DG_ARGS); \ default: break; \ } break; \ @@ -394,10 +394,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -409,10 +409,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -424,10 +424,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ case 64: return use_32bits \ - ? _flow_kernel(KN_ARGS) \ + ? _flow_kernel(KN_ARGS) \ : _flow_kernel(KN_ARGS); \ default: break; \ } break; \ @@ -439,10 +439,10 @@ inline void _flow_relax( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_relax(RX_ARGS) \ + ? _flow_relax(RX_ARGS) \ : _flow_relax(RX_ARGS); \ case 64: return use_32bits \ - ? _flow_relax(RX_ARGS) \ + ? _flow_relax(RX_ARGS) \ : _flow_relax(RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cuda/reg_flow_rls.cpp b/src/lib-cuda/reg_flow_rls.cpp index 8da7e4a..4ef0a81 100644 --- a/src/lib-cuda/reg_flow_rls.cpp +++ b/src/lib-cuda/reg_flow_rls.cpp @@ -207,10 +207,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_matvec_rls(RLS_MV_ARGS) \ + ? _flow_matvec_rls(RLS_MV_ARGS) \ : _flow_matvec_rls(RLS_MV_ARGS); \ case 64: return use_32bits \ - ? _flow_matvec_rls(RLS_MV_ARGS) \ + ? _flow_matvec_rls(RLS_MV_ARGS) \ : _flow_matvec_rls(RLS_MV_ARGS); \ default: break; \ } break; \ @@ -222,10 +222,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_diag_rls(RLS_DG_ARGS) \ + ? _flow_diag_rls(RLS_DG_ARGS) \ : _flow_diag_rls(RLS_DG_ARGS); \ case 64: return use_32bits \ - ? _flow_diag_rls(RLS_DG_ARGS) \ + ? _flow_diag_rls(RLS_DG_ARGS) \ : _flow_diag_rls(RLS_DG_ARGS); \ default: break; \ } break; \ @@ -237,10 +237,10 @@ inline void _flow_relax_rls( switch (code) { \ case kDLFloat: switch (bits) { \ case 32: return use_32bits \ - ? _flow_relax_rls(RLS_RX_ARGS) \ + ? _flow_relax_rls(RLS_RX_ARGS) \ : _flow_relax_rls(RLS_RX_ARGS); \ case 64: return use_32bits \ - ? _flow_relax_rls(RLS_RX_ARGS) \ + ? _flow_relax_rls(RLS_RX_ARGS) \ : _flow_relax_rls(RLS_RX_ARGS); \ default: break; \ } break; \ diff --git a/src/lib-cuda/resize.cpp b/src/lib-cuda/resize.cpp index a9cae09..3cf9a7c 100644 --- a/src/lib-cuda/resize.cpp +++ b/src/lib-cuda/resize.cpp @@ -62,10 +62,10 @@ inline void _resample( switch (code) { \ case kDLFloat: switch (inp.dtype.bits) { \ case 32: return ( \ - use_32bits ? _resample(args) \ + use_32bits ? _resample(args) \ : _resample(args)); \ case 64: return ( \ - use_32bits ? _resample(args) \ + use_32bits ? _resample(args) \ : _resample(args)); \ default: break; \ }; \ diff --git a/src/lib-cuda/restrict.cpp b/src/lib-cuda/restrict.cpp index a748982..71a8080 100644 --- a/src/lib-cuda/restrict.cpp +++ b/src/lib-cuda/restrict.cpp @@ -62,10 +62,10 @@ inline void _restriction( switch (code) { \ case kDLFloat: switch (inp.dtype.bits) { \ case 32: return ( \ - use_32bits ? _restriction(args) \ + use_32bits ? _restriction(args) \ : _restriction(args)); \ case 64: return ( \ - use_32bits ? _restriction(args) \ + use_32bits ? _restriction(args) \ : _restriction(args)); \ default: break; \ }; \ diff --git a/src/lib-cuda/splinc.cpp b/src/lib-cuda/splinc.cpp index 47529c0..a70d232 100644 --- a/src/lib-cuda/splinc.cpp +++ b/src/lib-cuda/splinc.cpp @@ -113,10 +113,10 @@ inline void _splinc( switch (code) { \ case kDLFloat: switch (inp_out.dtype.bits) { \ case 32: \ - if (use_32bits) DISPATCH_SPLINC_NPOLES(float, int32_t, args) \ + if (use_32bits) DISPATCH_SPLINC_NPOLES(float, off32_t, args) \ else DISPATCH_SPLINC_NPOLES(float, int64_t, args) \ case 64: \ - if (use_32bits) DISPATCH_SPLINC_NPOLES(double, int32_t, args) \ + if (use_32bits) DISPATCH_SPLINC_NPOLES(double, off32_t, args) \ else DISPATCH_SPLINC_NPOLES(double, int64_t, args) \ default: break; \ }; \