') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); improves dataloader performance by giovp · Pull Request #687 · scverse/spatialdata · GitHub
Skip to content

improves dataloader performance - #687

Merged
LucaMarconato merged 29 commits into
mainfrom
giovp/dataloader3
May 21, 2026
Merged

improves dataloader performance#687
LucaMarconato merged 29 commits into
mainfrom
giovp/dataloader3

Conversation

@giovp

@giovpgiovp commented Aug 21, 2024•
edited
Loading

Copy link
Copy Markdown
Member

replaces #565#622

iteration over 20 batches, single worker

new implementation
image

main
image

one annoying thing is that the "apply" method of the dataframe to get the bounding box selection is quite slow.

@giovp

giovp commented Sep 3, 2024•
edited
Loading

Copy link
Copy Markdown
MemberAuthor

quick push to try #699 where tiling is vectorized, removed the need for pandas.DataFrame.apply. Quite big speedup

image

This was referenced Sep 3, 2024
Merged
Closed
Closed
LucaMarconatoand others added 3 commits May 15, 2026 23:16
@LucaMarconato
Merge branch 'main' into giovp/dataloader3
df21672
@LucaMarconato@claude
Fix rasterize path and bugs in PR#687dataloader; add benchmark
e87d318
**Bugs fixed in datasets.py:**
- rasterize=True path was broken: __getitem__ always called image.sel() regardless
of rasterize flag, bypassing rasterize_fn entirely. Fixed by storing self._rasterize
and branching in __getitem__.
- ad.concat(*tables_l) unpacked the list as positional args, failing with >1 region.
Fixed to ad.concat(tables_l).
- Vectorized selection pre-computation was always run even for rasterize=True where
it is unused. Fixed by guarding with `if not rasterize`.
- Removed stale commented-out pandas.apply fallback code.
**Fixes in _utils.py:**
- Removed redundant nopython=True from @nb.njit (njit implies nopython=True,
and the argument caused a RuntimeWarning).
- Replaced invalid nb.types.Array[nb.float64, nb.float64] annotations with np.ndarray.
**Fixes in spatial_query.py:**
- Restored BoundingBoxRequest validation that was commented out. The validator's
__post_init__ already handles both 1-D (single box) and 2-D (multi-box) arrays.
**Benchmark (benchmark_dataloader.py):**
Synthetic 2048x2048 image, 500 circle regions (32 px radius), 3-channel.
Phase main PR (fixed) speedup
init ~162 ms ~20 ms ~8x
fetch 500 ~618 ms ~118 ms ~5x
per-tile ~1237 us ~235 us ~5x
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
@LucaMarconato
add asv benchmark for dataloader performance
a51cb95
@LucaMarconato

LucaMarconato commented May 21, 2026

Copy link
Copy Markdown
Member

I picked this up, performance indeed improves significantly with the new vectorized bounding box approach. Thanks @giovp

asv benchmarks result:

| Change | Before [accf496c] <main> | After [e87d3183] <giovp/dataloader3> | Ratio | Benchmark (Parameter) |
|----------|----------------------------|----------------------------------------|---------|------------------------------------------------|
| - | 618±20ms | 123±1ms | 0.2 | benchmark_dataloader.TimeDataloader.time_fetch |
| - | 169±20ms | 18.7±0.1ms | 0.11 | benchmark_dataloader.TimeDataloader.time_init |

@codecov

codecovBot commented May 21, 2026•
edited
Loading

Copy link
Copy Markdown

Codecov Report

✅ All modified and coverable lines are covered by tests.
✅ Project coverage is 92.55%. Comparing base (accf496) to head (a51cb95).
⚠️ Report is 24 commits behind head on main.

Additional details and impacted files
@@ Coverage Diff @@## main #687 +/- ##
==========================================
+ Coverage 92.28% 92.55% +0.26% 
==========================================
Files 51 51 Lines 7804 7763 -41 ==========================================
- Hits 7202 7185 -17 + Misses 602 578 -24
Files with missing linesCoverage Δ
src/spatialdata/_core/query/_utils.py93.26% <100.00%> (ø)
src/spatialdata/_core/query/spatial_query.py95.52% <ø> (ø)
src/spatialdata/dataloader/datasets.py90.52% <100.00%> (+0.27%)⬆️

... and 3 files with indirect coverage changes

🚀 New features to boost your workflow:
  • ❄️ Test Analytics: Detect flaky tests, report on failures, and find test suite problems.

@LucaMarconato
LucaMarconato merged commit 9cd4eb7 into mainMay 21, 2026
9 checks passed
@LucaMarconato
LucaMarconato deleted the giovp/dataloader3 branch May 21, 2026 12:06
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

release-added

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

2 participants

@giovp@LucaMarconato