') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); GeoParquet support for shapes; refactor format by LucaMarconato · Pull Request #542 · scverse/spatialdata · GitHub
Skip to content

GeoParquet support for shapes; refactor format - #542

Merged
LucaMarconato merged 36 commits into
mainfrom
prototype/geoparquet
Jul 24, 2024
Merged

GeoParquet support for shapes; refactor format#542
LucaMarconato merged 36 commits into
mainfrom
prototype/geoparquet

Conversation

@LucaMarconato

@LucaMarconatoLucaMarconato commented Apr 8, 2024

Copy link
Copy Markdown
Member

This PR introduces the use of GeoParquet to save shapes and replaces the old ragged array-based storage.

I also fixes the issue that only the geometry column was saved, now all the columns of the dataframe are saved.
Closes#311

Notes:

  • No performance change is observed for writing (they are both binary formats). But GeoParquet improves interoperability.
  • The PR is backward compatible: all previous datasets can be read, and the user can choose if writing the shapes with the latest format or with a previous format. This choice can be performed per-element.
  • Tests are added to ensure the compatibility between the shapes "v01" and "v02"

These last two points required a overhaul of format.py and related code, code that now is more capable and robust. A few notes on this:

  • For raster types we didn't introduce a "spatialdata format" because we were optimistic that transformations etc would have landed in NGFF faster; since they are not there yet, I believe that we need versioning also for raster types (which I now implemented).
  • To avoid confusion between the version property, that always gives 0.4 and the spatialdata version, I renamed the version property to spatialdata_format_version. When writing points, shapes and tables, we need just the second one (because NGFF doesn't support them); instead, while writing labels and images we need both: ome-zarr-py needs to know that we are working with 0.4, and spatialdata needs to know which spatial data format for raster we are using to augment the NGFF storage.
  • I have introduced a function _parse_version() that retrieves the spatialdata zarr version for a given element from its Zarr store.
  • The function _parse_version() less clean that I hoped, this because of the versioning was done in a heterogenous way for the elements:
    • Shapes and points saved and still save the version under .attrs['spatialdata_attrs']['version']
    • Tables saved the version under .attrs['version'], they still save the version there for backward compatibility even if switching to .attrs['spatialdata_attrs']['version'] would be cleaner.
    • Images and labels didn't save the version at all, now they save it under .attrs['spatialdata_attrs']['version']. If _parse_version() can't find the version for raster types, it is assumed that the spatialdata RasterFormatV01 is used.

LucaMarconatoand others added 27 commits March 21, 2024 23:27
Co-authored-by: Kevin Yamauchi <kevin.yamauchi@gmail.com>
* test read write on disk
* [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
* improved tests for workarounds for incremental io
* fixed tests
* improved comment
---------
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Luca Marconato <m.lucalmer@gmail.com>
Co-authored-by: Giovanni Palla <25887487+giovp@users.noreply.github.com>
@LucaMarconato
LucaMarconato marked this pull request as ready for review July 16, 2024 15:24
@LucaMarconatoLucaMarconato changed the title Prototype/geoparquetGeoParquet support for shapes; refactor formatJul 16, 2024
@LucaMarconato

Copy link
Copy Markdown
MemberAuthor

Ready for review, thanks 😊

@LucaMarconato
LucaMarconato requested a review from giovpJuly 16, 2024 15:25
@codecov

codecovBot commented Jul 16, 2024

Copy link
Copy Markdown

Codecov Report

Attention: Patch coverage is 89.34010% with 21 lines in your changes missing coverage. Please review.

Project coverage is 89.94%. Comparing base (6d8aeb8) to head (32c65af).
Report is 41 commits behind head on main.

Files with missing linesPatch %Lines
src/spatialdata/_io/format.py80.59%13 Missing ⚠️
src/spatialdata/_io/io_shapes.py92.30%3 Missing ⚠️
src/spatialdata/_io/io_table.py90.62%3 Missing ⚠️
src/spatialdata/_core/spatialdata.py88.88%1 Missing ⚠️
src/spatialdata/_io/io_zarr.py75.00%1 Missing ⚠️
Additional details and impacted files
@@ Coverage Diff @@## main #542 +/- ##
==========================================
- Coverage 90.04% 89.94% -0.10% 
==========================================
Files 44 44 Lines 6670 6755 +85 ==========================================
+ Hits 6006 6076 +70 - Misses 664 679 +15 
Files with missing linesCoverage Δ
src/spatialdata/_io/__init__.py100.00% <100.00%> (ø)
src/spatialdata/_io/_utils.py87.07% <100.00%> (+0.18%)⬆️
src/spatialdata/_io/io_points.py100.00% <100.00%> (ø)
src/spatialdata/_io/io_raster.py96.00% <100.00%> (+0.34%)⬆️
src/spatialdata/models/models.py87.66% <100.00%> (+0.05%)⬆️
src/spatialdata/_core/spatialdata.py90.89% <88.88%> (-0.09%)⬇️
src/spatialdata/_io/io_zarr.py88.50% <75.00%> (+0.13%)⬆️
src/spatialdata/_io/io_shapes.py95.23% <92.30%> (-2.73%)⬇️
src/spatialdata/_io/io_table.py93.87% <90.62%> (-6.13%)⬇️
src/spatialdata/_io/format.py84.35% <80.59%> (-5.12%)⬇️

@LucaMarconato

Copy link
Copy Markdown
MemberAuthor

Todo:

  • update changelog

@LucaMarconato

Copy link
Copy Markdown
MemberAuthor

The docs started failing in a different PR, I'd address that separately.

@giovpgiovp left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks great! Much needed update, small comments

Comment threadsrc/spatialdata/_core/spatialdata.py Outdated
Comment threadsrc/spatialdata/_core/spatialdata.py Outdated
Comment threadsrc/spatialdata/_io/format.py
Comment threadsrc/spatialdata/_io/format.py
Comment threadsrc/spatialdata/_io/io_shapes.py
Co-authored-by: Giovanni Palla <25887487+giovp@users.noreply.github.com>
@LucaMarconato

LucaMarconato commented Jul 22, 2024

Copy link
Copy Markdown
MemberAuthor

As you prompted, I have updated the docs. Now the API page from the docs has this in the end (left panel in the figure).

Please noticed that the first class is non-clickable (pink). This because we define the SpatialDataFormat class simply as

classSpatialDataFormat(CurrentFormat):
pass

But if we had it to the docs, since we inherit from ome_zarr.format.CurrentFormat, the user would get this confusing page (right panel in the figure).

docs

@LucaMarconato

Copy link
Copy Markdown
MemberAuthor

Ready to merge, I'll fix the (unrelated) problem with the docs in a new PR.

@LucaMarconato
LucaMarconato merged commit 744093d into mainJul 24, 2024
@LucaMarconato
LucaMarconato deleted the prototype/geoparquet branch July 24, 2024 12:39
@LucaMarconatoLucaMarconato mentioned this pull request Jul 24, 2024
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

Option to save columns in shapes layer

3 participants

@LucaMarconato@giovp@ArneDefauw