fix: point table validation to public sanitizer

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None:

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
  PLATFORM: ${{ matrix.os }}
  DISPLAY: :42
  run: |
- uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+ uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
  - name: Upload coverage to Codecov
  uses: codecov/codecov-action@v6
  with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
  skip: []
 repos:
  - repo: https://github.com/rbubley/mirrors-prettier
- rev: v3.9.4
+ rev: v3.9.6
  hooks:
  - id: prettier
  exclude: ^.github/workflows/test.yaml
  - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v2.1.0
+ rev: v2.3.1
  hooks:
  - id: mypy
  additional_dependencies: [numpy, types-requests]
  exclude: tests/|docs/
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.3
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
  "spatial_image>=1.2.3",
  "scikit-image",
  "scipy!=1.17.0",
+ "scverse-misc[datasets]>=0.1.0",
  "typing_extensions>=4.8.0",
  "universal_pathlib>=0.2.6",
  "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
  "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+ "network: marks tests that require network access; skipped by default, run with '--run-network'",
  "gpu: run test on GPU using CuPY.",
  "array_api: used by anndata.tests.helpers, not us",
  "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+ "0.13-2",
+ "0.13-3",
+ "0.12-2"
+ # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+ # every option where the if-condition is True gets included
+
+ # anndata 0.13
+ {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+ # anndata 0.12
+ {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+ # pandas 2
+ {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+ # pandas 3
+ {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
  - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm
  slot.
 
- The table can annotate regions (shapesor labels) and can be used to store additional information.
+ The table can annotate regions (shapes or labels) and can be used to store additional information.
  Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
  annotation directly.
  """
@@ -1114,6 +1114,7 @@ def write(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, convert string columns of all tables to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
  """
  from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
  from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
  parsed_formats=parsed,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
 
  if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
  parsed_formats: dict[str, SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
  group=element_type_group,
  name=element_name,
  element_format=parsed_formats["tables"],
+ convert_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  else:
  raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
  sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
  shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+ convert_table_strings_to_categoricals: bool = False,
  ) -> None:
  """
  Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
  shapes_geometry_encoding
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
  for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-raster_compressor
+ raster_compressor
  A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
  compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
  supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
  ordered for more efficient compression.
+ convert_table_strings_to_categoricals
+ If True, and if element to be written is a table, convert string columns to categoricals before writing.
+ Note that this will have a side effect of modifying string columns into categoricals in place.
 
  Notes
  -----
@@ -1332,6 +1343,7 @@ def write_element(
  sdata_formats=sdata_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  return
 
@@ -1368,6 +1380,7 @@ def write_element(
  parsed_formats=parsed_formats,
  shapes_geometry_encoding=shapes_geometry_encoding,
  raster_compressor=raster_compressor,
+ convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
  )
  # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
  if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
  # Exceptions were collected that we want to raise as a combined validation error.
  if self._collector.errors:
  raise ValidationError(
- title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+ title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
  errors=self._collector.errors,
  )
  return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+ """Exception raised when an unknown element format is encountered."""
+
+ def __init__(self, element_type: str, version_encountered: Format):
+ self.element_type = element_type
+ self.version_encountered = version_encountered
+ self.message = (
+ f"Encountered unknown element format version "
+ f"`{self.version_encountered}` for element of type `{self.element_type}`"
+ )
+ super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+ """Warning raised when writing to zarr v2 format."""
+
+ message = (
+ "Writing to zarr v2 format is currently deprecated in spatialdata "
+ "and will be removed in a future version. "
+ "Please consider writing to zarr v3."
+ )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
  element_format
  The format of the points element used to store it.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
  axes = get_axes_names(points)
  transformations = _get_transformations(points)
  assert transformations is not None # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
  overwrite_channel_names,
  overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentRasterFormat,
  RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="image",
  raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
  raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
  **metadata: JSONDict,
 ) -> None:
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  _write_raster(
  raster_type="labels",
  raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
  _write_metadata,
  overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
  CurrentShapesFormat,
  ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
  Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
  details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
  """
+ if element_format.zarr_format == 2:
+ warnings.warn(
+ message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+ )
+
  from spatialdata.config import settings
 
  if geometry_encoding is None: