Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joshmoore merged 6 commits into from
May 23, 2022
Merged

Mark V3 API experimental #1032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7c7a882
Mark V3 API experimental
joshmoore May 20, 2022
c1f9d5d
Activate ZVEA for minimal.yml
joshmoore May 20, 2022
145bad2
First round of skipping v3 tests w/o env
joshmoore May 20, 2022
6e59d20
Activate ZVEA for windows.yml
joshmoore May 23, 2022
6e95846
Skip all v3 tests if api is not enabled
joshmoore May 23, 2022
f78f6d9
skip codecov for the assertion
joshmoore May 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/minimal.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
activate-environment: minimal
- name: Tests
shell: "bash -l {0}"
env:
ZARR_V3_EXPERIMENTAL_API: 1
run: |
conda activate minimal
python -m pip install .
pytest -svx --timeout=300
- name: Fixture generation
shell: "bash -l {0}"
env:
ZARR_V3_EXPERIMENTAL_API: 1
run: |
conda activate minimal
rm -rf fixture/
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/python-package.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
ZARR_TEST_ABS: 1
ZARR_TEST_MONGO: 1
ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
run: |
conda activate zarr-env
mkdir ~/blob_emulator
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/windows-testing.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
pytest -sv --timeout=300
env:
ZARR_TEST_ABS: 1
ZARR_V3_EXPERIMENTAL_API: 1
- name: Conda info
shell: bash -l {0}
run: conda info
Expand Down
2 changes: 1 addition & 1 deletion docs/release.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
package has the necessary classes and functions for evaluating Zarr V3.
Since the format is not yet finalized, the classes and functions are not
automatically imported into the regular `zarr` name space. Setting the
`ZARR_V3_API_AVAILABLE` environment variable will activate them.
`ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down
10 changes: 9 additions & 1 deletion zarr/_storage/store.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
if not v3_api_available:
raise NotImplementedError(
"# V3 reading and writing is experimental! To enable support, set:\n"
"ZARR_V3_EXPERIMENTAL_API=1"
) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down
5 changes: 4 additions & 1 deletion zarr/convenience.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

else:

assert_zarr_v3_api_available()

sfx = _get_metadata_suffix(store) # type: ignore

def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
if store._store_version == 2:
ConsolidatedStoreClass = ConsolidatedMetadataStore
else:
assert_zarr_v3_api_available()
ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
# default is to store within 'consolidated' group on v3
if not metadata_key.startswith('meta/root/'):
Expand Down
5 changes: 4 additions & 1 deletion zarr/core.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
if zarr_version is None:
zarr_version = store._store_version

if zarr_version != 2:
assert_zarr_v3_api_available()

if chunk_store is not None:
chunk_store = normalize_store_arg(chunk_store,
zarr_version=zarr_version)
Expand Down
19 changes: 18 additions & 1 deletion zarr/hierarchy.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
if zarr_version is None:
zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

if zarr_version != 2:
assert_zarr_v3_api_available()

if chunk_store is not None:
chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
zarr_version=None):
if zarr_version is None:
zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

if zarr_version != 2:
assert_zarr_v3_api_available()

if store is None:
return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
store = _normalize_store_arg(store, zarr_version=zarr_version)
if zarr_version is None:
zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

if zarr_version != 2:
assert_zarr_v3_api_available()

if zarr_version == 3 and path is None:
raise ValueError(f"path must be provided for a v{zarr_version} group")
path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
zarr_version=zarr_version)
if zarr_version is None:
zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

if zarr_version != 2:
assert_zarr_v3_api_available()

if chunk_store is not None:
chunk_store = _normalize_store_arg(chunk_store,
storage_options=storage_options,
Expand Down
22 changes: 15 additions & 7 deletions zarr/tests/test_convenience.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
meta_root,
getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
ConsolidatedMetadataStoreV3,
DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
with pytest.raises(ValueError):
# no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
x = np.ones(8)
y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
x = np.ones(8)
with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
foo = np.arange(100)
bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
foo = np.arange(100)
bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
kwargs = _init_creation_kwargs(zarr_version)
g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

_version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
"""
https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

@pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Mark V3 API experimental#1032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

joshmoore
merged 6 commits into

from

May 23, 2022
 Merged


Mark V3 API experimental
#1032
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


6 commits


Select commit
Hold shift + click to select a range
7c7a882

Mark V3 API experimental


joshmoore May 20, 2022
c1f9d5d

Activate ZVEA for minimal.yml


joshmoore May 20, 2022
145bad2

First round of skipping v3 tests w/o env


joshmoore May 20, 2022
6e59d20

Activate ZVEA for windows.yml


joshmoore May 23, 2022
6e95846

Skip all v3 tests if api is not enabled


joshmoore May 23, 2022
f78f6d9

skip codecov for the assertion


joshmoore May 23, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


4 changes: 4 additions & 0 deletions
.github/workflows/minimal.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,16 @@ jobs:
 activate-environment: minimal
 - name: Tests
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 python -m pip install .
 pytest -svx --timeout=300
 - name: Fixture generation
 shell: "bash -l {0}"
 env:
 ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate minimal
 rm -rf fixture/
Expand Down

2 changes: 1 addition & 1 deletion
.github/workflows/python-package.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,7 +71,7 @@ jobs:
 ZARR_TEST_ABS: 1
 ZARR_TEST_MONGO: 1
 ZARR_TEST_REDIS: 1
ZARR_V3_API_AVAILABLE: 1
ZARR_V3_EXPERIMENTAL_API: 1
 run: |
 conda activate zarr-env
 mkdir ~/blob_emulator
Expand Down

1 change: 1 addition & 0 deletions
.github/workflows/windows-testing.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -51,6 +51,7 @@ jobs:
 pytest -sv --timeout=300
 env:
 ZARR_TEST_ABS: 1
 ZARR_V3_EXPERIMENTAL_API: 1
 - name: Conda info
 shell: bash -l {0}
 run: conda info
Expand Down

2 changes: 1 addition & 1 deletion
docs/release.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@ Enhancements
 package has the necessary classes and functions for evaluating Zarr V3.
 Since the format is not yet finalized, the classes and functions are not
 automatically imported into the regular `zarr` name space. Setting the
 `ZARR_V3_API_AVAILABLE` environment variable will activate them.
 `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them.
 By :user:`Greggory Lee <grlee77>`; :issue:`898`, :issue:`1006`, and :issue:`1007`.

* **Create FSStore from an existing fsspec filesystem**. If you have created
Expand Down

10 changes: 9 additions & 1 deletion
zarr/_storage/store.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,15 @@

DEFAULT_ZARR_VERSION = 2

v3_api_available = os.environ.get('ZARR_V3_API_AVAILABLE', '0').lower() not in ['0', 'false']
v3_api_available = os.environ.get('ZARR_V3_EXPERIMENTAL_API', '0').lower() not in ['0', 'false']


def assert_zarr_v3_api_available():
 if not v3_api_available:
 raise NotImplementedError(
 "# V3 reading and writing is experimental! To enable support, set:\n"
 "ZARR_V3_EXPERIMENTAL_API=1"
 ) # pragma: no cover


class BaseStore(MutableMapping):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@
import re
from collections.abc import Mapping, MutableMapping

from zarr._storage.store import data_root, meta_root
from zarr._storage.store import data_root, meta_root, assert_zarr_v3_api_available
from zarr.core import Array
from zarr.creation import array as _create_array
from zarr.creation import open_array
Expand DownExpand Up@@ -1209,6 +1209,8 @@ def is_zarr_key(key):

 else:

 assert_zarr_v3_api_available()

 sfx = _get_metadata_suffix(store) # type: ignore

 def is_zarr_key(key):
Expand DownExpand Up@@ -1288,6 +1290,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **
 if store._store_version == 2:
 ConsolidatedStoreClass = ConsolidatedMetadataStore
 else:
 assert_zarr_v3_api_available()
 ConsolidatedStoreClass = ConsolidatedMetadataStoreV3
 # default is to store within 'consolidated' group on v3
 if not metadata_key.startswith('meta/root/'):
Expand Down

5 changes: 4 additions & 1 deletion
zarr/core.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,7 @@
import numpy as np
from numcodecs.compat import ensure_bytes, ensure_ndarray

from zarr._storage.store import _prefix_to_attrs_key
from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available
from zarr.attrs import Attributes
from zarr.codecs import AsType, get_codec
from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError
Expand DownExpand Up@@ -171,6 +171,9 @@ def __init__(
 if zarr_version is None:
 zarr_version = store._store_version

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = normalize_store_arg(chunk_store,
 zarr_version=zarr_version)
Expand Down

19 changes: 18 additions & 1 deletion
zarr/hierarchy.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3,7 +3,8 @@

import numpy as np

from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, DEFAULT_ZARR_VERSION
from zarr._storage.store import (_get_metadata_suffix, data_root, meta_root,
 DEFAULT_ZARR_VERSION, assert_zarr_v3_api_available)
from zarr.attrs import Attributes
from zarr.core import Array
from zarr.creation import (array, create, empty, empty_like, full, full_like,
Expand DownExpand Up@@ -117,6 +118,10 @@ def __init__(self, store, path=None, read_only=False, chunk_store=None,
 store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store: BaseStore = _normalize_store_arg(chunk_store, zarr_version=zarr_version)
 self._store = store
Expand DownExpand Up@@ -1178,6 +1183,10 @@ def _normalize_store_arg(store, *, storage_options=None, mode="r",
 zarr_version=None):
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if store is None:
 return MemoryStore() if zarr_version == 2 else MemoryStoreV3()
 return normalize_store_arg(store,
Expand DownExpand Up@@ -1234,6 +1243,10 @@ def group(store=None, overwrite=False, chunk_store=None,
 store = _normalize_store_arg(store, zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if zarr_version == 3 and path is None:
 raise ValueError(f"path must be provided for a v{zarr_version} group")
 path = normalize_storage_path(path)
Expand DownExpand Up@@ -1305,6 +1318,10 @@ def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=N
 zarr_version=zarr_version)
 if zarr_version is None:
 zarr_version = getattr(store, '_store_version', DEFAULT_ZARR_VERSION)

 if zarr_version != 2:
 assert_zarr_v3_api_available()

 if chunk_store is not None:
 chunk_store = _normalize_store_arg(chunk_store,
 storage_options=storage_options,
Expand Down

22 changes: 15 additions & 7 deletions
zarr/tests/test_convenience.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@
 meta_root,
 getsize,
)
from zarr._storage.store import v3_api_available
from zarr._storage.v3 import (
 ConsolidatedMetadataStoreV3,
 DirectoryStoreV3,
Expand All@@ -44,6 +45,8 @@
)
from zarr.tests.util import have_fsspec

_VERSIONS = v3_api_available and (2, 3) or (2,)


def _init_creation_kwargs(zarr_version):
 kwargs = {'zarr_version': zarr_version}
Expand All@@ -52,7 +55,7 @@ def _init_creation_kwargs(zarr_version):
 return kwargs


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
def test_open_array(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -86,7 +89,7 @@ def test_open_array(path_type, zarr_version):
 open('doesnotexist', mode='r')


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_open_group(path_type, zarr_version):

 store = tempfile.mkdtemp()
Expand DownExpand Up@@ -116,7 +119,7 @@ def test_open_group(path_type, zarr_version):
 assert g.read_only


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_save_errors(zarr_version):
 with pytest.raises(ValueError):
 # no arrays provided
Expand All@@ -129,6 +132,7 @@ def test_save_errors(zarr_version):
 save('data/group.zarr', zarr_version=zarr_version)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_multiple_unnamed():
 x = np.ones(8)
 y = np.zeros(8)
Expand All@@ -142,6 +146,7 @@ def test_zarr_v3_save_multiple_unnamed():
 assert meta_root + 'dataset/arr_1.array.json' in store


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_zarr_v3_save_errors():
 x = np.ones(8)
 with pytest.raises(ValueError):
Expand All@@ -155,7 +160,7 @@ def test_zarr_v3_save_errors():
 save('data/group.zr3', x, zarr_version=3)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_lazy_loader(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -173,7 +178,7 @@ def test_lazy_loader(zarr_version):
 assert 'LazyLoader: ' in repr(loader)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_load_array(zarr_version):
 foo = np.arange(100)
 bar = np.arange(100, 0, -1)
Expand All@@ -192,7 +197,7 @@ def test_load_array(zarr_version):
 assert_array_equal(bar, array)


@pytest.mark.parametrize("zarr_version", [2, 3])
@pytest.mark.parametrize("zarr_version", _VERSIONS)
def test_tree(zarr_version):
 kwargs = _init_creation_kwargs(zarr_version)
 g1 = zarr.group(**kwargs)
Expand All@@ -205,7 +210,7 @@ def test_tree(zarr_version):
 assert str(zarr.tree(g1)) == str(g1.tree())


@pytest.mark.parametrize('zarr_version', [2, 3])
@pytest.mark.parametrize('zarr_version', _VERSIONS)
@pytest.mark.parametrize('stores_from_path', [False, True])
@pytest.mark.parametrize(
 'with_chunk_store,listable',
Expand DownExpand Up@@ -531,6 +536,7 @@ def test_if_exists(self):
 copy_store(source, dest, if_exists='foobar')


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyStoreV3(TestCopyStore):

 _version = 3
Expand DownExpand Up@@ -666,6 +672,7 @@ def test_copy_all():
 assert destination_group.subgroup.attrs["info"] == "sub attrs"


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
def test_copy_all_v3():
 """
 https://github.com/zarr-developers/zarr-python/issues/269
Expand DownExpand Up@@ -931,6 +938,7 @@ def test_logging(self, source, dest, tmpdir):
 copy(source['foo'], dest, dry_run=True, log=True)


@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
class TestCopyV3(TestCopy):

 @pytest.fixture(params=['zarr', 'hdf5'])
Expand Down
Loading