Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jhamman wants to merge 7 commits into from
Closed

Update ABSStore to current Azure Storage API version #620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7596c22
initial attempt to update azure storage version in zarr
Sep 27, 2020
08a9037
add getsize back and fixup listdir
Sep 28, 2020
59524fd
Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…
Nov 22, 2020
4af839d
overwrite existing blob in setitem
Nov 27, 2020
e601264
Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…
Jan 4, 2021
781c9bc
Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore
Jan 4, 2021
be323ac
Merge remote-tracking branch 'origin/master' into pr-620
joshmoore Feb 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion requirements_dev_optional.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down
75 changes: 30 additions & 45 deletions zarr/storage.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

def __init__(self, container, prefix='', account_name=None, account_key=None,
blob_service_kwargs=None):
from azure.storage.blob import BlockBlobService
from azure.storage.blob import ContainerClient
self.container = container
self.prefix = normalize_storage_path(prefix)
self.account_name = account_name
self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
self.account_key = account_key
if blob_service_kwargs is not None:
self.blob_service_kwargs = blob_service_kwargs
else: # pragma: no cover
self.blob_service_kwargs = dict()
self.client = BlockBlobService(self.account_name, self.account_key,
**self.blob_service_kwargs)
self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

# needed for pickling
def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
return state

def __setstate__(self, state):
from azure.storage.blob import BlockBlobService
from azure.storage.blob import ContainerClient
self.__dict__.update(state)
self.client = BlockBlobService(self.account_name, self.account_key,
**self.blob_service_kwargs)
self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

def _append_path_to_prefix(self, path):
if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
return path_norm

def __getitem__(self, key):
from azure.common import AzureMissingResourceHttpError
from azure.core.exceptions import ResourceNotFoundError
blob_name = self._append_path_to_prefix(key)
try:
blob = self.client.get_blob_to_bytes(self.container, blob_name)
return blob.content
except AzureMissingResourceHttpError:
return self.client.download_blob(blob_name).readall()
except ResourceNotFoundError:
raise KeyError('Blob %s not found' % blob_name)

def __setitem__(self, key, value):
value = ensure_bytes(value)
blob_name = self._append_path_to_prefix(key)
self.client.create_blob_from_bytes(self.container, blob_name, value)
self.client.upload_blob(blob_name, value, overwrite=True)

def __delitem__(self, key):
from azure.common import AzureMissingResourceHttpError
from azure.core.exceptions import ResourceNotFoundError
try:
self.client.delete_blob(self.container, self._append_path_to_prefix(key))
except AzureMissingResourceHttpError:
self.client.delete_blob(self._append_path_to_prefix(key))
except ResourceNotFoundError:
raise KeyError('Blob %s not found' % key)

def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
list_blobs_prefix = self.prefix + '/'
else:
list_blobs_prefix = None
for blob in self.client.list_blobs(self.container, list_blobs_prefix):
for blob in self.client.list_blobs(list_blobs_prefix):
yield self._strip_prefix_from_path(blob.name, self.prefix)

def __len__(self):
return len(self.keys())

def __contains__(self, key):
blob_name = self._append_path_to_prefix(key)
assert len(blob_name) >= 1
if self.client.exists(self.container, blob_name):
return True
else:
return False
return self.client.get_blob_client(blob_name).exists()

def listdir(self, path=None):
from azure.storage.blob import Blob
dir_path = normalize_storage_path(self._append_path_to_prefix(path))
if dir_path:
dir_path += '/'
items = list()
for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
if type(blob) == Blob:
items.append(self._strip_prefix_from_path(blob.name, dir_path))
else:
items.append(self._strip_prefix_from_path(
blob.name[:blob.name.find('/', len(dir_path))], dir_path))
items = set()
for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
items.add(self._strip_prefix_from_path(blob.name, dir_path))
return items

def rmdir(self, path=None):
dir_path = normalize_storage_path(self._append_path_to_prefix(path))
if dir_path:
dir_path += '/'
for blob in self.client.list_blobs(self.container, prefix=dir_path):
assert len(blob.name) >= 1
self.client.delete_blob(self.container, blob.name)
for blob in self.client.list_blobs(name_starts_with=dir_path):
self.client.delete_blob(blob)

def getsize(self, path=None):
from azure.storage.blob import Blob
store_path = normalize_storage_path(path)
fs_path = self.prefix
if store_path:
fs_path = self._append_path_to_prefix(store_path)

if fs_path != "" and self.client.exists(self.container, fs_path):
return self.client.get_blob_properties(
self.container, fs_path
).properties.content_length
fs_path = self._append_path_to_prefix(store_path)
blob_client = self.client.get_blob_client(fs_path)
if blob_client.exists():
return blob_client.get_blob_properties().size
else:
size = 0
if fs_path == '':
fs_path = None
else:
elif not fs_path.endswith('/'):
fs_path += '/'
for blob in self.client.list_blobs(self.container, prefix=fs_path,
delimiter='/'):
if type(blob) == Blob:
size += blob.properties.content_length
for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
if blob_client.exists():
size += blob_client.get_blob_properties().size
return size

def clear(self):
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Update ABSStore to current Azure Storage API version#620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

jhamman
wants to merge
7
commits into

from

 Closed


Update ABSStore to current Azure Storage API version
#620
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
7596c22

initial attempt to update azure storage version in zarr

Sep 27, 2020
08a9037

add getsize back and fixup listdir

Sep 28, 2020
59524fd

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Nov 22, 2020
4af839d

overwrite existing blob in setitem

Nov 27, 2020
e601264

Merge branch 'master' of github.com:zarr-developers/zarr into fix/abs…

Jan 4, 2021
781c9bc

Merge branch 'fix/absstore' of github.com:jhamman/zarr into fix/absstore

Jan 4, 2021
be323ac

Merge remote-tracking branch 'origin/master' into pr-620


joshmoore Feb 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
requirements_dev_optional.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ ipytree==0.1.3
# optional library requirements for services
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor
azure-storage-blob==2.0.1 # pyup: ignore
azure-storage-blob==12.5.0 # pyup: ignore
redis==3.3.8
pymongo==3.9.0
# optional test requirements
Expand Down

75 changes: 30 additions & 45 deletions
zarr/storage.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2147,17 +2147,18 @@ class ABSStore(MutableMapping):

 def __init__(self, container, prefix='', account_name=None, account_key=None,
 blob_service_kwargs=None):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.container = container
 self.prefix = normalize_storage_path(prefix)
 self.account_name = account_name
 self.account_account_url = f"https://{self.account_name}.blob.core.windows.net"
 self.account_key = account_key
 if blob_service_kwargs is not None:
 self.blob_service_kwargs = blob_service_kwargs
 else: # pragma: no cover
 self.blob_service_kwargs = dict()
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 # needed for pickling
 def __getstate__(self):
Expand All@@ -2166,10 +2167,10 @@ def __getstate__(self):
 return state

 def __setstate__(self, state):
 from azure.storage.blob import BlockBlobService
 from azure.storage.blob import ContainerClient
 self.__dict__.update(state)
 self.client = BlockBlobService(self.account_name, self.account_key,
 **self.blob_service_kwargs)
 self.client = ContainerClient(self.account_account_url, self.container,
credential=self.account_key, **self.blob_service_kwargs)

 def _append_path_to_prefix(self, path):
 if self.prefix == '':
Expand All@@ -2188,24 +2189,23 @@ def _strip_prefix_from_path(path, prefix):
 return path_norm

 def __getitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 blob_name = self._append_path_to_prefix(key)
 try:
 blob = self.client.get_blob_to_bytes(self.container, blob_name)
 return blob.content
 except AzureMissingResourceHttpError:
 return self.client.download_blob(blob_name).readall()
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % blob_name)

 def __setitem__(self, key, value):
 value = ensure_bytes(value)
 blob_name = self._append_path_to_prefix(key)
 self.client.create_blob_from_bytes(self.container, blob_name, value)
 self.client.upload_blob(blob_name, value, overwrite=True)

 def __delitem__(self, key):
 from azure.common import AzureMissingResourceHttpError
 from azure.core.exceptions import ResourceNotFoundError
 try:
 self.client.delete_blob(self.container, self._append_path_to_prefix(key))
 except AzureMissingResourceHttpError:
 self.client.delete_blob(self._append_path_to_prefix(key))
 except ResourceNotFoundError:
 raise KeyError('Blob %s not found' % key)

 def __eq__(self, other):
Expand All@@ -2223,63 +2223,48 @@ def __iter__(self):
 list_blobs_prefix = self.prefix + '/'
 else:
 list_blobs_prefix = None
 for blob in self.client.list_blobs(self.container, list_blobs_prefix):
 for blob in self.client.list_blobs(list_blobs_prefix):
 yield self._strip_prefix_from_path(blob.name, self.prefix)

 def __len__(self):
 return len(self.keys())

 def __contains__(self, key):
 blob_name = self._append_path_to_prefix(key)
 assert len(blob_name) >= 1
 if self.client.exists(self.container, blob_name):
 return True
 else:
 return False
 return self.client.get_blob_client(blob_name).exists()

 def listdir(self, path=None):
 from azure.storage.blob import Blob
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 items = list()
 for blob in self.client.list_blobs(self.container, prefix=dir_path, delimiter='/'):
 if type(blob) == Blob:
 items.append(self._strip_prefix_from_path(blob.name, dir_path))
 else:
 items.append(self._strip_prefix_from_path(
 blob.name[:blob.name.find('/', len(dir_path))], dir_path))
 items = set()
 for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/'):
 items.add(self._strip_prefix_from_path(blob.name, dir_path))
 return items

 def rmdir(self, path=None):
 dir_path = normalize_storage_path(self._append_path_to_prefix(path))
 if dir_path:
 dir_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=dir_path):
 assert len(blob.name) >= 1
 self.client.delete_blob(self.container, blob.name)
 for blob in self.client.list_blobs(name_starts_with=dir_path):
 self.client.delete_blob(blob)

 def getsize(self, path=None):
 from azure.storage.blob import Blob
 store_path = normalize_storage_path(path)
 fs_path = self.prefix
 if store_path:
 fs_path = self._append_path_to_prefix(store_path)

 if fs_path != "" and self.client.exists(self.container, fs_path):
 return self.client.get_blob_properties(
 self.container, fs_path
 ).properties.content_length
 fs_path = self._append_path_to_prefix(store_path)
 blob_client = self.client.get_blob_client(fs_path)
 if blob_client.exists():
 return blob_client.get_blob_properties().size
 else:
 size = 0
 if fs_path == '':
 fs_path = None
else:
elif not fs_path.endswith('/'):
 fs_path += '/'
 for blob in self.client.list_blobs(self.container, prefix=fs_path,
 delimiter='/'):
 if type(blob) == Blob:
 size += blob.properties.content_length
 for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'):
blob_client = self.client.get_blob_client(blob)
 if blob_client.exists():
 size += blob_client.get_blob_properties().size
 return size

 def clear(self):
Expand Down