Skip to content

cache catalog and capabilities#108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rweigel merged 1 commit into from
Aug 11, 2026
Merged

cache catalog and capabilities #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions hapiclient/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,8 +22,3 @@
import warnings
warnings.filterwarnings("ignore", message=".*urllib3.*OpenSSL.*")

if sys.version_info[0] < 3:
# Python 2.7
reload(sys)
sys.setdefaultencoding('utf8')

86 changes: 55 additions & 31 deletions hapiclient/cache.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,20 +32,19 @@ def cachedir(*args):
return os.path.join(args[0], server2dirname(args[1]))


def request2path(*args):
# request2path(server, dataset, parameters, start, stop)
# request2path(server, dataset, parameters, start, stop, cachedir)
def request2path(server, dataset=None, parameters=None, start=None, stop=None, cache_dir=None, endpoint=None):

import os
import re
import platform

if len(args) == 5:
# Use default if cachedir not given.
if cache_dir is None:
# Use default if cache_dir not given.
cachedirectory = cachedir()
else:
cachedirectory = args[5]
cachedirectory = cache_dir

args = list(args)
args = [server, dataset, parameters, start, stop]

# Replace forbidden characters in directory and filename
# Replacements assume that there will be no name collisions,
Expand All@@ -68,36 +67,58 @@ def request2path(*args):
)

for element in reps:
args[1] = re.sub(element[0], element[1], args[1])
args[2] = re.sub(element[0], element[1], args[2])
if dataset is not None:
dataset = re.sub(element[0], element[1], dataset)
if parameters is not None:
parameters = re.sub(element[0], element[1], parameters)

else:
args[1] = re.sub('/','@forwardslash@',args[1])
args[2] = re.sub('/','@forwardslash@',args[2])
if dataset is not None:
dataset = re.sub('/','@forwardslash@', dataset)
if parameters is not None:
parameters = re.sub('/','@forwardslash@', parameters)

# To shorten filenames.
args[3] = re.sub(r'-|:|\.|Z', '', args[3])
args[4] = re.sub(r'-|:|\.|Z', '', args[4])
if start is not None:
start = re.sub(r'-|:|\.|Z', '', start)
if stop is not None:
stop = re.sub(r'-|:|\.|Z', '', stop)

# URL subdirectory
urldirectory = server2dirname(args[0])
fname = '%s_%s_%s_%s' % (args[1], args[2], args[3], args[4])

return os.path.join(cachedirectory, urldirectory, fname)
if not dataset and not endpoint:
raise ValueError('Either dataset or endpoint must be specified.')

if endpoint is None:
endpoint = ''

if dataset is None:
fname = endpoint
else:
fname = dataset
if parameters is not None:
fname += '_' + parameters
if start is not None:
fname += '_' + start
if stop is not None:
fname += '_' + stop

return os.path.join(cachedirectory, urldirectory, endpoint, fname)


def meta_cache_paths(SERVER, DATASET, cachedir):
def meta_cache_paths(server, dataset, endpoint, cache_dir):
"""Return dict with metadata cache directory and file names."""

fname_root = request2path(SERVER, DATASET, '', '', '', cachedir)
fname_root = request2path(server, dataset, cache_dir=cache_dir, endpoint=endpoint)

return {
'json': fname_root + '.json',
'pkl': fname_root + '.pkl'
}


def meta_cache_read(SERVER, DATASET, opts):
def meta_cache_read(server, dataset, endpoint, opts):
"""Read metadata from PKL cache. Returns meta dict or None."""

import os
Expand All@@ -106,10 +127,13 @@ def meta_cache_read(SERVER, DATASET, opts):
from hapiclient.log import log

if not opts["usecache"]:
log('Not checking metadata cache because usecache is False.')
if endpoint in ['', 'info']:
log(f'Not checking metadata cache for /info?dataset={dataset} response because usecache is False.')
else:
log(f'Not checking metadata cache for /{endpoint} response because usecache is False.')
return None

fnamepkl = meta_cache_paths(SERVER, DATASET, opts['cachedir'])['pkl']
fnamepkl = meta_cache_paths(server, dataset, endpoint, opts['cachedir'])['pkl']
if os.path.isfile(fnamepkl):
log('Reading %s' % os.path.basename(fnamepkl))
with open(fnamepkl, 'rb') as f:
Expand All@@ -121,7 +145,7 @@ def meta_cache_read(SERVER, DATASET, opts):
return None


def meta_cache_write(meta, SERVER, DATASET, opts):
def meta_cache_write(meta, server, dataset, endpoint, opts):
"""Write metadata to JSON and PKL cache files."""

import os
Expand All@@ -132,7 +156,7 @@ def meta_cache_write(meta, SERVER, DATASET, opts):
if not opts["cache"]:
return

paths = meta_cache_paths(SERVER, DATASET, opts['cachedir'])
paths = meta_cache_paths(server, dataset, endpoint, opts['cachedir'])
fnamejson, fnamepkl = paths['json'], paths['pkl']

log('Writing %s ' % os.path.basename(fnamejson))
Expand All@@ -142,10 +166,10 @@ def meta_cache_write(meta, SERVER, DATASET, opts):
write_atomic(fnamepkl, meta)


def data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, cachedir):
def data_cache_paths(server, dataset, parameters, start, stop, cache_dir):
"""Return dict with data cache file names."""

fname_root = request2path(SERVER, DATASET, PARAMETERS, START, STOP, cachedir)
fname_root = request2path(server, dataset, parameters, start, stop, cache_dir, 'data')

return {
'csv': fname_root + '.csv',
Expand All@@ -155,7 +179,7 @@ def data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, cachedir):
}


def data_cache_read_metax(SERVER, DATASET, PARAMETERS, START, STOP, opts):
def data_cache_read_metax(server, dataset, parameters, start, stop, opts):
"""Read extended request metadata from PKL cache. Returns meta dict or None."""

import os
Expand All@@ -167,7 +191,7 @@ def data_cache_read_metax(SERVER, DATASET, PARAMETERS, START, STOP, opts):
log('Not checking subsetted metadata cache because usecache is False.')
return None

fnamepklx = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])['pkl']
fnamepklx = data_cache_paths(server, dataset, parameters, start, stop, opts['cachedir'])['pkl']
if os.path.isfile(fnamepklx):
log('Reading subsetted metadata cache %s' % os.path.basename(fnamepklx))
with open(fnamepklx, 'rb') as f:
Expand All@@ -178,7 +202,7 @@ def data_cache_read_metax(SERVER, DATASET, PARAMETERS, START, STOP, opts):
return None


def data_cache_read_npy(SERVER, DATASET, PARAMETERS, START, STOP, opts):
def data_cache_read_npy(server, dataset, parameters, start, stop, opts):
"""Read cached numpy data array. Returns None if not cached."""

import os
Expand All@@ -189,7 +213,7 @@ def data_cache_read_npy(SERVER, DATASET, PARAMETERS, START, STOP, opts):
if not opts["usecache"]:
return None

fnamenpy = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])['npy']
fnamenpy = data_cache_paths(server, dataset, parameters, start, stop, opts['cachedir'])['npy']

if not os.path.isfile(fnamenpy):
return None
Expand All@@ -201,7 +225,7 @@ def data_cache_read_npy(SERVER, DATASET, PARAMETERS, START, STOP, opts):
return data


def data_cache_write(data_result, meta, SERVER, DATASET, PARAMETERS, START, STOP, opts):
def data_cache_write(data_result, meta, server, dataset, parameters, start, stop, opts):
"""Write data array and extended metadata to cache files.

Also updates meta with file-related x_ fields before writing.
Expand All@@ -212,10 +236,10 @@ def data_cache_write(data_result, meta, SERVER, DATASET, PARAMETERS, START, STOP
from hapiclient.log import log
from hapiclient.util import write_atomic

data_paths = data_cache_paths(SERVER, DATASET, PARAMETERS, START, STOP, opts['cachedir'])
data_paths = data_cache_paths(server, dataset, parameters, start, stop, opts['cachedir'])
fnamecsv, fnamebin, fnamenpy, fnamepklx = data_paths['csv'], data_paths['bin'], data_paths['npy'], data_paths['pkl']

meta_paths = meta_cache_paths(SERVER, DATASET, opts['cachedir'])
meta_paths = meta_cache_paths(server, dataset, 'info', opts['cachedir'])
fnamejson, fnamepkl = meta_paths['json'], meta_paths['pkl']

meta.update({"x_metaFileParsed": fnamepkl})
Expand Down
45 changes: 9 additions & 36 deletions hapiclient/capabilities.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,22 @@

def capabilities(SERVER):
"""Return the capabilities of a HAPI server.
def capabilities(SERVER, opts):
"""Return the /capabilities response from a HAPI server.

Args:
SERVER (str): The base URL of the HAPI server.

Returns:
dict: A dictionary containing the capabilities of the server.
"""
from hapiclient.util import urlopen

caps = urlopen(SERVER + '/capabilities', parse_json=True)
import hapiclient as hc

return caps


def get_format(SERVER, format):
"""Return the transport format to use, accounting for server capabilities.

If the requested format is not supported by the server, falls back to 'csv'.
"""
caps = hc.cache.meta_cache_read(SERVER, None, 'capabilities', opts)
if caps is not None:
return caps

from hapiclient.util import error
caps = hc.util.urlopen(SERVER + '/capabilities', parse_json=True)

cformats = ['csv', 'binary'] # client formats
if format not in cformats:
msg = 'This client does not handle streaming format "%s". Available options: %s'
error(msg % (format, ', '.join(cformats)))
hc.cache.meta_cache_write(caps, SERVER, None, 'capabilities', opts)

if format != 'csv':
caps = capabilities(SERVER)
if "outputFormats" not in caps:
return 'csv'

formats = caps.get("outputFormats", []) # Server formats
if len(formats) == 0:
return 'csv'

if format not in formats:
#from hapiclient.util import warning
#msg = 'Requested streaming format "%s" not available from %s. Will use "csv". Available options: %s'
#warning(msg % (format, SERVER, ', '.join(formats)))
format = 'csv'

if 'binary' not in formats:
format = 'csv'

return format
return caps
16 changes: 10 additions & 6 deletions hapiclient/catalog.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
from hapiclient.log import log
from hapiclient.util import urlopen
def catalog(SERVER, opts):

import hapiclient as hc

cat = hc.cache.meta_cache_read(SERVER, None, 'catalog', opts)
if cat is not None:
return cat

def catalog(SERVER):
# TODO: Cache
url = SERVER + '/catalog'
meta = urlopen(url, parse_json=True)
cat = hc.util.urlopen(url, parse_json=True)

hc.cache.meta_cache_write(cat, SERVER, None, 'catalog', opts)

return meta
return cat
Loading
Loading