diff --git a/quest/api/datasets.py b/quest/api/datasets.py index 2619fd2b..0b7e47a3 100644 --- a/quest/api/datasets.py +++ b/quest/api/datasets.py @@ -60,10 +60,10 @@ def download(feature, file_path, dataset=None, **kwargs): @add_async -def publish(publisher_uri, options=None): +def publish(publisher_uri, **kwargs): provider, publisher, feature = util.parse_service_uri(publisher_uri) driver = util.load_providers()[provider] - data = driver.publish(publisher=publisher, options=options) + data = driver.publish(publisher=publisher, **kwargs) return data @add_async diff --git a/quest/api/services.py b/quest/api/services.py index 2c225f3c..1dd2a195 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -127,6 +127,7 @@ def add_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service added' + util.load_providers(update_cache=True) else: msg = 'service already present' else: @@ -159,6 +160,7 @@ def delete_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service removed' + util.load_providers(update_cache=True) else: msg = 'service not found' diff --git a/quest/services/base/provider_base.py b/quest/services/base/provider_base.py index 7bc08b45..7adcb4d3 100644 --- a/quest/services/base/provider_base.py +++ b/quest/services/base/provider_base.py @@ -283,8 +283,8 @@ def download_options(self, service, fmt): """ return self.services[service].download_options(fmt) - def publish(self, publisher, options): - return self.publishers[publisher].publish(options) + def publish(self, publisher, **kwargs): + return self.publishers[publisher].publish(**kwargs) def publish_options(self, publisher, fmt): return self.publishers[publisher].publish_options(fmt) diff --git a/quest/services/base/publish_base.py b/quest/services/base/publish_base.py index 0e6a80cb..a19ca1fb 100644 --- a/quest/services/base/publish_base.py +++ b/quest/services/base/publish_base.py @@ -38,5 +38,5 @@ def publish_options(self, fmt): return schema - def publish(self, options): + def publish(self, **kwargs): raise NotImplementedError() diff --git a/quest/services/base/service_base.py b/quest/services/base/service_base.py index 5dae3ace..27195b4d 100644 --- a/quest/services/base/service_base.py +++ b/quest/services/base/service_base.py @@ -92,7 +92,7 @@ def download_options(self, fmt): return schema - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): raise NotImplementedError() def get_features(self, **kwargs): @@ -138,7 +138,7 @@ class SingleFileServiceBase(ServiceBase): """Base file for datasets that are a single file download eg elevation raster etc """ - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): feature_id = util.construct_service_uri(self.provider.name, self.name, feature) feature = self.provider.get_features(self.name).loc[feature_id] reserved = feature.get('reserved') diff --git a/quest/services/cuahsi_hs.py b/quest/services/cuahsi_hs.py index 2557440d..100caf9e 100644 --- a/quest/services/cuahsi_hs.py +++ b/quest/services/cuahsi_hs.py @@ -9,6 +9,7 @@ import param import os + class HSServiceBase(SingleFileServiceBase): @property @@ -35,7 +36,7 @@ class HSGeoService(HSServiceBase): def get_features(self, **kwargs): - results = list(self.hs.resources()) + results = list(self.hs.resources(coverage_type="box", north="90", south="-90", east="180", west="-180")) if len(results) == 0: raise ValueError('No resource available from HydroShare.') @@ -126,21 +127,18 @@ class HSPublisher(PublishBase): 'Time Series': 'TimeSeriesResource' } - title = param.String(default="example title", doc="Title of resource", precedence=2) - abstract = param.String(default="example abstract", precedence=3, - doc="An description of the resource to be added to HydroShare.") - keywords = param.List(default=[], precedence=4, doc="list of keyword strings to describe the resource") - dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, precedence=5, - doc="dataset to publish to HydroShare") - resource_type = param.ObjectSelector(doc='parameter', precedence=1, objects=sorted(_resource_type_map.keys())) + resource_type = param.ObjectSelector(default=None, doc="", precedence=1, objects=sorted(_resource_type_map.keys())) + title = param.String(default="", doc="", precedence=2) + abstract = param.String(default="", doc="", precedence=3) + keywords = param.List(default=[], doc="", precedence=4) + dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, doc="", precedence=5) @property def hs(self): return self.provider.get_hs() - def publish(self, options=None): - - p = param.ParamOverrides(self, options) + def publish(self, **kwargs): + p = param.ParamOverrides(self, kwargs) valid_file_paths = [] valid_extensions = [] diff --git a/quest/services/kitware_girder.py b/quest/services/kitware_girder.py index 9a4bf34e..ba7bd674 100644 --- a/quest/services/kitware_girder.py +++ b/quest/services/kitware_girder.py @@ -34,9 +34,9 @@ class GirderPublisher(PublishBase): def gc(self): return self.provider.get_gc() - def publish(self, options=None): + def publish(self, **kwargs): try: - p = param.ParamOverrides(self, options) + p = param.ParamOverrides(self, kwargs) params = {'name': p.title, 'description': p.collection_description} resource_information_dict = self.gc.createResource(path='collection', params=params) folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], diff --git a/quest/services/noaa_coastwatch.py b/quest/services/noaa_coastwatch.py index bb79ba92..6e082007 100644 --- a/quest/services/noaa_coastwatch.py +++ b/quest/services/noaa_coastwatch.py @@ -65,8 +65,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/noaa_ncdc.py b/quest/services/noaa_ncdc.py index 5bd26238..6bbe1d5f 100644 --- a/quest/services/noaa_ncdc.py +++ b/quest/services/noaa_ncdc.py @@ -74,8 +74,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/template_service.py b/quest/services/template_service.py index 4d0cd8cf..b4e4ba6f 100644 --- a/quest/services/template_service.py +++ b/quest/services/template_service.py @@ -26,7 +26,7 @@ class ExampleServiceBase(ServiceBase): smtk_template = None _parameter_map = dict() - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwwargs): metadata = {} # get metadata from service data = None # data structure containing downloaded data diff --git a/quest/services/user_provider.py b/quest/services/user_provider.py index c6647215..4e525127 100644 --- a/quest/services/user_provider.py +++ b/quest/services/user_provider.py @@ -51,7 +51,7 @@ def instance(cls, service_name, service_data, provider, uri, is_remote): return self - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): if self.datasets_mapping is not None: fnames = self.datasets_mapping if isinstance(dict, self.datasets_mapping): diff --git a/quest/services/usgs_nwis.py b/quest/services/usgs_nwis.py index 08233a39..cc81b72b 100644 --- a/quest/services/usgs_nwis.py +++ b/quest/services/usgs_nwis.py @@ -18,8 +18,8 @@ class NwisServiceBase(TimePeriodServiceBase): period = param.String(default='P365D', precedence=4, doc='time period (e.g. P365D = 365 days or P4W = 4 weeks)') smtk_template = 'start_end_or_period.sbt' - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) parameter = p.parameter start = p.start diff --git a/quest/util/misc.py b/quest/util/misc.py index fac84738..5f03d405 100644 --- a/quest/util/misc.py +++ b/quest/util/misc.py @@ -19,6 +19,7 @@ from uuid import uuid4, UUID import quest +the_providers = None def generate_cache(update=False): """Downloads features for all services and caches results. @@ -238,22 +239,27 @@ def load_drivers(namespace, names=None): return {name: driver.DriverManager(namespace, name, invoke_on_load='True') for name in names} -def load_providers(): - settings = get_settings() +def load_providers(update_cache=False): + global the_providers - # add web services + settings = get_settings() web_services = list_drivers('services') web_services.remove('user') - providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} + if update_cache or the_providers is None: + providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} - if len(settings.get('USER_SERVICES', [])) > 0: - for uri in settings.get('USER_SERVICES', []): - try: - drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver - providers['user-' + drv.name] = drv - except Exception as e: - logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + if len(settings.get('USER_SERVICES', [])) > 0: + for uri in settings.get('USER_SERVICES', []): + try: + drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver + providers['user-' + drv.name] = drv + except Exception as e: + logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + + the_providers = providers + else: + providers = the_providers return providers