From 37838a79246631a5a389cb59c20c0b6d0aa4c0bb Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 01/44] Dataset display name fix (More readable). --- quest/api/services.py | 2 - quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ quest/services/noaa_ncep.py | 80 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 quest/services/ISEP_Girder.py create mode 100644 quest/services/noaa_ncep.py diff --git a/quest/api/services.py b/quest/api/services.py index 2c225f3c..b2b78f0d 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -78,8 +78,6 @@ def get_publishers(expand=None, publisher_type=None): Returns: providers (list or dict,Default=list): list of all available providers - - """ providers = util.load_providers() publishers = {} diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py new file mode 100644 index 00000000..284823f9 --- /dev/null +++ b/quest/services/noaa_ncep.py @@ -0,0 +1,80 @@ +from .base import ProviderBase, ServiceBase +from ncep_client import NCEP_Client +from shapely.geometry import box +import pandas as pd +import param + + +class NCEPServiceBase(ServiceBase): + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class NCEP_GFS_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_gfs" + display_name = "NCEP GFS Service" + description = 'NCEP GFS is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = ncep.get_provider_types("Global Forecast System") + _possible_products = ncep.get_provider_products("Global Forecast System") + _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def get_features(self, **kwargs): + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "GFS" or p.product == "GDAS": + raise ValueError("Please specify a specific product not GFS or GDAS") + + results = ncep.get_ncep_product_data(ncep_provider="Global Forecast System", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEPProvider(ProviderBase): + service_base_class = NCEPServiceBase + publishers_list = [] + display_name = 'NCEP GFS Provider' + description = 'Services avaliable through the NOAA NCEP Server.' + organization_name = 'National Centers for Environmental Prediction' + organization_abbr = 'NCEP' + From 47b78dc08227317f74be6234e78c9f9b59074c70 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 14:54:34 -0500 Subject: [PATCH 02/44] Added the noaa_ncep plugin and updated the conda environment file. --- py3_conda_environment.yml | 5 ++++- quest/services/noaa_ncep.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/py3_conda_environment.yml b/py3_conda_environment.yml index 07a0f9f7..e4b5d74e 100644 --- a/py3_conda_environment.yml +++ b/py3_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -41,4 +42,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 284823f9..9a9280b9 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -72,9 +72,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - publishers_list = [] display_name = 'NCEP GFS Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' - organization_abbr = 'NCEP' - + organization_abbr = 'NCEP' \ No newline at end of file From e790d72f22627063b44e3b5651664dfabb6ec5d9 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 15:51:57 -0500 Subject: [PATCH 03/44] NCEP test and setup.cfg were updated. --- test/data.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/data.py b/test/data.py index 01bc2e35..22d6512f 100644 --- a/test/data.py +++ b/test/data.py @@ -203,6 +203,75 @@ 'title': 'NWIS Instantaneous Values Service Download Options'}, 'svc://cuahsi-hydroshare:hs_geo': {}, 'svc://cuahsi-hydroshare:hs_norm': {}, + 'svc://noaa-ncep:ncep_gfs': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [['enkf', 'enkf'], + ['gdas', 'gdas'], + ['gfs', 'gfs'], + ['gfsmos', 'gfsmos'], + ['sst', 'sst']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['GDAS', 'GDAS'], ['GFS', 'GFS']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [["0.50 deg 'full' file description", + "0.50 deg 'full' file description"], + ['32km Lambert Conformal grid', '32km Lambert Conformal grid'], + ['Atmospheric Analysis', 'Atmospheric Analysis'], + ['Atmospheric Model Data', 'Atmospheric Model Data'], + ['BUFR Sounding Files per Station', 'BUFR Sounding Files per Station'], + ['Binary Universal Form for the Representation of meteorological data (BUFR)', + 'Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['GDAS', 'GDAS'], + ['GFS', 'GFS'], + ['Global longitude-latitude grid', 'Global longitude-latitude grid'], + ['MDL Extratropical Storm Surge', 'MDL Extratropical Storm Surge'], + ['MOS Aviation Product', 'MOS Aviation Product'], + ['Prepared BUFR files', 'Prepared BUFR files'], + ['Pressure Level Data', 'Pressure Level Data'], + ['Sigma Atmospheric Model Data', 'Sigma Atmospheric Model Data'], + ['Smart Initialization Guam', 'Smart Initialization Guam'], + ['Surface Analysis', 'Surface Analysis'], + ['Surface Boundary Conditions', 'Surface Boundary Conditions'], + ['Surface Flux', 'Surface Flux'], + ['T1534 Semi-Lagrangian grid', 'T1534 Semi-Lagrangian grid'], + ['Time Dependent Satellite Bias Correction', + 'Time Dependent Satellite Bias Correction'], + ['Tropical Cyclone Vital Statistics', + 'Tropical Cyclone Vital Statistics'], + ['WAFS/ICAO/International Exchange/FOS Grids', + 'WAFS/ICAO/International Exchange/FOS Grids'], + ['World Area Forecast System', 'World Area Forecast System'], + ['global longitude-latitude grid (1.0 deg)', + 'global longitude-latitude grid (1.0 deg)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP GFS Service Download Options'} } From 00aa22cb25925357295af303c120675d619ee737 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 04/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From 5ff0e5ef8ac82e584193bec91258c3d1f8fdb079 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:40:01 -0500 Subject: [PATCH 05/44] Updated the docs and the python 2 environment file for ncep and terrapin. --- docs/install.rst | 9 +++------ py2_conda_environment.yml | 5 ++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index eb4c8416..08123b28 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,11 +12,11 @@ Install Released Conda Package a. Install Quest into new environment:: - conda create -n quest -c conda-forge -c erdc quest terrapin + conda create -n quest -c conda-forge -c erdc quest b. Install Quest into existing environment:: - conda install -c erdc -c conda-forge quest terrapin + conda install -c erdc -c conda-forge quest .. note:: @@ -49,11 +49,8 @@ Conda Install Optional ........ - d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel:: - conda install -c erdc -c conda-forge terrapin - - e. Run tests:: + d. Run tests:: python setup.py test diff --git a/py2_conda_environment.yml b/py2_conda_environment.yml index 67ae4cb0..638fb57d 100644 --- a/py2_conda_environment.yml +++ b/py2_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -40,4 +41,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file From e0cd550ade31d5f0af8ea2251a8a8a85e63aa63a Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 16 Apr 2018 11:45:42 -0500 Subject: [PATCH 06/44] Minor updates (#38) * Added the cuahsi coverage fix. The change was not getting all the resources and just getting all resources that had a coverage from the world. * Made kwargs the standard over using options, params, and kwargs. * Updated the load_providers to only update if update-cache is True or if the global is None. --- quest/api/datasets.py | 4 ++-- quest/api/services.py | 2 ++ quest/services/base/provider_base.py | 4 ++-- quest/services/base/publish_base.py | 2 +- quest/services/base/service_base.py | 4 ++-- quest/services/cuahsi_hs.py | 20 +++++++++----------- quest/services/kitware_girder.py | 4 ++-- quest/services/noaa_coastwatch.py | 4 ++-- quest/services/noaa_ncdc.py | 4 ++-- quest/services/template_service.py | 2 +- quest/services/user_provider.py | 2 +- quest/services/usgs_nwis.py | 4 ++-- quest/util/misc.py | 28 +++++++++++++++++----------- 13 files changed, 45 insertions(+), 39 deletions(-) diff --git a/quest/api/datasets.py b/quest/api/datasets.py index 2619fd2b..0b7e47a3 100644 --- a/quest/api/datasets.py +++ b/quest/api/datasets.py @@ -60,10 +60,10 @@ def download(feature, file_path, dataset=None, **kwargs): @add_async -def publish(publisher_uri, options=None): +def publish(publisher_uri, **kwargs): provider, publisher, feature = util.parse_service_uri(publisher_uri) driver = util.load_providers()[provider] - data = driver.publish(publisher=publisher, options=options) + data = driver.publish(publisher=publisher, **kwargs) return data @add_async diff --git a/quest/api/services.py b/quest/api/services.py index 2c225f3c..1dd2a195 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -127,6 +127,7 @@ def add_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service added' + util.load_providers(update_cache=True) else: msg = 'service already present' else: @@ -159,6 +160,7 @@ def delete_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service removed' + util.load_providers(update_cache=True) else: msg = 'service not found' diff --git a/quest/services/base/provider_base.py b/quest/services/base/provider_base.py index 7bc08b45..7adcb4d3 100644 --- a/quest/services/base/provider_base.py +++ b/quest/services/base/provider_base.py @@ -283,8 +283,8 @@ def download_options(self, service, fmt): """ return self.services[service].download_options(fmt) - def publish(self, publisher, options): - return self.publishers[publisher].publish(options) + def publish(self, publisher, **kwargs): + return self.publishers[publisher].publish(**kwargs) def publish_options(self, publisher, fmt): return self.publishers[publisher].publish_options(fmt) diff --git a/quest/services/base/publish_base.py b/quest/services/base/publish_base.py index 0e6a80cb..a19ca1fb 100644 --- a/quest/services/base/publish_base.py +++ b/quest/services/base/publish_base.py @@ -38,5 +38,5 @@ def publish_options(self, fmt): return schema - def publish(self, options): + def publish(self, **kwargs): raise NotImplementedError() diff --git a/quest/services/base/service_base.py b/quest/services/base/service_base.py index 5dae3ace..27195b4d 100644 --- a/quest/services/base/service_base.py +++ b/quest/services/base/service_base.py @@ -92,7 +92,7 @@ def download_options(self, fmt): return schema - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): raise NotImplementedError() def get_features(self, **kwargs): @@ -138,7 +138,7 @@ class SingleFileServiceBase(ServiceBase): """Base file for datasets that are a single file download eg elevation raster etc """ - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): feature_id = util.construct_service_uri(self.provider.name, self.name, feature) feature = self.provider.get_features(self.name).loc[feature_id] reserved = feature.get('reserved') diff --git a/quest/services/cuahsi_hs.py b/quest/services/cuahsi_hs.py index 2557440d..100caf9e 100644 --- a/quest/services/cuahsi_hs.py +++ b/quest/services/cuahsi_hs.py @@ -9,6 +9,7 @@ import param import os + class HSServiceBase(SingleFileServiceBase): @property @@ -35,7 +36,7 @@ class HSGeoService(HSServiceBase): def get_features(self, **kwargs): - results = list(self.hs.resources()) + results = list(self.hs.resources(coverage_type="box", north="90", south="-90", east="180", west="-180")) if len(results) == 0: raise ValueError('No resource available from HydroShare.') @@ -126,21 +127,18 @@ class HSPublisher(PublishBase): 'Time Series': 'TimeSeriesResource' } - title = param.String(default="example title", doc="Title of resource", precedence=2) - abstract = param.String(default="example abstract", precedence=3, - doc="An description of the resource to be added to HydroShare.") - keywords = param.List(default=[], precedence=4, doc="list of keyword strings to describe the resource") - dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, precedence=5, - doc="dataset to publish to HydroShare") - resource_type = param.ObjectSelector(doc='parameter', precedence=1, objects=sorted(_resource_type_map.keys())) + resource_type = param.ObjectSelector(default=None, doc="", precedence=1, objects=sorted(_resource_type_map.keys())) + title = param.String(default="", doc="", precedence=2) + abstract = param.String(default="", doc="", precedence=3) + keywords = param.List(default=[], doc="", precedence=4) + dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, doc="", precedence=5) @property def hs(self): return self.provider.get_hs() - def publish(self, options=None): - - p = param.ParamOverrides(self, options) + def publish(self, **kwargs): + p = param.ParamOverrides(self, kwargs) valid_file_paths = [] valid_extensions = [] diff --git a/quest/services/kitware_girder.py b/quest/services/kitware_girder.py index 9a4bf34e..ba7bd674 100644 --- a/quest/services/kitware_girder.py +++ b/quest/services/kitware_girder.py @@ -34,9 +34,9 @@ class GirderPublisher(PublishBase): def gc(self): return self.provider.get_gc() - def publish(self, options=None): + def publish(self, **kwargs): try: - p = param.ParamOverrides(self, options) + p = param.ParamOverrides(self, kwargs) params = {'name': p.title, 'description': p.collection_description} resource_information_dict = self.gc.createResource(path='collection', params=params) folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], diff --git a/quest/services/noaa_coastwatch.py b/quest/services/noaa_coastwatch.py index bb79ba92..6e082007 100644 --- a/quest/services/noaa_coastwatch.py +++ b/quest/services/noaa_coastwatch.py @@ -65,8 +65,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/noaa_ncdc.py b/quest/services/noaa_ncdc.py index 5bd26238..6bbe1d5f 100644 --- a/quest/services/noaa_ncdc.py +++ b/quest/services/noaa_ncdc.py @@ -74,8 +74,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/template_service.py b/quest/services/template_service.py index 4d0cd8cf..b4e4ba6f 100644 --- a/quest/services/template_service.py +++ b/quest/services/template_service.py @@ -26,7 +26,7 @@ class ExampleServiceBase(ServiceBase): smtk_template = None _parameter_map = dict() - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwwargs): metadata = {} # get metadata from service data = None # data structure containing downloaded data diff --git a/quest/services/user_provider.py b/quest/services/user_provider.py index c6647215..4e525127 100644 --- a/quest/services/user_provider.py +++ b/quest/services/user_provider.py @@ -51,7 +51,7 @@ def instance(cls, service_name, service_data, provider, uri, is_remote): return self - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): if self.datasets_mapping is not None: fnames = self.datasets_mapping if isinstance(dict, self.datasets_mapping): diff --git a/quest/services/usgs_nwis.py b/quest/services/usgs_nwis.py index 08233a39..cc81b72b 100644 --- a/quest/services/usgs_nwis.py +++ b/quest/services/usgs_nwis.py @@ -18,8 +18,8 @@ class NwisServiceBase(TimePeriodServiceBase): period = param.String(default='P365D', precedence=4, doc='time period (e.g. P365D = 365 days or P4W = 4 weeks)') smtk_template = 'start_end_or_period.sbt' - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) parameter = p.parameter start = p.start diff --git a/quest/util/misc.py b/quest/util/misc.py index fac84738..5f03d405 100644 --- a/quest/util/misc.py +++ b/quest/util/misc.py @@ -19,6 +19,7 @@ from uuid import uuid4, UUID import quest +the_providers = None def generate_cache(update=False): """Downloads features for all services and caches results. @@ -238,22 +239,27 @@ def load_drivers(namespace, names=None): return {name: driver.DriverManager(namespace, name, invoke_on_load='True') for name in names} -def load_providers(): - settings = get_settings() +def load_providers(update_cache=False): + global the_providers - # add web services + settings = get_settings() web_services = list_drivers('services') web_services.remove('user') - providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} + if update_cache or the_providers is None: + providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} - if len(settings.get('USER_SERVICES', [])) > 0: - for uri in settings.get('USER_SERVICES', []): - try: - drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver - providers['user-' + drv.name] = drv - except Exception as e: - logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + if len(settings.get('USER_SERVICES', [])) > 0: + for uri in settings.get('USER_SERVICES', []): + try: + drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver + providers['user-' + drv.name] = drv + except Exception as e: + logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + + the_providers = providers + else: + providers = the_providers return providers From c48863846c89499910ca6902a5673d1c93b6a165 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 07/44] Dataset display name fix (More readable). --- quest/api/services.py | 2 - quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ quest/services/noaa_ncep.py | 80 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 quest/services/ISEP_Girder.py create mode 100644 quest/services/noaa_ncep.py diff --git a/quest/api/services.py b/quest/api/services.py index 1dd2a195..0fd2b26e 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -78,8 +78,6 @@ def get_publishers(expand=None, publisher_type=None): Returns: providers (list or dict,Default=list): list of all available providers - - """ providers = util.load_providers() publishers = {} diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py new file mode 100644 index 00000000..284823f9 --- /dev/null +++ b/quest/services/noaa_ncep.py @@ -0,0 +1,80 @@ +from .base import ProviderBase, ServiceBase +from ncep_client import NCEP_Client +from shapely.geometry import box +import pandas as pd +import param + + +class NCEPServiceBase(ServiceBase): + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class NCEP_GFS_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_gfs" + display_name = "NCEP GFS Service" + description = 'NCEP GFS is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = ncep.get_provider_types("Global Forecast System") + _possible_products = ncep.get_provider_products("Global Forecast System") + _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def get_features(self, **kwargs): + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "GFS" or p.product == "GDAS": + raise ValueError("Please specify a specific product not GFS or GDAS") + + results = ncep.get_ncep_product_data(ncep_provider="Global Forecast System", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEPProvider(ProviderBase): + service_base_class = NCEPServiceBase + publishers_list = [] + display_name = 'NCEP GFS Provider' + description = 'Services avaliable through the NOAA NCEP Server.' + organization_name = 'National Centers for Environmental Prediction' + organization_abbr = 'NCEP' + From f9121ce81ec211ef9a77bb99d4e8e31637cb5232 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 14:54:34 -0500 Subject: [PATCH 08/44] Added the noaa_ncep plugin and updated the conda environment file. --- py3_conda_environment.yml | 5 ++++- quest/services/noaa_ncep.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/py3_conda_environment.yml b/py3_conda_environment.yml index 07a0f9f7..e4b5d74e 100644 --- a/py3_conda_environment.yml +++ b/py3_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -41,4 +42,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 284823f9..9a9280b9 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -72,9 +72,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - publishers_list = [] display_name = 'NCEP GFS Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' - organization_abbr = 'NCEP' - + organization_abbr = 'NCEP' \ No newline at end of file From ab0920bcb5aaf7f7010e0fb55347f4872c860bcd Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 15:51:57 -0500 Subject: [PATCH 09/44] NCEP test and setup.cfg were updated. --- test/data.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/data.py b/test/data.py index 01bc2e35..22d6512f 100644 --- a/test/data.py +++ b/test/data.py @@ -203,6 +203,75 @@ 'title': 'NWIS Instantaneous Values Service Download Options'}, 'svc://cuahsi-hydroshare:hs_geo': {}, 'svc://cuahsi-hydroshare:hs_norm': {}, + 'svc://noaa-ncep:ncep_gfs': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [['enkf', 'enkf'], + ['gdas', 'gdas'], + ['gfs', 'gfs'], + ['gfsmos', 'gfsmos'], + ['sst', 'sst']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['GDAS', 'GDAS'], ['GFS', 'GFS']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [["0.50 deg 'full' file description", + "0.50 deg 'full' file description"], + ['32km Lambert Conformal grid', '32km Lambert Conformal grid'], + ['Atmospheric Analysis', 'Atmospheric Analysis'], + ['Atmospheric Model Data', 'Atmospheric Model Data'], + ['BUFR Sounding Files per Station', 'BUFR Sounding Files per Station'], + ['Binary Universal Form for the Representation of meteorological data (BUFR)', + 'Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['GDAS', 'GDAS'], + ['GFS', 'GFS'], + ['Global longitude-latitude grid', 'Global longitude-latitude grid'], + ['MDL Extratropical Storm Surge', 'MDL Extratropical Storm Surge'], + ['MOS Aviation Product', 'MOS Aviation Product'], + ['Prepared BUFR files', 'Prepared BUFR files'], + ['Pressure Level Data', 'Pressure Level Data'], + ['Sigma Atmospheric Model Data', 'Sigma Atmospheric Model Data'], + ['Smart Initialization Guam', 'Smart Initialization Guam'], + ['Surface Analysis', 'Surface Analysis'], + ['Surface Boundary Conditions', 'Surface Boundary Conditions'], + ['Surface Flux', 'Surface Flux'], + ['T1534 Semi-Lagrangian grid', 'T1534 Semi-Lagrangian grid'], + ['Time Dependent Satellite Bias Correction', + 'Time Dependent Satellite Bias Correction'], + ['Tropical Cyclone Vital Statistics', + 'Tropical Cyclone Vital Statistics'], + ['WAFS/ICAO/International Exchange/FOS Grids', + 'WAFS/ICAO/International Exchange/FOS Grids'], + ['World Area Forecast System', 'World Area Forecast System'], + ['global longitude-latitude grid (1.0 deg)', + 'global longitude-latitude grid (1.0 deg)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP GFS Service Download Options'} } From e48dab0115b1c1d4663d74950c6df7e18e9066f5 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 10/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From b586e500be5289e61e4ce6e8ba85aec5cb0cb66f Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:40:01 -0500 Subject: [PATCH 11/44] Updated the docs and the python 2 environment file for ncep and terrapin. --- docs/install.rst | 9 +++------ py2_conda_environment.yml | 5 ++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index eb4c8416..08123b28 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,11 +12,11 @@ Install Released Conda Package a. Install Quest into new environment:: - conda create -n quest -c conda-forge -c erdc quest terrapin + conda create -n quest -c conda-forge -c erdc quest b. Install Quest into existing environment:: - conda install -c erdc -c conda-forge quest terrapin + conda install -c erdc -c conda-forge quest .. note:: @@ -49,11 +49,8 @@ Conda Install Optional ........ - d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel:: - conda install -c erdc -c conda-forge terrapin - - e. Run tests:: + d. Run tests:: python setup.py test diff --git a/py2_conda_environment.yml b/py2_conda_environment.yml index 67ae4cb0..638fb57d 100644 --- a/py2_conda_environment.yml +++ b/py2_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -40,4 +41,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file From f076882e0fc1c563f93132a8c95a7c8587f5ff3c Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 13:12:21 -0500 Subject: [PATCH 12/44] The setup.cfg never got updated or added to this commit. This fixes the tests. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 97e56815..89448a07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = quest.services = user = quest.services.user_provider:UserProvider noaa-coast = quest.services.noaa_coastwatch:NoaaProvider + noaa-ncep = quest.services.noaa_ncep:NCEPProvider noaa-ncdc = quest.services.noaa_ncdc:NcdcProvider usgs-nwis = quest.services.usgs_nwis:NwisProvider usgs-ned = quest.services.usgs_ned:UsgsNedProvider From 649678e567a7b534d08cf95754b73d0c21ee9f30 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 15:00:52 -0500 Subject: [PATCH 13/44] Addeed another service from ncep, and added the sorted call to the lists. --- quest/services/noaa_ncep.py | 68 ++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 9a9280b9..d1fab433 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -8,7 +8,9 @@ class NCEPServiceBase(ServiceBase): def get_features(self, **kwargs): - raise NotImplementedError() + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature class NCEP_GFS_Service(NCEPServiceBase): @@ -23,9 +25,9 @@ class NCEP_GFS_Service(NCEPServiceBase): ] feature_id = "ncep" # These are slowing down the api because it has to load the web page. - _possible_types = ncep.get_provider_types("Global Forecast System") - _possible_products = ncep.get_provider_products("Global Forecast System") - _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _possible_types = sorted(ncep.get_provider_types("Global Forecast System")) + _possible_products = sorted(ncep.get_provider_products("Global Forecast System")) + _possible_formats = sorted(ncep.get_formats_of_a_product("Global Forecast System")) _parameter_map = {} date = param.String(default=None, doc="YYYYMMDD", precedence=1) @@ -37,11 +39,6 @@ class NCEP_GFS_Service(NCEPServiceBase): type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) - def get_features(self, **kwargs): - the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} - feature = pd.DataFrame(the_feature, index=[0]) - return feature - def download(self, feature, file_path, dataset, **params): ncep = NCEP_Client() p = param.ParamOverrides(self, params) @@ -70,6 +67,59 @@ def download(self, feature, file_path, dataset, **params): return metadata +class NCEP_NAM_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_nam" + display_name = "NCEP NAM Service" + description = 'NCEP NAM is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = sorted(ncep.get_provider_types("North American Model")) + _possible_products = sorted(ncep.get_provider_products("North American Model")) + _possible_formats = sorted(ncep.get_formats_of_a_product("North American Model")) + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "NAM": + raise ValueError("Please specify a specific product not NAM") + + results = ncep.get_ncep_product_data(ncep_provider="North American Model", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' From 0466a295d0f1c252e53e743151c69feb92c05d54 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Tue, 17 Apr 2018 08:33:34 -0500 Subject: [PATCH 14/44] Fixed the test and added the sorted functions into this plugin. --- quest/services/noaa_ncep.py | 1 + test/data.py | 111 ++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index d1fab433..c67d475a 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -120,6 +120,7 @@ def download(self, feature, file_path, dataset, **params): return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' diff --git a/test/data.py b/test/data.py index 22d6512f..46acdfbe 100644 --- a/test/data.py +++ b/test/data.py @@ -226,11 +226,7 @@ {'default': None, 'description': 'Paramerter', 'name': 'format', - 'range': [['enkf', 'enkf'], - ['gdas', 'gdas'], - ['gfs', 'gfs'], - ['gfsmos', 'gfsmos'], - ['sst', 'sst']], + 'range': [], 'type': 'ObjectSelector'}, {'default': None, 'description': 'Parameter2', @@ -271,7 +267,110 @@ ['global longitude-latitude grid (1.0 deg)', 'global longitude-latitude grid (1.0 deg)']], 'type': 'ObjectSelector'}], - 'title': 'NCEP GFS Service Download Options'} + 'title': 'NCEP GFS Service Download Options'}, + 'svc://noaa-ncep:ncep_nam': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['NAM', 'NAM']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [['NAM', 'NAM'], + ['NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)', + 'NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)'], + ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', + 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', + 'NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))'], + ['NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)', + 'NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)'], + ['NAM 182 AFWA Grid - North Pacific (12-km Resolution)', + 'NAM 182 AFWA Grid - North Pacific (12-km Resolution)'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))'], + ['NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)', + 'NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)'], + ['NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))', + 'NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))'], + ['NAM 215 AWIPS Grid - CONUS (20-km Resolution)', + 'NAM 215 AWIPS Grid - CONUS (20-km Resolution)'], + ['NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)', + 'NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)', + 'NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)'], + ['NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))', + 'NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))'], + ['NAM IMS Snow Grid (24-km Resolution)', + 'NAM IMS Snow Grid (24-km Resolution)'], + ['NAM MOS', 'NAM MOS'], + ['NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)', + 'NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)'], + ['NAM NEST over ALASKA (6 km Resolution - Grid 198)', + 'NAM NEST over ALASKA (6 km Resolution - Grid 198)'], + ['NAM NEST over CONUS (5 km Resolution - Grid 227)', + 'NAM NEST over CONUS (5 km Resolution - Grid 227)'], + ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', + 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], + ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)'], + ['NAM Precipitation Analysis (4km Analysis mapped to 15km grid)', + 'NAM Precipitation Analysis (4km Analysis mapped to 15km grid)'], + ['NAM Precipitation Analysis (Stage 4)', + 'NAM Precipitation Analysis (Stage 4)'], + ['NAM Radar and Gauge Analysis under Stage2', + 'NAM Radar and Gauge Analysis under Stage2']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP NAM Service Download Options'} } From cccb4963df4afbd1764bda0b74f2fc56ed09f07d Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Wed, 18 Apr 2018 08:22:42 -0500 Subject: [PATCH 15/44] Nasa Provider Plugin (#40) * I got the basic authentication piece setup for the nasa provider. * Made a small fix to the authentication fix. This provider is working again. * Nasa provider syntax fix. * Minor updates (#38) * Added the cuahsi coverage fix. The change was not getting all the resources and just getting all resources that had a coverage from the world. * Made kwargs the standard over using options, params, and kwargs. * Updated the load_providers to only update if update-cache is True or if the global is None. * I got the basic authentication piece setup for the nasa provider. * Made a small fix to the authentication fix. This provider is working again. * Nasa provider syntax fix. * I never added the uncommented the nasa download options to the tests in data.py. This fixes the tests. --- quest/services/nasa.py | 45 ++++++++++++++++++++++++++++++++++++++++-- setup.cfg | 2 +- test/data.py | 4 ++-- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/quest/services/nasa.py b/quest/services/nasa.py index 05f2e3cf..3bbe1a46 100644 --- a/quest/services/nasa.py +++ b/quest/services/nasa.py @@ -2,6 +2,8 @@ """ from .base import ProviderBase, SingleFileServiceBase +from ..api.database import get_db, db_session +from getpass import getpass import pandas as pd import requests @@ -20,8 +22,15 @@ class NasaServiceBase(SingleFileServiceBase): 'elevation': 'elevation' } - def _read_granules(short_name, page_num): - return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry'] + @property + def info(self): + return self.provider.get_user_info() + + def _read_granules(self, short_name, page_num): + try: + return requests.get(granules_url % (short_name, page_num), auth=(self.info['username'], self.info['password'])).json()['feed']['entry'] + except ValueError: + return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry'] def get_features(self, **kwargs): page_num = 0 @@ -128,3 +137,35 @@ class NasaProvider(ProviderBase): description = 'Services available through the NASA' organization_name = 'National Aeronautic and Space Administration' organization_abbr = 'NASA' + + def get_user_info(self): + the_info = self.credentials + return the_info + + def authenticate_me(self, **kwargs): + + username = input("Enter Username: ") + password = getpass("Enter Password: ") + + try: + db = get_db() + with db_session: + p = db.Providers.select().filter(provider=self.name).first() + + provider_metadata = { + 'provider': self.name, + 'username': username, + 'password': password, + } + + if p is None: + db.Providers(**provider_metadata) + else: + p.set(**provider_metadata) + + return True + + except: + print("Either credentials invalid or unable to connect to HydroShare.") + + return False diff --git a/setup.cfg b/setup.cfg index 97e56815..4525e6f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ quest.services = usgs-nlcd = quest.services.usgs_nlcd:UsgsNlcdProvider cuahsi-hydroshare = quest.services.cuahsi_hs:HSProvider kitware-girder = quest.services.kitware_girder:GirderProvider - # nasa = quest.services.nasa:NasaProvider + nasa = quest.services.nasa:NasaProvider quest.filters = ts-unit-conversion = quest.filters.timeseries.timeseries:TsUnitConversion diff --git a/test/data.py b/test/data.py index 01bc2e35..fc367886 100644 --- a/test/data.py +++ b/test/data.py @@ -3,8 +3,8 @@ """ DOWNLOAD_OPTIONS_FROM_ALL_SERVICES = { - # 'svc://nasa:srtm-3-arc-second': {}, - # 'svc://nasa:srtm-30-arc-second': {}, + 'svc://nasa:srtm-3-arc-second': {}, + 'svc://nasa:srtm-30-arc-second': {}, 'svc://noaa-ncdc:ghcn-daily': {'properties': [{'default': None, 'description': 'parameter', 'name': 'parameter', From ff48887ab5f26ee59f2b455d8d66acb865730fb5 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 16/44] Dataset display name fix (More readable). --- quest/api/services.py | 2 - quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ quest/services/noaa_ncep.py | 80 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 quest/services/ISEP_Girder.py create mode 100644 quest/services/noaa_ncep.py diff --git a/quest/api/services.py b/quest/api/services.py index 1dd2a195..0fd2b26e 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -78,8 +78,6 @@ def get_publishers(expand=None, publisher_type=None): Returns: providers (list or dict,Default=list): list of all available providers - - """ providers = util.load_providers() publishers = {} diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py new file mode 100644 index 00000000..284823f9 --- /dev/null +++ b/quest/services/noaa_ncep.py @@ -0,0 +1,80 @@ +from .base import ProviderBase, ServiceBase +from ncep_client import NCEP_Client +from shapely.geometry import box +import pandas as pd +import param + + +class NCEPServiceBase(ServiceBase): + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class NCEP_GFS_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_gfs" + display_name = "NCEP GFS Service" + description = 'NCEP GFS is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = ncep.get_provider_types("Global Forecast System") + _possible_products = ncep.get_provider_products("Global Forecast System") + _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def get_features(self, **kwargs): + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "GFS" or p.product == "GDAS": + raise ValueError("Please specify a specific product not GFS or GDAS") + + results = ncep.get_ncep_product_data(ncep_provider="Global Forecast System", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEPProvider(ProviderBase): + service_base_class = NCEPServiceBase + publishers_list = [] + display_name = 'NCEP GFS Provider' + description = 'Services avaliable through the NOAA NCEP Server.' + organization_name = 'National Centers for Environmental Prediction' + organization_abbr = 'NCEP' + From 6e7f1a8f5dbfd7b31766c71d28d7c670317781f1 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 14:54:34 -0500 Subject: [PATCH 17/44] Added the noaa_ncep plugin and updated the conda environment file. --- py3_conda_environment.yml | 5 ++++- quest/services/noaa_ncep.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/py3_conda_environment.yml b/py3_conda_environment.yml index 07a0f9f7..e4b5d74e 100644 --- a/py3_conda_environment.yml +++ b/py3_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -41,4 +42,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 284823f9..9a9280b9 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -72,9 +72,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - publishers_list = [] display_name = 'NCEP GFS Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' - organization_abbr = 'NCEP' - + organization_abbr = 'NCEP' \ No newline at end of file From 9b23c1bd2121f2c874b6ea9e3ad7bddbf4b389b0 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 15:51:57 -0500 Subject: [PATCH 18/44] NCEP test and setup.cfg were updated. --- test/data.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/data.py b/test/data.py index fc367886..e0eff4c8 100644 --- a/test/data.py +++ b/test/data.py @@ -203,6 +203,75 @@ 'title': 'NWIS Instantaneous Values Service Download Options'}, 'svc://cuahsi-hydroshare:hs_geo': {}, 'svc://cuahsi-hydroshare:hs_norm': {}, + 'svc://noaa-ncep:ncep_gfs': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [['enkf', 'enkf'], + ['gdas', 'gdas'], + ['gfs', 'gfs'], + ['gfsmos', 'gfsmos'], + ['sst', 'sst']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['GDAS', 'GDAS'], ['GFS', 'GFS']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [["0.50 deg 'full' file description", + "0.50 deg 'full' file description"], + ['32km Lambert Conformal grid', '32km Lambert Conformal grid'], + ['Atmospheric Analysis', 'Atmospheric Analysis'], + ['Atmospheric Model Data', 'Atmospheric Model Data'], + ['BUFR Sounding Files per Station', 'BUFR Sounding Files per Station'], + ['Binary Universal Form for the Representation of meteorological data (BUFR)', + 'Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['GDAS', 'GDAS'], + ['GFS', 'GFS'], + ['Global longitude-latitude grid', 'Global longitude-latitude grid'], + ['MDL Extratropical Storm Surge', 'MDL Extratropical Storm Surge'], + ['MOS Aviation Product', 'MOS Aviation Product'], + ['Prepared BUFR files', 'Prepared BUFR files'], + ['Pressure Level Data', 'Pressure Level Data'], + ['Sigma Atmospheric Model Data', 'Sigma Atmospheric Model Data'], + ['Smart Initialization Guam', 'Smart Initialization Guam'], + ['Surface Analysis', 'Surface Analysis'], + ['Surface Boundary Conditions', 'Surface Boundary Conditions'], + ['Surface Flux', 'Surface Flux'], + ['T1534 Semi-Lagrangian grid', 'T1534 Semi-Lagrangian grid'], + ['Time Dependent Satellite Bias Correction', + 'Time Dependent Satellite Bias Correction'], + ['Tropical Cyclone Vital Statistics', + 'Tropical Cyclone Vital Statistics'], + ['WAFS/ICAO/International Exchange/FOS Grids', + 'WAFS/ICAO/International Exchange/FOS Grids'], + ['World Area Forecast System', 'World Area Forecast System'], + ['global longitude-latitude grid (1.0 deg)', + 'global longitude-latitude grid (1.0 deg)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP GFS Service Download Options'} } From f40a67ed3256658c26297b397a3bb9aa9113bd08 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 19/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From 2ff755cf1a8c0e47b7e62a3f94c9c93ec90a5062 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:40:01 -0500 Subject: [PATCH 20/44] Updated the docs and the python 2 environment file for ncep and terrapin. --- docs/install.rst | 9 +++------ py2_conda_environment.yml | 5 ++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index eb4c8416..08123b28 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,11 +12,11 @@ Install Released Conda Package a. Install Quest into new environment:: - conda create -n quest -c conda-forge -c erdc quest terrapin + conda create -n quest -c conda-forge -c erdc quest b. Install Quest into existing environment:: - conda install -c erdc -c conda-forge quest terrapin + conda install -c erdc -c conda-forge quest .. note:: @@ -49,11 +49,8 @@ Conda Install Optional ........ - d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel:: - conda install -c erdc -c conda-forge terrapin - - e. Run tests:: + d. Run tests:: python setup.py test diff --git a/py2_conda_environment.yml b/py2_conda_environment.yml index 67ae4cb0..638fb57d 100644 --- a/py2_conda_environment.yml +++ b/py2_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -40,4 +41,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file From e90f09b0ba41e330a3b1e2b4b70c4eea38ee256a Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 21/44] Dataset display name fix (More readable). --- quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + From 3906d8785d764f1611098bdb4a4f3d30904b9592 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 22/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From cb32c405b78b0ac1b8067a202ed7efceba834ccd Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 13:12:21 -0500 Subject: [PATCH 23/44] The setup.cfg never got updated or added to this commit. This fixes the tests. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 4525e6f9..7f02c18a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = quest.services = user = quest.services.user_provider:UserProvider noaa-coast = quest.services.noaa_coastwatch:NoaaProvider + noaa-ncep = quest.services.noaa_ncep:NCEPProvider noaa-ncdc = quest.services.noaa_ncdc:NcdcProvider usgs-nwis = quest.services.usgs_nwis:NwisProvider usgs-ned = quest.services.usgs_ned:UsgsNedProvider From 34c36239cf43e7eca433caf569297aa771db6105 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 15:00:52 -0500 Subject: [PATCH 24/44] Addeed another service from ncep, and added the sorted call to the lists. --- quest/services/noaa_ncep.py | 68 ++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 9a9280b9..d1fab433 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -8,7 +8,9 @@ class NCEPServiceBase(ServiceBase): def get_features(self, **kwargs): - raise NotImplementedError() + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature class NCEP_GFS_Service(NCEPServiceBase): @@ -23,9 +25,9 @@ class NCEP_GFS_Service(NCEPServiceBase): ] feature_id = "ncep" # These are slowing down the api because it has to load the web page. - _possible_types = ncep.get_provider_types("Global Forecast System") - _possible_products = ncep.get_provider_products("Global Forecast System") - _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _possible_types = sorted(ncep.get_provider_types("Global Forecast System")) + _possible_products = sorted(ncep.get_provider_products("Global Forecast System")) + _possible_formats = sorted(ncep.get_formats_of_a_product("Global Forecast System")) _parameter_map = {} date = param.String(default=None, doc="YYYYMMDD", precedence=1) @@ -37,11 +39,6 @@ class NCEP_GFS_Service(NCEPServiceBase): type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) - def get_features(self, **kwargs): - the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} - feature = pd.DataFrame(the_feature, index=[0]) - return feature - def download(self, feature, file_path, dataset, **params): ncep = NCEP_Client() p = param.ParamOverrides(self, params) @@ -70,6 +67,59 @@ def download(self, feature, file_path, dataset, **params): return metadata +class NCEP_NAM_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_nam" + display_name = "NCEP NAM Service" + description = 'NCEP NAM is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = sorted(ncep.get_provider_types("North American Model")) + _possible_products = sorted(ncep.get_provider_products("North American Model")) + _possible_formats = sorted(ncep.get_formats_of_a_product("North American Model")) + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "NAM": + raise ValueError("Please specify a specific product not NAM") + + results = ncep.get_ncep_product_data(ncep_provider="North American Model", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' From 9133312459e8d3d6a28557eb574e7c77d7731ba9 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Tue, 17 Apr 2018 08:33:34 -0500 Subject: [PATCH 25/44] Fixed the test and added the sorted functions into this plugin. --- quest/services/noaa_ncep.py | 1 + test/data.py | 111 ++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index d1fab433..c67d475a 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -120,6 +120,7 @@ def download(self, feature, file_path, dataset, **params): return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' diff --git a/test/data.py b/test/data.py index e0eff4c8..34883ab9 100644 --- a/test/data.py +++ b/test/data.py @@ -226,11 +226,7 @@ {'default': None, 'description': 'Paramerter', 'name': 'format', - 'range': [['enkf', 'enkf'], - ['gdas', 'gdas'], - ['gfs', 'gfs'], - ['gfsmos', 'gfsmos'], - ['sst', 'sst']], + 'range': [], 'type': 'ObjectSelector'}, {'default': None, 'description': 'Parameter2', @@ -271,7 +267,110 @@ ['global longitude-latitude grid (1.0 deg)', 'global longitude-latitude grid (1.0 deg)']], 'type': 'ObjectSelector'}], - 'title': 'NCEP GFS Service Download Options'} + 'title': 'NCEP GFS Service Download Options'}, + 'svc://noaa-ncep:ncep_nam': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['NAM', 'NAM']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [['NAM', 'NAM'], + ['NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)', + 'NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)'], + ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', + 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', + 'NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))'], + ['NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)', + 'NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)'], + ['NAM 182 AFWA Grid - North Pacific (12-km Resolution)', + 'NAM 182 AFWA Grid - North Pacific (12-km Resolution)'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))'], + ['NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)', + 'NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)'], + ['NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))', + 'NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))'], + ['NAM 215 AWIPS Grid - CONUS (20-km Resolution)', + 'NAM 215 AWIPS Grid - CONUS (20-km Resolution)'], + ['NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)', + 'NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)', + 'NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)'], + ['NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))', + 'NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))'], + ['NAM IMS Snow Grid (24-km Resolution)', + 'NAM IMS Snow Grid (24-km Resolution)'], + ['NAM MOS', 'NAM MOS'], + ['NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)', + 'NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)'], + ['NAM NEST over ALASKA (6 km Resolution - Grid 198)', + 'NAM NEST over ALASKA (6 km Resolution - Grid 198)'], + ['NAM NEST over CONUS (5 km Resolution - Grid 227)', + 'NAM NEST over CONUS (5 km Resolution - Grid 227)'], + ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', + 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], + ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)'], + ['NAM Precipitation Analysis (4km Analysis mapped to 15km grid)', + 'NAM Precipitation Analysis (4km Analysis mapped to 15km grid)'], + ['NAM Precipitation Analysis (Stage 4)', + 'NAM Precipitation Analysis (Stage 4)'], + ['NAM Radar and Gauge Analysis under Stage2', + 'NAM Radar and Gauge Analysis under Stage2']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP NAM Service Download Options'} } From 5e3f17e0a785c045fa34bd6db420667dd38006cb Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Thu, 19 Apr 2018 11:43:22 -0500 Subject: [PATCH 26/44] Fixed the test for ncep provider. --- quest/services/noaa_ncep.py | 2 +- quest/util/config.py | 3 +++ test/data.py | 9 ++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index c67d475a..287c0345 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -123,7 +123,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - display_name = 'NCEP GFS Provider' + display_name = 'NCEP Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' organization_abbr = 'NCEP' \ No newline at end of file diff --git a/quest/util/config.py b/quest/util/config.py index 9dbdcd18..ae045744 100644 --- a/quest/util/config.py +++ b/quest/util/config.py @@ -80,6 +80,9 @@ def update_settings(config={}): from .. import init init() + from . import load_providers + load_providers(update_cache=True) + return True diff --git a/test/data.py b/test/data.py index 34883ab9..0b2855da 100644 --- a/test/data.py +++ b/test/data.py @@ -373,10 +373,9 @@ 'title': 'NCEP NAM Service Download Options'} } - SERVICES_FEATURE_COUNT = [ - # ('svc://nasa:srtm-3-arc-second', 14297, 1000), - # ('svc://nasa:srtm-30-arc-second', 27, 10), + ('svc://nasa:srtm-3-arc-second', 14297, 1000), + ('svc://nasa:srtm-30-arc-second', 27, 10), ('svc://noaa-ncdc:ghcn-daily', 104126, 5000), ('svc://noaa-ncdc:gsod', 28754, 1500), ('svc://noaa-coast:coops-meteorological', 375, 50), @@ -421,8 +420,8 @@ } SERVICE_FEATURE_DOWNLOAD_OPTIONS = [ - # ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), - # ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), + ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), + ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), ('svc://noaa-ncdc:ghcn-daily/ACW00011604', {'parameter': 'air_temperature:daily:total', 'start': '1949-01-01', 'end': '1949-01-02'}), ('svc://noaa-ncdc:gsod/717580-99999', {'parameter': 'air_temperature:daily:max', 'start': '2016-01-01', 'end': '2016-01-02'}), ('svc://noaa-coast:coops-meteorological/1611400', {'parameter': 'air_temperature', 'start': '2015-05-23', 'end': '2015-05-24'}), From 313d6d05a6febd39182835f7838c45a20dc4eb15 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Thu, 19 Apr 2018 11:50:09 -0500 Subject: [PATCH 27/44] Fixed the test for ncep provider part 2. --- test/data.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/data.py b/test/data.py index 0b2855da..05cbe56f 100644 --- a/test/data.py +++ b/test/data.py @@ -302,8 +302,6 @@ 'description': 'Parameter3', 'name': 'product', 'range': [['NAM', 'NAM'], - ['NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)', - 'NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)'], ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', @@ -362,13 +360,7 @@ ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', - 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)'], - ['NAM Precipitation Analysis (4km Analysis mapped to 15km grid)', - 'NAM Precipitation Analysis (4km Analysis mapped to 15km grid)'], - ['NAM Precipitation Analysis (Stage 4)', - 'NAM Precipitation Analysis (Stage 4)'], - ['NAM Radar and Gauge Analysis under Stage2', - 'NAM Radar and Gauge Analysis under Stage2']], + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)']], 'type': 'ObjectSelector'}], 'title': 'NCEP NAM Service Download Options'} } From d2af8abc839785c369a4521b680e758105d5c97d Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Fri, 20 Apr 2018 16:04:45 -0500 Subject: [PATCH 28/44] Updated the authentication to work with QuestWeb, and added a method for getting the status of a providers authentication. (#41) --- quest/api/__init__.py | 3 ++- quest/api/services.py | 26 ++++++++++++++++++++++++-- quest/services/cuahsi_hs.py | 5 ++--- quest/services/kitware_girder.py | 4 ++-- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/quest/api/__init__.py b/quest/api/__init__.py index b6254bd4..51e92e10 100644 --- a/quest/api/__init__.py +++ b/quest/api/__init__.py @@ -25,6 +25,7 @@ 'download_options', 'get_active_project', 'get_api_version', + 'get_auth_status', 'get_collections', 'get_datasets', 'get_features', @@ -56,7 +57,6 @@ 'save_settings', 'set_active_project', 'stage_for_download', - 'stage_for_publish', 'unauthenticate_provider', 'update_metadata', # replaces update_collection, update_feature, update_dataset 'update_settings', @@ -150,6 +150,7 @@ get_services, add_provider, delete_provider, + get_auth_status, authenticate_provider, unauthenticate_provider, ) diff --git a/quest/api/services.py b/quest/api/services.py index 1dd2a195..0b39c6dc 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -4,6 +4,7 @@ """ from __future__ import absolute_import from __future__ import print_function +from ..api.database import get_db, db_session from .. import util import os import requests @@ -167,7 +168,28 @@ def delete_provider(uri): return msg -def authenticate_provider(uri): +def get_auth_status(uri): + """Check to see if a provider has been authenticated + + Args: + uri (string, Required): + uri of 'user service' + Returns: + True on success + False on not authenticated + + """ + db = get_db() + with db_session: + p = db.Providers.select().filter(provider=uri).first() + + if p is None: + return False + + return True + + +def authenticate_provider(uri, **kwargs): """Authenticate the user. Args: @@ -178,7 +200,7 @@ def authenticate_provider(uri): """ driver = util.load_providers()[uri] - driver.authenticate_me() + driver.authenticate_me(**kwargs) def unauthenticate_provider(uri): diff --git a/quest/services/cuahsi_hs.py b/quest/services/cuahsi_hs.py index 100caf9e..ec33c387 100644 --- a/quest/services/cuahsi_hs.py +++ b/quest/services/cuahsi_hs.py @@ -4,7 +4,6 @@ from shapely.geometry import Point, box from ..api.metadata import get_metadata from ..util import param_util -from getpass import getpass import pandas as pd import param import os @@ -260,8 +259,8 @@ def get_hs(self, auth=None, require_valid_auth=False): def authenticate_me(self, **kwargs): - username = input("Enter Username: ") - password = getpass("Enter Password: ") + username = kwargs['username'] + password = kwargs['password'] try: auth = HydroShareAuthBasic(username=username, password=password) diff --git a/quest/services/kitware_girder.py b/quest/services/kitware_girder.py index ba7bd674..af1f8e6c 100644 --- a/quest/services/kitware_girder.py +++ b/quest/services/kitware_girder.py @@ -77,8 +77,8 @@ def get_gc(self): def authenticate_me(self, **kwargs): connection_info = 'https://data.kitware.com/api/v1' - username = input("Enter Username: ") - password = getpass("Enter Password: ") + username = kwargs['username'] + password = kwargs['password'] try: gc = girder_client.GirderClient(apiUrl=connection_info) From 3289b3ec992f5db1a889e5f92a131077624ef972 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 29/44] Dataset display name fix (More readable). --- quest/api/services.py | 2 - quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ quest/services/noaa_ncep.py | 80 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 quest/services/ISEP_Girder.py create mode 100644 quest/services/noaa_ncep.py diff --git a/quest/api/services.py b/quest/api/services.py index 0b39c6dc..fbd3001a 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -79,8 +79,6 @@ def get_publishers(expand=None, publisher_type=None): Returns: providers (list or dict,Default=list): list of all available providers - - """ providers = util.load_providers() publishers = {} diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py new file mode 100644 index 00000000..284823f9 --- /dev/null +++ b/quest/services/noaa_ncep.py @@ -0,0 +1,80 @@ +from .base import ProviderBase, ServiceBase +from ncep_client import NCEP_Client +from shapely.geometry import box +import pandas as pd +import param + + +class NCEPServiceBase(ServiceBase): + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class NCEP_GFS_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_gfs" + display_name = "NCEP GFS Service" + description = 'NCEP GFS is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = ncep.get_provider_types("Global Forecast System") + _possible_products = ncep.get_provider_products("Global Forecast System") + _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def get_features(self, **kwargs): + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "GFS" or p.product == "GDAS": + raise ValueError("Please specify a specific product not GFS or GDAS") + + results = ncep.get_ncep_product_data(ncep_provider="Global Forecast System", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEPProvider(ProviderBase): + service_base_class = NCEPServiceBase + publishers_list = [] + display_name = 'NCEP GFS Provider' + description = 'Services avaliable through the NOAA NCEP Server.' + organization_name = 'National Centers for Environmental Prediction' + organization_abbr = 'NCEP' + From 059fec608a2d94542d985aada73db806c9e73828 Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 14:54:34 -0500 Subject: [PATCH 30/44] Added the noaa_ncep plugin and updated the conda environment file. --- py3_conda_environment.yml | 5 ++++- quest/services/noaa_ncep.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/py3_conda_environment.yml b/py3_conda_environment.yml index 07a0f9f7..e4b5d74e 100644 --- a/py3_conda_environment.yml +++ b/py3_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -41,4 +42,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 284823f9..9a9280b9 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -72,9 +72,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - publishers_list = [] display_name = 'NCEP GFS Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' - organization_abbr = 'NCEP' - + organization_abbr = 'NCEP' \ No newline at end of file From 47b067d4143b2fdebe65f9492227f2007762724e Mon Sep 17 00:00:00 2001 From: Aaron Valoroso Date: Mon, 9 Apr 2018 15:51:57 -0500 Subject: [PATCH 31/44] NCEP test and setup.cfg were updated. --- test/data.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/data.py b/test/data.py index fc367886..e0eff4c8 100644 --- a/test/data.py +++ b/test/data.py @@ -203,6 +203,75 @@ 'title': 'NWIS Instantaneous Values Service Download Options'}, 'svc://cuahsi-hydroshare:hs_geo': {}, 'svc://cuahsi-hydroshare:hs_norm': {}, + 'svc://noaa-ncep:ncep_gfs': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [['enkf', 'enkf'], + ['gdas', 'gdas'], + ['gfs', 'gfs'], + ['gfsmos', 'gfsmos'], + ['sst', 'sst']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['GDAS', 'GDAS'], ['GFS', 'GFS']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [["0.50 deg 'full' file description", + "0.50 deg 'full' file description"], + ['32km Lambert Conformal grid', '32km Lambert Conformal grid'], + ['Atmospheric Analysis', 'Atmospheric Analysis'], + ['Atmospheric Model Data', 'Atmospheric Model Data'], + ['BUFR Sounding Files per Station', 'BUFR Sounding Files per Station'], + ['Binary Universal Form for the Representation of meteorological data (BUFR)', + 'Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['GDAS', 'GDAS'], + ['GFS', 'GFS'], + ['Global longitude-latitude grid', 'Global longitude-latitude grid'], + ['MDL Extratropical Storm Surge', 'MDL Extratropical Storm Surge'], + ['MOS Aviation Product', 'MOS Aviation Product'], + ['Prepared BUFR files', 'Prepared BUFR files'], + ['Pressure Level Data', 'Pressure Level Data'], + ['Sigma Atmospheric Model Data', 'Sigma Atmospheric Model Data'], + ['Smart Initialization Guam', 'Smart Initialization Guam'], + ['Surface Analysis', 'Surface Analysis'], + ['Surface Boundary Conditions', 'Surface Boundary Conditions'], + ['Surface Flux', 'Surface Flux'], + ['T1534 Semi-Lagrangian grid', 'T1534 Semi-Lagrangian grid'], + ['Time Dependent Satellite Bias Correction', + 'Time Dependent Satellite Bias Correction'], + ['Tropical Cyclone Vital Statistics', + 'Tropical Cyclone Vital Statistics'], + ['WAFS/ICAO/International Exchange/FOS Grids', + 'WAFS/ICAO/International Exchange/FOS Grids'], + ['World Area Forecast System', 'World Area Forecast System'], + ['global longitude-latitude grid (1.0 deg)', + 'global longitude-latitude grid (1.0 deg)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP GFS Service Download Options'} } From fdb30cd0c1f3fc0dd26c6ea90af4d92750015d16 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 32/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From e928842ea32d33fda3022d363b8178d6fb21bcad Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:40:01 -0500 Subject: [PATCH 33/44] Updated the docs and the python 2 environment file for ncep and terrapin. --- docs/install.rst | 9 +++------ py2_conda_environment.yml | 5 ++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index eb4c8416..08123b28 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,11 +12,11 @@ Install Released Conda Package a. Install Quest into new environment:: - conda create -n quest -c conda-forge -c erdc quest terrapin + conda create -n quest -c conda-forge -c erdc quest b. Install Quest into existing environment:: - conda install -c erdc -c conda-forge quest terrapin + conda install -c erdc -c conda-forge quest .. note:: @@ -49,11 +49,8 @@ Conda Install Optional ........ - d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel:: - conda install -c erdc -c conda-forge terrapin - - e. Run tests:: + d. Run tests:: python setup.py test diff --git a/py2_conda_environment.yml b/py2_conda_environment.yml index 67ae4cb0..638fb57d 100644 --- a/py2_conda_environment.yml +++ b/py2_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -40,4 +41,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file From fa8d7dfb459a893fdec754da9e6c2bf8c9add20b Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 34/44] Dataset display name fix (More readable). --- quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + From a5efdef957e811fc7720d5fa86e4d4d6af39b935 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 35/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From ba740799eb331b5946eef2369ef575a6e6295650 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 13:12:21 -0500 Subject: [PATCH 36/44] The setup.cfg never got updated or added to this commit. This fixes the tests. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 4525e6f9..7f02c18a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = quest.services = user = quest.services.user_provider:UserProvider noaa-coast = quest.services.noaa_coastwatch:NoaaProvider + noaa-ncep = quest.services.noaa_ncep:NCEPProvider noaa-ncdc = quest.services.noaa_ncdc:NcdcProvider usgs-nwis = quest.services.usgs_nwis:NwisProvider usgs-ned = quest.services.usgs_ned:UsgsNedProvider From 00bf61017dd401307e9eb686a4ef48a02371c65a Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 16 Apr 2018 15:00:52 -0500 Subject: [PATCH 37/44] Addeed another service from ncep, and added the sorted call to the lists. --- quest/services/noaa_ncep.py | 68 ++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index 9a9280b9..d1fab433 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -8,7 +8,9 @@ class NCEPServiceBase(ServiceBase): def get_features(self, **kwargs): - raise NotImplementedError() + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature class NCEP_GFS_Service(NCEPServiceBase): @@ -23,9 +25,9 @@ class NCEP_GFS_Service(NCEPServiceBase): ] feature_id = "ncep" # These are slowing down the api because it has to load the web page. - _possible_types = ncep.get_provider_types("Global Forecast System") - _possible_products = ncep.get_provider_products("Global Forecast System") - _possible_formats = ncep.get_formats_of_a_product("Global Forecast System") + _possible_types = sorted(ncep.get_provider_types("Global Forecast System")) + _possible_products = sorted(ncep.get_provider_products("Global Forecast System")) + _possible_formats = sorted(ncep.get_formats_of_a_product("Global Forecast System")) _parameter_map = {} date = param.String(default=None, doc="YYYYMMDD", precedence=1) @@ -37,11 +39,6 @@ class NCEP_GFS_Service(NCEPServiceBase): type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) - def get_features(self, **kwargs): - the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} - feature = pd.DataFrame(the_feature, index=[0]) - return feature - def download(self, feature, file_path, dataset, **params): ncep = NCEP_Client() p = param.ParamOverrides(self, params) @@ -70,6 +67,59 @@ def download(self, feature, file_path, dataset, **params): return metadata +class NCEP_NAM_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_nam" + display_name = "NCEP NAM Service" + description = 'NCEP NAM is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = sorted(ncep.get_provider_types("North American Model")) + _possible_products = sorted(ncep.get_provider_products("North American Model")) + _possible_formats = sorted(ncep.get_formats_of_a_product("North American Model")) + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "NAM": + raise ValueError("Please specify a specific product not NAM") + + results = ncep.get_ncep_product_data(ncep_provider="North American Model", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' From f63a73461c3e4e17846aabc5733742c41ae9da22 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Tue, 17 Apr 2018 08:33:34 -0500 Subject: [PATCH 38/44] Fixed the test and added the sorted functions into this plugin. --- quest/services/noaa_ncep.py | 1 + test/data.py | 111 ++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index d1fab433..c67d475a 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -120,6 +120,7 @@ def download(self, feature, file_path, dataset, **params): return metadata + class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase display_name = 'NCEP GFS Provider' diff --git a/test/data.py b/test/data.py index e0eff4c8..34883ab9 100644 --- a/test/data.py +++ b/test/data.py @@ -226,11 +226,7 @@ {'default': None, 'description': 'Paramerter', 'name': 'format', - 'range': [['enkf', 'enkf'], - ['gdas', 'gdas'], - ['gfs', 'gfs'], - ['gfsmos', 'gfsmos'], - ['sst', 'sst']], + 'range': [], 'type': 'ObjectSelector'}, {'default': None, 'description': 'Parameter2', @@ -271,7 +267,110 @@ ['global longitude-latitude grid (1.0 deg)', 'global longitude-latitude grid (1.0 deg)']], 'type': 'ObjectSelector'}], - 'title': 'NCEP GFS Service Download Options'} + 'title': 'NCEP GFS Service Download Options'}, + 'svc://noaa-ncep:ncep_nam': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['NAM', 'NAM']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [['NAM', 'NAM'], + ['NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)', + 'NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)'], + ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', + 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', + 'NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))'], + ['NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)', + 'NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)'], + ['NAM 182 AFWA Grid - North Pacific (12-km Resolution)', + 'NAM 182 AFWA Grid - North Pacific (12-km Resolution)'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))'], + ['NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)', + 'NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)'], + ['NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))', + 'NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))'], + ['NAM 215 AWIPS Grid - CONUS (20-km Resolution)', + 'NAM 215 AWIPS Grid - CONUS (20-km Resolution)'], + ['NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)', + 'NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)', + 'NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)'], + ['NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))', + 'NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))'], + ['NAM IMS Snow Grid (24-km Resolution)', + 'NAM IMS Snow Grid (24-km Resolution)'], + ['NAM MOS', 'NAM MOS'], + ['NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)', + 'NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)'], + ['NAM NEST over ALASKA (6 km Resolution - Grid 198)', + 'NAM NEST over ALASKA (6 km Resolution - Grid 198)'], + ['NAM NEST over CONUS (5 km Resolution - Grid 227)', + 'NAM NEST over CONUS (5 km Resolution - Grid 227)'], + ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', + 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], + ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)'], + ['NAM Precipitation Analysis (4km Analysis mapped to 15km grid)', + 'NAM Precipitation Analysis (4km Analysis mapped to 15km grid)'], + ['NAM Precipitation Analysis (Stage 4)', + 'NAM Precipitation Analysis (Stage 4)'], + ['NAM Radar and Gauge Analysis under Stage2', + 'NAM Radar and Gauge Analysis under Stage2']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP NAM Service Download Options'} } From 60e4ed337c2a9a30cbc890f04b60cd57336f918e Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 39/44] Dataset display name fix (More readable). --- quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + From 64a6240fa89da1be559bd92482374ee46667110a Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 40/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From 9d0e1fb8d7b881ef32e9588097f8da0216b9a224 Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Wed, 4 Apr 2018 15:29:14 -0500 Subject: [PATCH 41/44] Dataset display name fix (More readable). --- quest/services/ISEP_Girder.py | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py new file mode 100644 index 00000000..e3cdfc47 --- /dev/null +++ b/quest/services/ISEP_Girder.py @@ -0,0 +1,90 @@ +from .base import ProviderBase, SingleFileServiceBase, PublishBase +from ..api.database import get_db, db_session +from shapely.geometry import Point, box +from ..api.metadata import get_metadata +from ..util import param_util +from getpass import getpass +import girder_client +import pandas as pd +import param +import os + + +class ISEPServiceBase(SingleFileServiceBase): + + token = param.String(default=None, doc="Token #", precedence=1) + + @property + def gc(self): + return self.provider.get_gc() + + def get_features(self, **kwargs): + raise NotImplementedError() + + +class ISEPPublisher(PublishBase): + publisher_name = "girder_pub" + display_name = "Girder Publisher" + description = "Girder is a repository for ERDC ERS." + + token = param.String(default=None, doc="Token #", precedence=1) + title = param.String(default="example title", doc="Title of resource", precedence=2) + collection_description = param.String(default="", doc="Description of resource", precedence=3) + folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) + folder_description = param.String(default="", doc="Folder Description", precedence=5) + # Have the option to make the resource public. + dataset = param_util.DatasetListSelector(default=(), + filters={'status': 'downloaded'}, + precedence=6, + doc="dataset to publish to HydroShare") + + def __init__(self, provider, **kwargs): + super(ISEPPublisher, self).__init__(provider, **kwargs) + + @property + def gc(self): + return self.provider.get_gc() + + def publish(self, options=None): + try: + with self.gc.session() as session: + p = param.ParamOverrides(self, options) + session.verify = '/home/valoroso/DoD_Cert.pem' + self.gc.token = p.token + params = {'name': p.title, 'description': p.collection_description} + resource_information_dict = self.gc.createResource(path='collection', params=params) + folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], + name=p.folder_name, + description=p.folder_description, + parentType='collection') + for dataset in p.dataset: + dataset_metadata = get_metadata(dataset)[dataset] + fpath = dataset_metadata['file_path'] + self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) + + except Exception as e: + raise e + + return resource_information_dict['_id'] + + +class GirderProvider(ProviderBase): + service_base_class = ISEPServiceBase + publishers_list = [ISEPPublisher] + display_name = 'Girder Services' + description = 'Services avaliable through the AFRL Girder Server.' + organization_name = 'U.S. Air Force Research Laboratory' + organization_abbr = 'AFRL' + + def get_gc(self): + connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' + try: + gc = girder_client.GirderClient(apiUrl=connection_info) + except: + raise ValueError("Cannot connect to the ISEP Girder.") + + return gc + + def authenticate_me(self, **kwargs): + pass + From dddc5e926da9db8b328048164a404f663317035f Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Mon, 9 Apr 2018 16:36:44 -0500 Subject: [PATCH 42/44] Removing the placeholder for the girder site. --- quest/services/ISEP_Girder.py | 90 ----------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 quest/services/ISEP_Girder.py diff --git a/quest/services/ISEP_Girder.py b/quest/services/ISEP_Girder.py deleted file mode 100644 index e3cdfc47..00000000 --- a/quest/services/ISEP_Girder.py +++ /dev/null @@ -1,90 +0,0 @@ -from .base import ProviderBase, SingleFileServiceBase, PublishBase -from ..api.database import get_db, db_session -from shapely.geometry import Point, box -from ..api.metadata import get_metadata -from ..util import param_util -from getpass import getpass -import girder_client -import pandas as pd -import param -import os - - -class ISEPServiceBase(SingleFileServiceBase): - - token = param.String(default=None, doc="Token #", precedence=1) - - @property - def gc(self): - return self.provider.get_gc() - - def get_features(self, **kwargs): - raise NotImplementedError() - - -class ISEPPublisher(PublishBase): - publisher_name = "girder_pub" - display_name = "Girder Publisher" - description = "Girder is a repository for ERDC ERS." - - token = param.String(default=None, doc="Token #", precedence=1) - title = param.String(default="example title", doc="Title of resource", precedence=2) - collection_description = param.String(default="", doc="Description of resource", precedence=3) - folder_name = param.String(default="example folder title", doc="Folder Title", precedence=4) - folder_description = param.String(default="", doc="Folder Description", precedence=5) - # Have the option to make the resource public. - dataset = param_util.DatasetListSelector(default=(), - filters={'status': 'downloaded'}, - precedence=6, - doc="dataset to publish to HydroShare") - - def __init__(self, provider, **kwargs): - super(ISEPPublisher, self).__init__(provider, **kwargs) - - @property - def gc(self): - return self.provider.get_gc() - - def publish(self, options=None): - try: - with self.gc.session() as session: - p = param.ParamOverrides(self, options) - session.verify = '/home/valoroso/DoD_Cert.pem' - self.gc.token = p.token - params = {'name': p.title, 'description': p.collection_description} - resource_information_dict = self.gc.createResource(path='collection', params=params) - folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], - name=p.folder_name, - description=p.folder_description, - parentType='collection') - for dataset in p.dataset: - dataset_metadata = get_metadata(dataset)[dataset] - fpath = dataset_metadata['file_path'] - self.gc.uploadFileToFolder(folder_creation_dict['_id'], fpath) - - except Exception as e: - raise e - - return resource_information_dict['_id'] - - -class GirderProvider(ProviderBase): - service_base_class = ISEPServiceBase - publishers_list = [ISEPPublisher] - display_name = 'Girder Services' - description = 'Services avaliable through the AFRL Girder Server.' - organization_name = 'U.S. Air Force Research Laboratory' - organization_abbr = 'AFRL' - - def get_gc(self): - connection_info = 'https://talonw4.afrl.hpc.mil/api/v1' - try: - gc = girder_client.GirderClient(apiUrl=connection_info) - except: - raise ValueError("Cannot connect to the ISEP Girder.") - - return gc - - def authenticate_me(self, **kwargs): - pass - From a7fe17901a9217b11c483676a61289fd247cfbeb Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Thu, 19 Apr 2018 11:43:22 -0500 Subject: [PATCH 43/44] Fixed the test for ncep provider. --- quest/services/noaa_ncep.py | 2 +- quest/util/config.py | 3 +++ test/data.py | 9 ++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py index c67d475a..287c0345 100644 --- a/quest/services/noaa_ncep.py +++ b/quest/services/noaa_ncep.py @@ -123,7 +123,7 @@ def download(self, feature, file_path, dataset, **params): class NCEPProvider(ProviderBase): service_base_class = NCEPServiceBase - display_name = 'NCEP GFS Provider' + display_name = 'NCEP Provider' description = 'Services avaliable through the NOAA NCEP Server.' organization_name = 'National Centers for Environmental Prediction' organization_abbr = 'NCEP' \ No newline at end of file diff --git a/quest/util/config.py b/quest/util/config.py index 9dbdcd18..ae045744 100644 --- a/quest/util/config.py +++ b/quest/util/config.py @@ -80,6 +80,9 @@ def update_settings(config={}): from .. import init init() + from . import load_providers + load_providers(update_cache=True) + return True diff --git a/test/data.py b/test/data.py index 34883ab9..0b2855da 100644 --- a/test/data.py +++ b/test/data.py @@ -373,10 +373,9 @@ 'title': 'NCEP NAM Service Download Options'} } - SERVICES_FEATURE_COUNT = [ - # ('svc://nasa:srtm-3-arc-second', 14297, 1000), - # ('svc://nasa:srtm-30-arc-second', 27, 10), + ('svc://nasa:srtm-3-arc-second', 14297, 1000), + ('svc://nasa:srtm-30-arc-second', 27, 10), ('svc://noaa-ncdc:ghcn-daily', 104126, 5000), ('svc://noaa-ncdc:gsod', 28754, 1500), ('svc://noaa-coast:coops-meteorological', 375, 50), @@ -421,8 +420,8 @@ } SERVICE_FEATURE_DOWNLOAD_OPTIONS = [ - # ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), - # ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), + ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), + ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), ('svc://noaa-ncdc:ghcn-daily/ACW00011604', {'parameter': 'air_temperature:daily:total', 'start': '1949-01-01', 'end': '1949-01-02'}), ('svc://noaa-ncdc:gsod/717580-99999', {'parameter': 'air_temperature:daily:max', 'start': '2016-01-01', 'end': '2016-01-02'}), ('svc://noaa-coast:coops-meteorological/1611400', {'parameter': 'air_temperature', 'start': '2015-05-23', 'end': '2015-05-24'}), From 0be5eecb5f1930423c642e4e7305c2de6477067f Mon Sep 17 00:00:00 2001 From: AaronV77 Date: Thu, 19 Apr 2018 11:50:09 -0500 Subject: [PATCH 44/44] Fixed the test for ncep provider part 2. --- test/data.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/data.py b/test/data.py index 0b2855da..05cbe56f 100644 --- a/test/data.py +++ b/test/data.py @@ -302,8 +302,6 @@ 'description': 'Parameter3', 'name': 'product', 'range': [['NAM', 'NAM'], - ['NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)', - 'NAM (RTMA version) CONUS Precipitation Analysis (Stage2 4km Resolution)'], ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', @@ -362,13 +360,7 @@ ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', - 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)'], - ['NAM Precipitation Analysis (4km Analysis mapped to 15km grid)', - 'NAM Precipitation Analysis (4km Analysis mapped to 15km grid)'], - ['NAM Precipitation Analysis (Stage 4)', - 'NAM Precipitation Analysis (Stage 4)'], - ['NAM Radar and Gauge Analysis under Stage2', - 'NAM Radar and Gauge Analysis under Stage2']], + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)']], 'type': 'ObjectSelector'}], 'title': 'NCEP NAM Service Download Options'} }