diff --git a/docs/install.rst b/docs/install.rst index eb4c8416..08123b28 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,11 +12,11 @@ Install Released Conda Package a. Install Quest into new environment:: - conda create -n quest -c conda-forge -c erdc quest terrapin + conda create -n quest -c conda-forge -c erdc quest b. Install Quest into existing environment:: - conda install -c erdc -c conda-forge quest terrapin + conda install -c erdc -c conda-forge quest .. note:: @@ -49,11 +49,8 @@ Conda Install Optional ........ - d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel:: - conda install -c erdc -c conda-forge terrapin - - e. Run tests:: + d. Run tests:: python setup.py test diff --git a/py2_conda_environment.yml b/py2_conda_environment.yml index 67ae4cb0..638fb57d 100644 --- a/py2_conda_environment.yml +++ b/py2_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -40,4 +41,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/py3_conda_environment.yml b/py3_conda_environment.yml index 07a0f9f7..e4b5d74e 100644 --- a/py3_conda_environment.yml +++ b/py3_conda_environment.yml @@ -11,6 +11,7 @@ name: quest channels: - conda-forge + - erdc - defaults dependencies: @@ -41,4 +42,6 @@ dependencies: - sphinx_rtd_theme - hs_restclient - jupyter - - girder-client \ No newline at end of file + - girder-client + - terrapin + - ncep_client \ No newline at end of file diff --git a/quest/api/__init__.py b/quest/api/__init__.py index b6254bd4..51e92e10 100644 --- a/quest/api/__init__.py +++ b/quest/api/__init__.py @@ -25,6 +25,7 @@ 'download_options', 'get_active_project', 'get_api_version', + 'get_auth_status', 'get_collections', 'get_datasets', 'get_features', @@ -56,7 +57,6 @@ 'save_settings', 'set_active_project', 'stage_for_download', - 'stage_for_publish', 'unauthenticate_provider', 'update_metadata', # replaces update_collection, update_feature, update_dataset 'update_settings', @@ -150,6 +150,7 @@ get_services, add_provider, delete_provider, + get_auth_status, authenticate_provider, unauthenticate_provider, ) diff --git a/quest/api/datasets.py b/quest/api/datasets.py index 2619fd2b..0b7e47a3 100644 --- a/quest/api/datasets.py +++ b/quest/api/datasets.py @@ -60,10 +60,10 @@ def download(feature, file_path, dataset=None, **kwargs): @add_async -def publish(publisher_uri, options=None): +def publish(publisher_uri, **kwargs): provider, publisher, feature = util.parse_service_uri(publisher_uri) driver = util.load_providers()[provider] - data = driver.publish(publisher=publisher, options=options) + data = driver.publish(publisher=publisher, **kwargs) return data @add_async diff --git a/quest/api/services.py b/quest/api/services.py index 2c225f3c..fbd3001a 100644 --- a/quest/api/services.py +++ b/quest/api/services.py @@ -4,6 +4,7 @@ """ from __future__ import absolute_import from __future__ import print_function +from ..api.database import get_db, db_session from .. import util import os import requests @@ -78,8 +79,6 @@ def get_publishers(expand=None, publisher_type=None): Returns: providers (list or dict,Default=list): list of all available providers - - """ providers = util.load_providers() publishers = {} @@ -127,6 +126,7 @@ def add_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service added' + util.load_providers(update_cache=True) else: msg = 'service already present' else: @@ -159,13 +159,35 @@ def delete_provider(uri): util.update_settings({'USER_SERVICES': user_services}) util.save_settings() msg = 'service removed' + util.load_providers(update_cache=True) else: msg = 'service not found' return msg -def authenticate_provider(uri): +def get_auth_status(uri): + """Check to see if a provider has been authenticated + + Args: + uri (string, Required): + uri of 'user service' + Returns: + True on success + False on not authenticated + + """ + db = get_db() + with db_session: + p = db.Providers.select().filter(provider=uri).first() + + if p is None: + return False + + return True + + +def authenticate_provider(uri, **kwargs): """Authenticate the user. Args: @@ -176,7 +198,7 @@ def authenticate_provider(uri): """ driver = util.load_providers()[uri] - driver.authenticate_me() + driver.authenticate_me(**kwargs) def unauthenticate_provider(uri): diff --git a/quest/services/base/provider_base.py b/quest/services/base/provider_base.py index 7bc08b45..7adcb4d3 100644 --- a/quest/services/base/provider_base.py +++ b/quest/services/base/provider_base.py @@ -283,8 +283,8 @@ def download_options(self, service, fmt): """ return self.services[service].download_options(fmt) - def publish(self, publisher, options): - return self.publishers[publisher].publish(options) + def publish(self, publisher, **kwargs): + return self.publishers[publisher].publish(**kwargs) def publish_options(self, publisher, fmt): return self.publishers[publisher].publish_options(fmt) diff --git a/quest/services/base/publish_base.py b/quest/services/base/publish_base.py index 0e6a80cb..a19ca1fb 100644 --- a/quest/services/base/publish_base.py +++ b/quest/services/base/publish_base.py @@ -38,5 +38,5 @@ def publish_options(self, fmt): return schema - def publish(self, options): + def publish(self, **kwargs): raise NotImplementedError() diff --git a/quest/services/base/service_base.py b/quest/services/base/service_base.py index 5dae3ace..27195b4d 100644 --- a/quest/services/base/service_base.py +++ b/quest/services/base/service_base.py @@ -92,7 +92,7 @@ def download_options(self, fmt): return schema - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): raise NotImplementedError() def get_features(self, **kwargs): @@ -138,7 +138,7 @@ class SingleFileServiceBase(ServiceBase): """Base file for datasets that are a single file download eg elevation raster etc """ - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): feature_id = util.construct_service_uri(self.provider.name, self.name, feature) feature = self.provider.get_features(self.name).loc[feature_id] reserved = feature.get('reserved') diff --git a/quest/services/cuahsi_hs.py b/quest/services/cuahsi_hs.py index 2557440d..ec33c387 100644 --- a/quest/services/cuahsi_hs.py +++ b/quest/services/cuahsi_hs.py @@ -4,11 +4,11 @@ from shapely.geometry import Point, box from ..api.metadata import get_metadata from ..util import param_util -from getpass import getpass import pandas as pd import param import os + class HSServiceBase(SingleFileServiceBase): @property @@ -35,7 +35,7 @@ class HSGeoService(HSServiceBase): def get_features(self, **kwargs): - results = list(self.hs.resources()) + results = list(self.hs.resources(coverage_type="box", north="90", south="-90", east="180", west="-180")) if len(results) == 0: raise ValueError('No resource available from HydroShare.') @@ -126,21 +126,18 @@ class HSPublisher(PublishBase): 'Time Series': 'TimeSeriesResource' } - title = param.String(default="example title", doc="Title of resource", precedence=2) - abstract = param.String(default="example abstract", precedence=3, - doc="An description of the resource to be added to HydroShare.") - keywords = param.List(default=[], precedence=4, doc="list of keyword strings to describe the resource") - dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, precedence=5, - doc="dataset to publish to HydroShare") - resource_type = param.ObjectSelector(doc='parameter', precedence=1, objects=sorted(_resource_type_map.keys())) + resource_type = param.ObjectSelector(default=None, doc="", precedence=1, objects=sorted(_resource_type_map.keys())) + title = param.String(default="", doc="", precedence=2) + abstract = param.String(default="", doc="", precedence=3) + keywords = param.List(default=[], doc="", precedence=4) + dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, doc="", precedence=5) @property def hs(self): return self.provider.get_hs() - def publish(self, options=None): - - p = param.ParamOverrides(self, options) + def publish(self, **kwargs): + p = param.ParamOverrides(self, kwargs) valid_file_paths = [] valid_extensions = [] @@ -262,8 +259,8 @@ def get_hs(self, auth=None, require_valid_auth=False): def authenticate_me(self, **kwargs): - username = input("Enter Username: ") - password = getpass("Enter Password: ") + username = kwargs['username'] + password = kwargs['password'] try: auth = HydroShareAuthBasic(username=username, password=password) diff --git a/quest/services/kitware_girder.py b/quest/services/kitware_girder.py index 9a4bf34e..af1f8e6c 100644 --- a/quest/services/kitware_girder.py +++ b/quest/services/kitware_girder.py @@ -34,9 +34,9 @@ class GirderPublisher(PublishBase): def gc(self): return self.provider.get_gc() - def publish(self, options=None): + def publish(self, **kwargs): try: - p = param.ParamOverrides(self, options) + p = param.ParamOverrides(self, kwargs) params = {'name': p.title, 'description': p.collection_description} resource_information_dict = self.gc.createResource(path='collection', params=params) folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'], @@ -77,8 +77,8 @@ def get_gc(self): def authenticate_me(self, **kwargs): connection_info = 'https://data.kitware.com/api/v1' - username = input("Enter Username: ") - password = getpass("Enter Password: ") + username = kwargs['username'] + password = kwargs['password'] try: gc = girder_client.GirderClient(apiUrl=connection_info) diff --git a/quest/services/nasa.py b/quest/services/nasa.py index 05f2e3cf..3bbe1a46 100644 --- a/quest/services/nasa.py +++ b/quest/services/nasa.py @@ -2,6 +2,8 @@ """ from .base import ProviderBase, SingleFileServiceBase +from ..api.database import get_db, db_session +from getpass import getpass import pandas as pd import requests @@ -20,8 +22,15 @@ class NasaServiceBase(SingleFileServiceBase): 'elevation': 'elevation' } - def _read_granules(short_name, page_num): - return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry'] + @property + def info(self): + return self.provider.get_user_info() + + def _read_granules(self, short_name, page_num): + try: + return requests.get(granules_url % (short_name, page_num), auth=(self.info['username'], self.info['password'])).json()['feed']['entry'] + except ValueError: + return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry'] def get_features(self, **kwargs): page_num = 0 @@ -128,3 +137,35 @@ class NasaProvider(ProviderBase): description = 'Services available through the NASA' organization_name = 'National Aeronautic and Space Administration' organization_abbr = 'NASA' + + def get_user_info(self): + the_info = self.credentials + return the_info + + def authenticate_me(self, **kwargs): + + username = input("Enter Username: ") + password = getpass("Enter Password: ") + + try: + db = get_db() + with db_session: + p = db.Providers.select().filter(provider=self.name).first() + + provider_metadata = { + 'provider': self.name, + 'username': username, + 'password': password, + } + + if p is None: + db.Providers(**provider_metadata) + else: + p.set(**provider_metadata) + + return True + + except: + print("Either credentials invalid or unable to connect to HydroShare.") + + return False diff --git a/quest/services/noaa_coastwatch.py b/quest/services/noaa_coastwatch.py index bb79ba92..6e082007 100644 --- a/quest/services/noaa_coastwatch.py +++ b/quest/services/noaa_coastwatch.py @@ -65,8 +65,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/noaa_ncdc.py b/quest/services/noaa_ncdc.py index 5bd26238..6bbe1d5f 100644 --- a/quest/services/noaa_ncdc.py +++ b/quest/services/noaa_ncdc.py @@ -74,8 +74,8 @@ def parameter_map(self, invert=False): return pmap - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) self.parameter = p.parameter self.end = pd.to_datetime(p.end) self.start = pd.to_datetime(p.start) diff --git a/quest/services/noaa_ncep.py b/quest/services/noaa_ncep.py new file mode 100644 index 00000000..287c0345 --- /dev/null +++ b/quest/services/noaa_ncep.py @@ -0,0 +1,129 @@ +from .base import ProviderBase, ServiceBase +from ncep_client import NCEP_Client +from shapely.geometry import box +import pandas as pd +import param + + +class NCEPServiceBase(ServiceBase): + + def get_features(self, **kwargs): + the_feature = {"service_id": "ncep", "display_name": "ncep", "geometry": box(-180, -90, 180, 90)} + feature = pd.DataFrame(the_feature, index=[0]) + return feature + + +class NCEP_GFS_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_gfs" + display_name = "NCEP GFS Service" + description = 'NCEP GFS is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = sorted(ncep.get_provider_types("Global Forecast System")) + _possible_products = sorted(ncep.get_provider_products("Global Forecast System")) + _possible_formats = sorted(ncep.get_formats_of_a_product("Global Forecast System")) + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "GFS" or p.product == "GDAS": + raise ValueError("Please specify a specific product not GFS or GDAS") + + results = ncep.get_ncep_product_data(ncep_provider="Global Forecast System", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEP_NAM_Service(NCEPServiceBase): + ncep = NCEP_Client() + service_name = "ncep_nam" + display_name = "NCEP NAM Service" + description = 'NCEP NAM is a noaa repository for global weather data.' + service_type = "norm-discrete" + geographical_areas = ['Worldwide'] + bounding_boxes = [ + [-180, -90, 180, 90], + ] + feature_id = "ncep" + # These are slowing down the api because it has to load the web page. + _possible_types = sorted(ncep.get_provider_types("North American Model")) + _possible_products = sorted(ncep.get_provider_products("North American Model")) + _possible_formats = sorted(ncep.get_formats_of_a_product("North American Model")) + _parameter_map = {} + + date = param.String(default=None, doc="YYYYMMDD", precedence=1) + res = param.String(default=None, doc="Froecast Resolution", precedence=2) + cycle = param.String(default=None, doc="Forecast Cycle Runtime", precedence=3) + start = param.String(default=None, doc="Forecast start time (f###)", precedence=4) + end = param.String(default=None, doc="Forecast end time (f###)", precedence=5) + format = param.ObjectSelector(default=None, doc="Paramerter", objects=_possible_formats, precedence=6) + type = param.ObjectSelector(default=None, doc="Parameter2", objects=_possible_types, precedence=7) + product = param.ObjectSelector(default=None, doc="Parameter3", objects=_possible_products, precedence=8) + + def download(self, feature, file_path, dataset, **params): + ncep = NCEP_Client() + p = param.ParamOverrides(self, params) + + if p.product == "NAM": + raise ValueError("Please specify a specific product not NAM") + + results = ncep.get_ncep_product_data(ncep_provider="North American Model", product_type=p.type, + product_date=p.date, resolution=p.res, cycle_runtime=p.cycle, + forecast_start=p.start, forecast_end=p.end, product_format=p.format, + name_of_product=p.product) + print(results) + if len(results) > 0: + ncep.download_data(file_path, results) + metadata = { + 'metadata': results, + 'file_path': file_path, + 'file_format': 'weather-specific', + 'datatype': 'weather', + 'parameter': "ncep_parameter", + 'unit': "unkown", + } + else: + raise ValueError("There is no data found on those parameters.") + + return metadata + + +class NCEPProvider(ProviderBase): + service_base_class = NCEPServiceBase + display_name = 'NCEP Provider' + description = 'Services avaliable through the NOAA NCEP Server.' + organization_name = 'National Centers for Environmental Prediction' + organization_abbr = 'NCEP' \ No newline at end of file diff --git a/quest/services/template_service.py b/quest/services/template_service.py index 4d0cd8cf..b4e4ba6f 100644 --- a/quest/services/template_service.py +++ b/quest/services/template_service.py @@ -26,7 +26,7 @@ class ExampleServiceBase(ServiceBase): smtk_template = None _parameter_map = dict() - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwwargs): metadata = {} # get metadata from service data = None # data structure containing downloaded data diff --git a/quest/services/user_provider.py b/quest/services/user_provider.py index c6647215..4e525127 100644 --- a/quest/services/user_provider.py +++ b/quest/services/user_provider.py @@ -51,7 +51,7 @@ def instance(cls, service_name, service_data, provider, uri, is_remote): return self - def download(self, feature, file_path, dataset, **params): + def download(self, feature, file_path, dataset, **kwargs): if self.datasets_mapping is not None: fnames = self.datasets_mapping if isinstance(dict, self.datasets_mapping): diff --git a/quest/services/usgs_nwis.py b/quest/services/usgs_nwis.py index 08233a39..cc81b72b 100644 --- a/quest/services/usgs_nwis.py +++ b/quest/services/usgs_nwis.py @@ -18,8 +18,8 @@ class NwisServiceBase(TimePeriodServiceBase): period = param.String(default='P365D', precedence=4, doc='time period (e.g. P365D = 365 days or P4W = 4 weeks)') smtk_template = 'start_end_or_period.sbt' - def download(self, feature, file_path, dataset, **params): - p = param.ParamOverrides(self, params) + def download(self, feature, file_path, dataset, **kwargs): + p = param.ParamOverrides(self, kwargs) parameter = p.parameter start = p.start diff --git a/quest/util/config.py b/quest/util/config.py index 9dbdcd18..ae045744 100644 --- a/quest/util/config.py +++ b/quest/util/config.py @@ -80,6 +80,9 @@ def update_settings(config={}): from .. import init init() + from . import load_providers + load_providers(update_cache=True) + return True diff --git a/quest/util/misc.py b/quest/util/misc.py index fac84738..5f03d405 100644 --- a/quest/util/misc.py +++ b/quest/util/misc.py @@ -19,6 +19,7 @@ from uuid import uuid4, UUID import quest +the_providers = None def generate_cache(update=False): """Downloads features for all services and caches results. @@ -238,22 +239,27 @@ def load_drivers(namespace, names=None): return {name: driver.DriverManager(namespace, name, invoke_on_load='True') for name in names} -def load_providers(): - settings = get_settings() +def load_providers(update_cache=False): + global the_providers - # add web services + settings = get_settings() web_services = list_drivers('services') web_services.remove('user') - providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} + if update_cache or the_providers is None: + providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services} - if len(settings.get('USER_SERVICES', [])) > 0: - for uri in settings.get('USER_SERVICES', []): - try: - drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver - providers['user-' + drv.name] = drv - except Exception as e: - logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + if len(settings.get('USER_SERVICES', [])) > 0: + for uri in settings.get('USER_SERVICES', []): + try: + drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver + providers['user-' + drv.name] = drv + except Exception as e: + logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e))) + + the_providers = providers + else: + providers = the_providers return providers diff --git a/setup.cfg b/setup.cfg index 97e56815..7f02c18a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,13 +31,14 @@ console_scripts = quest.services = user = quest.services.user_provider:UserProvider noaa-coast = quest.services.noaa_coastwatch:NoaaProvider + noaa-ncep = quest.services.noaa_ncep:NCEPProvider noaa-ncdc = quest.services.noaa_ncdc:NcdcProvider usgs-nwis = quest.services.usgs_nwis:NwisProvider usgs-ned = quest.services.usgs_ned:UsgsNedProvider usgs-nlcd = quest.services.usgs_nlcd:UsgsNlcdProvider cuahsi-hydroshare = quest.services.cuahsi_hs:HSProvider kitware-girder = quest.services.kitware_girder:GirderProvider - # nasa = quest.services.nasa:NasaProvider + nasa = quest.services.nasa:NasaProvider quest.filters = ts-unit-conversion = quest.filters.timeseries.timeseries:TsUnitConversion diff --git a/test/data.py b/test/data.py index 01bc2e35..05cbe56f 100644 --- a/test/data.py +++ b/test/data.py @@ -3,8 +3,8 @@ """ DOWNLOAD_OPTIONS_FROM_ALL_SERVICES = { - # 'svc://nasa:srtm-3-arc-second': {}, - # 'svc://nasa:srtm-30-arc-second': {}, + 'svc://nasa:srtm-3-arc-second': {}, + 'svc://nasa:srtm-30-arc-second': {}, 'svc://noaa-ncdc:ghcn-daily': {'properties': [{'default': None, 'description': 'parameter', 'name': 'parameter', @@ -203,12 +203,171 @@ 'title': 'NWIS Instantaneous Values Service Download Options'}, 'svc://cuahsi-hydroshare:hs_geo': {}, 'svc://cuahsi-hydroshare:hs_norm': {}, + 'svc://noaa-ncep:ncep_gfs': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['GDAS', 'GDAS'], ['GFS', 'GFS']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [["0.50 deg 'full' file description", + "0.50 deg 'full' file description"], + ['32km Lambert Conformal grid', '32km Lambert Conformal grid'], + ['Atmospheric Analysis', 'Atmospheric Analysis'], + ['Atmospheric Model Data', 'Atmospheric Model Data'], + ['BUFR Sounding Files per Station', 'BUFR Sounding Files per Station'], + ['Binary Universal Form for the Representation of meteorological data (BUFR)', + 'Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['GDAS', 'GDAS'], + ['GFS', 'GFS'], + ['Global longitude-latitude grid', 'Global longitude-latitude grid'], + ['MDL Extratropical Storm Surge', 'MDL Extratropical Storm Surge'], + ['MOS Aviation Product', 'MOS Aviation Product'], + ['Prepared BUFR files', 'Prepared BUFR files'], + ['Pressure Level Data', 'Pressure Level Data'], + ['Sigma Atmospheric Model Data', 'Sigma Atmospheric Model Data'], + ['Smart Initialization Guam', 'Smart Initialization Guam'], + ['Surface Analysis', 'Surface Analysis'], + ['Surface Boundary Conditions', 'Surface Boundary Conditions'], + ['Surface Flux', 'Surface Flux'], + ['T1534 Semi-Lagrangian grid', 'T1534 Semi-Lagrangian grid'], + ['Time Dependent Satellite Bias Correction', + 'Time Dependent Satellite Bias Correction'], + ['Tropical Cyclone Vital Statistics', + 'Tropical Cyclone Vital Statistics'], + ['WAFS/ICAO/International Exchange/FOS Grids', + 'WAFS/ICAO/International Exchange/FOS Grids'], + ['World Area Forecast System', 'World Area Forecast System'], + ['global longitude-latitude grid (1.0 deg)', + 'global longitude-latitude grid (1.0 deg)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP GFS Service Download Options'}, + 'svc://noaa-ncep:ncep_nam': {'properties': [{'default': None, + 'description': 'YYYYMMDD', + 'name': 'date', + 'type': 'String'}, + {'default': None, + 'description': 'Froecast Resolution', + 'name': 'res', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast Cycle Runtime', + 'name': 'cycle', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast start time (f###)', + 'name': 'start', + 'type': 'String'}, + {'default': None, + 'description': 'Forecast end time (f###)', + 'name': 'end', + 'type': 'String'}, + {'default': None, + 'description': 'Paramerter', + 'name': 'format', + 'range': [], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter2', + 'name': 'type', + 'range': [['NAM', 'NAM']], + 'type': 'ObjectSelector'}, + {'default': None, + 'description': 'Parameter3', + 'name': 'product', + 'range': [['NAM', 'NAM'], + ['NAM - Binary Universal Form for the Representation of meteorological data (BUFR)', + 'NAM - Binary Universal Form for the Representation of meteorological data (BUFR)'], + ['NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))', + 'NAM 104 AWIPS Grid (N. Hemisphere polar stereographic grid (NGM Super C grid))'], + ['NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)', + 'NAM 181 AFWA Grid - Central America/Caribbean (12-km Resolution)'], + ['NAM 182 AFWA Grid - North Pacific (12-km Resolution)', + 'NAM 182 AFWA Grid - North Pacific (12-km Resolution)'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated lat/lon grid using the 60 NAM hybrid levels(NAM 12km Domain))'], + ['NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))', + 'NAM 190 Grid - CONUS (Staggered B-grid on rotated latitude/longitude grid (NAM 12km Domain))'], + ['NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 195 Grid over Puerto Rico (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 196 Grid over Hawaii (2.5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 197 Grid - CONUS (5-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)', + 'NAM 198 Grid over Alaska (6-km Resolution) (NAM Smartinit for NDFD)'], + ['NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)', + 'NAM 211 AWIPS Grid - Regional - CONUS (81-km Resolution)'], + ['NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))', + 'NAM 212 AWIPS Grid - Regional - CONUS (Double Resolution (40-km Resolution))'], + ['NAM 215 AWIPS Grid - CONUS (20-km Resolution)', + 'NAM 215 AWIPS Grid - CONUS (20-km Resolution)'], + ['NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)', + 'NAM 216 AWIPS Grid - Regional - Alaska (45-km Resolution)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)', + 'NAM 218 AWIPS Grid - CONUS (12-km Resolution; full complement of surface-based fields)'], + ['NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 218 AWIPS Grid - CONUS - (12-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)', + 'NAM 221 AWIPS Grid - High Resolution North American Master Grid (32-km Resolution)'], + ['NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 221 AWIPS Grid - N. American Master (32-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of pressure level fields and some surface-based fields)'], + ['NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)', + 'NAM 242 AWIPS Grid - Over Alaska (11.25 KM Resolution; full complement of surface-based fields)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)', + 'NAM 243 AWIPS Grid - Eastern North Pacific (40-km Resolution) (GOES Simulated Brightness Temp.)'], + ['NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))', + 'NAM 243 AWIPS Grid - Eastern North Pacific (Double Resolution (40-km Resolution))'], + ['NAM IMS Snow Grid (24-km Resolution)', + 'NAM IMS Snow Grid (24-km Resolution)'], + ['NAM MOS', 'NAM MOS'], + ['NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)', + 'NAM NEST - FIRE WEATHER (1.33 km CONUS / 1.5 km Alaska Resolution)'], + ['NAM NEST over ALASKA (6 km Resolution - Grid 198)', + 'NAM NEST over ALASKA (6 km Resolution - Grid 198)'], + ['NAM NEST over CONUS (5 km Resolution - Grid 227)', + 'NAM NEST over CONUS (5 km Resolution - Grid 227)'], + ['NAM NEST over HAWAII (3 km Resolution - Grid 196)', + 'NAM NEST over HAWAII (3 km Resolution - Grid 196)'], + ['NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)', + 'NAM NEST over PUERTO RICO (3 km Resolution - Grid 194)']], + 'type': 'ObjectSelector'}], + 'title': 'NCEP NAM Service Download Options'} } - SERVICES_FEATURE_COUNT = [ - # ('svc://nasa:srtm-3-arc-second', 14297, 1000), - # ('svc://nasa:srtm-30-arc-second', 27, 10), + ('svc://nasa:srtm-3-arc-second', 14297, 1000), + ('svc://nasa:srtm-30-arc-second', 27, 10), ('svc://noaa-ncdc:ghcn-daily', 104126, 5000), ('svc://noaa-ncdc:gsod', 28754, 1500), ('svc://noaa-coast:coops-meteorological', 375, 50), @@ -253,8 +412,8 @@ } SERVICE_FEATURE_DOWNLOAD_OPTIONS = [ - # ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), - # ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), + ('svc://nasa:srtm-3-arc-second/G1034711987-LPDAAC_ECS' , None), + ('svc://nasa:srtm-30-arc-second/G1005651728-LPDAAC_ECS', None), ('svc://noaa-ncdc:ghcn-daily/ACW00011604', {'parameter': 'air_temperature:daily:total', 'start': '1949-01-01', 'end': '1949-01-02'}), ('svc://noaa-ncdc:gsod/717580-99999', {'parameter': 'air_temperature:daily:max', 'start': '2016-01-01', 'end': '2016-01-02'}), ('svc://noaa-coast:coops-meteorological/1611400', {'parameter': 'air_temperature', 'start': '2015-05-23', 'end': '2015-05-24'}),