Ncep provider

docs/install.rst

@@ -12,11 +12,11 @@ Install Released Conda Package
 
     a. Install Quest into new environment::
 
-        conda create -n quest -c conda-forge -c erdc quest terrapin
+        conda create -n quest -c conda-forge -c erdc quest
 
     b. Install Quest into existing environment::
 
-        conda install -c erdc -c conda-forge quest terrapin
+        conda install -c erdc -c conda-forge quest
 
     .. note::
 
@@ -49,11 +49,8 @@ Conda Install
 
     Optional
     ........
-    d. Some filters (e.g. watershed delineation) require the package `terrapin` that is available through the es-conda-channel::
 
-        conda install -c erdc -c conda-forge terrapin
-
-    e. Run tests::
+    d. Run tests::
 
         python setup.py test
 

py2_conda_environment.yml

@@ -11,6 +11,7 @@ name: quest
 
 channels:
     - conda-forge
+    - erdc
     - defaults
 
 dependencies:
@@ -40,4 +41,6 @@ dependencies:
     - sphinx_rtd_theme
     - hs_restclient
     - jupyter
-    - girder-client
+    - girder-client
+    - terrapin
+    - ncep_client

py3_conda_environment.yml

@@ -11,6 +11,7 @@ name: quest
 
 channels:
     - conda-forge
+    - erdc
     - defaults
 
 dependencies:
@@ -41,4 +42,6 @@ dependencies:
     - sphinx_rtd_theme
     - hs_restclient
     - jupyter
-    - girder-client
+    - girder-client
+    - terrapin
+    - ncep_client

quest/api/__init__.py

@@ -25,6 +25,7 @@
     'download_options',
     'get_active_project',
     'get_api_version',
+    'get_auth_status',
     'get_collections',
     'get_datasets',
     'get_features',
@@ -56,7 +57,6 @@
     'save_settings',
     'set_active_project',
     'stage_for_download',
-    'stage_for_publish',
     'unauthenticate_provider',
     'update_metadata',  # replaces update_collection, update_feature, update_dataset
     'update_settings',
@@ -150,6 +150,7 @@
     get_services,
     add_provider,
     delete_provider,
+    get_auth_status,
     authenticate_provider,
     unauthenticate_provider,
 )

quest/api/datasets.py

@@ -60,10 +60,10 @@ def download(feature, file_path, dataset=None, **kwargs):
 
 
 @add_async
-def publish(publisher_uri, options=None):
+def publish(publisher_uri, **kwargs):
     provider, publisher, feature = util.parse_service_uri(publisher_uri)
     driver = util.load_providers()[provider]
-    data = driver.publish(publisher=publisher, options=options)
+    data = driver.publish(publisher=publisher, **kwargs)
     return data
 
 @add_async

quest/api/services.py

@@ -4,6 +4,7 @@
 """
 from __future__ import absolute_import
 from __future__ import print_function
+from ..api.database import get_db, db_session
 from .. import util
 import os
 import requests
@@ -78,8 +79,6 @@ def get_publishers(expand=None, publisher_type=None):
     Returns:
         providers (list or dict,Default=list):
             list of all available providers
-
-
     """
     providers = util.load_providers()
     publishers = {}
@@ -127,6 +126,7 @@ def add_provider(uri):
             util.update_settings({'USER_SERVICES': user_services})
             util.save_settings()
             msg = 'service added'
+            util.load_providers(update_cache=True)
         else:
             msg = 'service already present'
     else:
@@ -159,13 +159,35 @@ def delete_provider(uri):
         util.update_settings({'USER_SERVICES': user_services})
         util.save_settings()
         msg = 'service removed'
+        util.load_providers(update_cache=True)
     else:
         msg = 'service not found'
 
     return msg
 
 
-def authenticate_provider(uri):
+def get_auth_status(uri):
+    """Check to see if a provider has been authenticated
+
+    Args:
+        uri (string, Required):
+            uri of 'user service'
+     Returns:
+        True on success
+        False on not authenticated
+
+    """
+    db = get_db()
+    with db_session:
+        p = db.Providers.select().filter(provider=uri).first()
+
+        if p is None:
+            return False
+
+    return True
+
+
+def authenticate_provider(uri, **kwargs):
     """Authenticate the user.
 
     Args:
@@ -176,7 +198,7 @@ def authenticate_provider(uri):
 
     """
     driver = util.load_providers()[uri]
-    driver.authenticate_me()
+    driver.authenticate_me(**kwargs)
 
 
 def unauthenticate_provider(uri):

quest/services/base/provider_base.py

@@ -283,8 +283,8 @@ def download_options(self, service, fmt):
         """
         return self.services[service].download_options(fmt)
 
-    def publish(self, publisher, options):
-        return self.publishers[publisher].publish(options)
+    def publish(self, publisher, **kwargs):
+        return self.publishers[publisher].publish(**kwargs)
 
     def publish_options(self, publisher, fmt):
         return self.publishers[publisher].publish_options(fmt)

quest/services/base/publish_base.py

@@ -38,5 +38,5 @@ def publish_options(self, fmt):
 
         return schema
 
-    def publish(self, options):
+    def publish(self, **kwargs):
         raise NotImplementedError()

quest/services/base/service_base.py

@@ -92,7 +92,7 @@ def download_options(self, fmt):
 
         return schema
 
-    def download(self, feature, file_path, dataset, **params):
+    def download(self, feature, file_path, dataset, **kwargs):
         raise NotImplementedError()
 
     def get_features(self, **kwargs):
@@ -138,7 +138,7 @@ class SingleFileServiceBase(ServiceBase):
     """Base file for datasets that are a single file download
     eg elevation raster etc
     """
-    def download(self, feature, file_path, dataset, **params):
+    def download(self, feature, file_path, dataset, **kwargs):
         feature_id = util.construct_service_uri(self.provider.name, self.name, feature)
         feature = self.provider.get_features(self.name).loc[feature_id]
         reserved = feature.get('reserved')

quest/services/cuahsi_hs.py

@@ -4,11 +4,11 @@
 from shapely.geometry import Point, box
 from ..api.metadata import get_metadata
 from ..util import param_util
-from getpass import getpass
 import pandas as pd
 import param
 import os
 
+
 class HSServiceBase(SingleFileServiceBase):
 
     @property
@@ -35,7 +35,7 @@ class HSGeoService(HSServiceBase):
 
     def get_features(self, **kwargs):
 
-        results = list(self.hs.resources())
+        results = list(self.hs.resources(coverage_type="box", north="90", south="-90", east="180", west="-180"))
 
         if len(results) == 0:
             raise ValueError('No resource available from HydroShare.')
@@ -126,21 +126,18 @@ class HSPublisher(PublishBase):
         'Time Series': 'TimeSeriesResource'
     }
 
-    title = param.String(default="example title", doc="Title of resource", precedence=2)
-    abstract = param.String(default="example abstract", precedence=3,
-                            doc="An description of the resource to be added to HydroShare.")
-    keywords = param.List(default=[], precedence=4, doc="list of keyword strings to describe the resource")
-    dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, precedence=5,
-                                             doc="dataset to publish to HydroShare")
-    resource_type = param.ObjectSelector(doc='parameter', precedence=1, objects=sorted(_resource_type_map.keys()))
+    resource_type = param.ObjectSelector(default=None, doc="", precedence=1, objects=sorted(_resource_type_map.keys()))
+    title = param.String(default="", doc="", precedence=2)
+    abstract = param.String(default="", doc="", precedence=3)
+    keywords = param.List(default=[], doc="", precedence=4)
+    dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, doc="", precedence=5)
 
     @property
     def hs(self):
         return self.provider.get_hs()
 
-    def publish(self, options=None):
-
-        p = param.ParamOverrides(self, options)
+    def publish(self, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         valid_file_paths = []
         valid_extensions = []
 
@@ -262,8 +259,8 @@ def get_hs(self, auth=None, require_valid_auth=False):
 
     def authenticate_me(self, **kwargs):
 
-        username = input("Enter Username: ")
-        password = getpass("Enter Password: ")
+        username = kwargs['username']
+        password = kwargs['password']
 
         try:
             auth = HydroShareAuthBasic(username=username, password=password)

quest/services/kitware_girder.py

@@ -34,9 +34,9 @@ class GirderPublisher(PublishBase):
     def gc(self):
         return self.provider.get_gc()
 
-    def publish(self, options=None):
+    def publish(self, **kwargs):
         try:
-            p = param.ParamOverrides(self, options)
+            p = param.ParamOverrides(self, kwargs)
             params = {'name': p.title, 'description': p.collection_description}
             resource_information_dict = self.gc.createResource(path='collection', params=params)
             folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'],
@@ -77,8 +77,8 @@ def get_gc(self):
 
     def authenticate_me(self, **kwargs):
         connection_info = 'https://data.kitware.com/api/v1'
-        username = input("Enter Username: ")
-        password = getpass("Enter Password: ")
+        username = kwargs['username']
+        password = kwargs['password']
 
         try:
             gc = girder_client.GirderClient(apiUrl=connection_info)

quest/services/nasa.py

@@ -2,6 +2,8 @@
 
 """
 from .base import ProviderBase, SingleFileServiceBase
+from ..api.database import get_db, db_session
+from getpass import getpass
 import pandas as pd
 import requests
 
@@ -20,8 +22,15 @@ class NasaServiceBase(SingleFileServiceBase):
         'elevation': 'elevation'
     }
 
-    def _read_granules(short_name, page_num):
-        return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry']
+    @property
+    def info(self):
+        return self.provider.get_user_info()
+
+    def _read_granules(self, short_name, page_num):
+        try:
+            return requests.get(granules_url % (short_name, page_num), auth=(self.info['username'], self.info['password'])).json()['feed']['entry']
+        except ValueError:
+            return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry']
 
     def get_features(self, **kwargs):
         page_num = 0
@@ -128,3 +137,35 @@ class NasaProvider(ProviderBase):
     description = 'Services available through the NASA'
     organization_name = 'National Aeronautic and Space Administration'
     organization_abbr = 'NASA'
+
+    def get_user_info(self):
+        the_info = self.credentials
+        return the_info
+
+    def authenticate_me(self, **kwargs):
+
+        username = input("Enter Username: ")
+        password = getpass("Enter Password: ")
+
+        try:
+            db = get_db()
+            with db_session:
+                p = db.Providers.select().filter(provider=self.name).first()
+
+                provider_metadata = {
+                    'provider': self.name,
+                    'username': username,
+                    'password': password,
+                }
+
+                if p is None:
+                    db.Providers(**provider_metadata)
+                else:
+                    p.set(**provider_metadata)
+
+            return True
+
+        except:
+            print("Either credentials invalid or unable to connect to HydroShare.")
+
+        return False

quest/services/noaa_coastwatch.py

@@ -65,8 +65,8 @@ def parameter_map(self, invert=False):
 
         return pmap
 
-    def download(self, feature, file_path, dataset, **params):
-        p = param.ParamOverrides(self, params)
+    def download(self, feature, file_path, dataset, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         self.parameter = p.parameter
         self.end = pd.to_datetime(p.end)
         self.start = pd.to_datetime(p.start)

quest/services/noaa_ncdc.py

@@ -74,8 +74,8 @@ def parameter_map(self, invert=False):
 
         return pmap
 
-    def download(self, feature, file_path, dataset, **params):
-        p = param.ParamOverrides(self, params)
+    def download(self, feature, file_path, dataset, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         self.parameter = p.parameter
         self.end = pd.to_datetime(p.end)
         self.start = pd.to_datetime(p.start)