Nasa Provider Plugin

quest/api/datasets.py

@@ -60,10 +60,10 @@ def download(feature, file_path, dataset=None, **kwargs):
 
 
 @add_async
-def publish(publisher_uri, options=None):
+def publish(publisher_uri, **kwargs):
     provider, publisher, feature = util.parse_service_uri(publisher_uri)
     driver = util.load_providers()[provider]
-    data = driver.publish(publisher=publisher, options=options)
+    data = driver.publish(publisher=publisher, **kwargs)
     return data
 
 @add_async

quest/api/services.py

@@ -127,6 +127,7 @@ def add_provider(uri):
             util.update_settings({'USER_SERVICES': user_services})
             util.save_settings()
             msg = 'service added'
+            util.load_providers(update_cache=True)
         else:
             msg = 'service already present'
     else:
@@ -159,6 +160,7 @@ def delete_provider(uri):
         util.update_settings({'USER_SERVICES': user_services})
         util.save_settings()
         msg = 'service removed'
+        util.load_providers(update_cache=True)
     else:
         msg = 'service not found'
 

quest/services/base/provider_base.py

@@ -283,8 +283,8 @@ def download_options(self, service, fmt):
         """
         return self.services[service].download_options(fmt)
 
-    def publish(self, publisher, options):
-        return self.publishers[publisher].publish(options)
+    def publish(self, publisher, **kwargs):
+        return self.publishers[publisher].publish(**kwargs)
 
     def publish_options(self, publisher, fmt):
         return self.publishers[publisher].publish_options(fmt)

quest/services/base/publish_base.py

@@ -38,5 +38,5 @@ def publish_options(self, fmt):
 
         return schema
 
-    def publish(self, options):
+    def publish(self, **kwargs):
         raise NotImplementedError()

quest/services/base/service_base.py

@@ -92,7 +92,7 @@ def download_options(self, fmt):
 
         return schema
 
-    def download(self, feature, file_path, dataset, **params):
+    def download(self, feature, file_path, dataset, **kwargs):
         raise NotImplementedError()
 
     def get_features(self, **kwargs):
@@ -138,7 +138,7 @@ class SingleFileServiceBase(ServiceBase):
     """Base file for datasets that are a single file download
     eg elevation raster etc
     """
-    def download(self, feature, file_path, dataset, **params):
+    def download(self, feature, file_path, dataset, **kwargs):
         feature_id = util.construct_service_uri(self.provider.name, self.name, feature)
         feature = self.provider.get_features(self.name).loc[feature_id]
         reserved = feature.get('reserved')

quest/services/cuahsi_hs.py

@@ -9,6 +9,7 @@
 import param
 import os
 
+
 class HSServiceBase(SingleFileServiceBase):
 
     @property
@@ -35,7 +36,7 @@ class HSGeoService(HSServiceBase):
 
     def get_features(self, **kwargs):
 
-        results = list(self.hs.resources())
+        results = list(self.hs.resources(coverage_type="box", north="90", south="-90", east="180", west="-180"))
 
         if len(results) == 0:
             raise ValueError('No resource available from HydroShare.')
@@ -126,21 +127,18 @@ class HSPublisher(PublishBase):
         'Time Series': 'TimeSeriesResource'
     }
 
-    title = param.String(default="example title", doc="Title of resource", precedence=2)
-    abstract = param.String(default="example abstract", precedence=3,
-                            doc="An description of the resource to be added to HydroShare.")
-    keywords = param.List(default=[], precedence=4, doc="list of keyword strings to describe the resource")
-    dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, precedence=5,
-                                             doc="dataset to publish to HydroShare")
-    resource_type = param.ObjectSelector(doc='parameter', precedence=1, objects=sorted(_resource_type_map.keys()))
+    resource_type = param.ObjectSelector(default=None, doc="", precedence=1, objects=sorted(_resource_type_map.keys()))
+    title = param.String(default="", doc="", precedence=2)
+    abstract = param.String(default="", doc="", precedence=3)
+    keywords = param.List(default=[], doc="", precedence=4)
+    dataset = param_util.DatasetListSelector(default=(), filters={'status': 'downloaded'}, doc="", precedence=5)
 
     @property
     def hs(self):
         return self.provider.get_hs()
 
-    def publish(self, options=None):
-
-        p = param.ParamOverrides(self, options)
+    def publish(self, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         valid_file_paths = []
         valid_extensions = []
 

quest/services/kitware_girder.py

@@ -34,9 +34,9 @@ class GirderPublisher(PublishBase):
     def gc(self):
         return self.provider.get_gc()
 
-    def publish(self, options=None):
+    def publish(self, **kwargs):
         try:
-            p = param.ParamOverrides(self, options)
+            p = param.ParamOverrides(self, kwargs)
             params = {'name': p.title, 'description': p.collection_description}
             resource_information_dict = self.gc.createResource(path='collection', params=params)
             folder_creation_dict = self.gc.createFolder(parentId=resource_information_dict['_id'],

quest/services/nasa.py

@@ -2,6 +2,8 @@
 
 """
 from .base import ProviderBase, SingleFileServiceBase
+from ..api.database import get_db, db_session
+from getpass import getpass
 import pandas as pd
 import requests
 
@@ -20,8 +22,15 @@ class NasaServiceBase(SingleFileServiceBase):
         'elevation': 'elevation'
     }
 
-    def _read_granules(short_name, page_num):
-        return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry']
+    @property
+    def info(self):
+        return self.provider.get_user_info()
+
+    def _read_granules(self, short_name, page_num):
+        try:
+            return requests.get(granules_url % (short_name, page_num), auth=(self.info['username'], self.info['password'])).json()['feed']['entry']
+        except ValueError:
+            return requests.get(granules_url % (short_name, page_num)).json()['feed']['entry']
 
     def get_features(self, **kwargs):
         page_num = 0
@@ -128,3 +137,35 @@ class NasaProvider(ProviderBase):
     description = 'Services available through the NASA'
     organization_name = 'National Aeronautic and Space Administration'
     organization_abbr = 'NASA'
+
+    def get_user_info(self):
+        the_info = self.credentials
+        return the_info
+
+    def authenticate_me(self, **kwargs):
+
+        username = input("Enter Username: ")
+        password = getpass("Enter Password: ")
+
+        try:
+            db = get_db()
+            with db_session:
+                p = db.Providers.select().filter(provider=self.name).first()
+
+                provider_metadata = {
+                    'provider': self.name,
+                    'username': username,
+                    'password': password,
+                }
+
+                if p is None:
+                    db.Providers(**provider_metadata)
+                else:
+                    p.set(**provider_metadata)
+
+            return True
+
+        except:
+            print("Either credentials invalid or unable to connect to HydroShare.")
+
+        return False

quest/services/noaa_coastwatch.py

@@ -65,8 +65,8 @@ def parameter_map(self, invert=False):
 
         return pmap
 
-    def download(self, feature, file_path, dataset, **params):
-        p = param.ParamOverrides(self, params)
+    def download(self, feature, file_path, dataset, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         self.parameter = p.parameter
         self.end = pd.to_datetime(p.end)
         self.start = pd.to_datetime(p.start)

quest/services/noaa_ncdc.py

@@ -74,8 +74,8 @@ def parameter_map(self, invert=False):
 
         return pmap
 
-    def download(self, feature, file_path, dataset, **params):
-        p = param.ParamOverrides(self, params)
+    def download(self, feature, file_path, dataset, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
         self.parameter = p.parameter
         self.end = pd.to_datetime(p.end)
         self.start = pd.to_datetime(p.start)

quest/services/template_service.py

@@ -26,7 +26,7 @@ class ExampleServiceBase(ServiceBase):
     smtk_template = None
     _parameter_map = dict()
 
-    def download(self, feature, file_path, dataset, **params):
+    def download(self, feature, file_path, dataset, **kwwargs):
         metadata = {}  # get metadata from service
         data = None  # data structure containing downloaded data
 

quest/services/user_provider.py

@@ -51,7 +51,7 @@ def instance(cls, service_name, service_data, provider, uri, is_remote):
 
             return self
 
-        def download(self, feature, file_path, dataset, **params):
+        def download(self, feature, file_path, dataset, **kwargs):
             if self.datasets_mapping is not None:
                 fnames = self.datasets_mapping
                 if isinstance(dict, self.datasets_mapping):

quest/services/usgs_nwis.py

@@ -18,8 +18,8 @@ class NwisServiceBase(TimePeriodServiceBase):
     period = param.String(default='P365D', precedence=4, doc='time period (e.g. P365D = 365 days or P4W = 4 weeks)')
     smtk_template = 'start_end_or_period.sbt'
 
-    def download(self, feature, file_path, dataset, **params):
-        p = param.ParamOverrides(self, params)
+    def download(self, feature, file_path, dataset, **kwargs):
+        p = param.ParamOverrides(self, kwargs)
 
         parameter = p.parameter
         start = p.start

quest/util/misc.py

@@ -19,6 +19,7 @@
 from uuid import uuid4, UUID
 import quest
 
+the_providers = None
 
 def generate_cache(update=False):
     """Downloads features for all services and caches results.
@@ -238,22 +239,27 @@ def load_drivers(namespace, names=None):
     return {name: driver.DriverManager(namespace, name, invoke_on_load='True') for name in names}
 
 
-def load_providers():
-    settings = get_settings()
+def load_providers(update_cache=False):
+    global the_providers
 
-    # add web services
+    settings = get_settings()
     web_services = list_drivers('services')
     web_services.remove('user')
 
-    providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services}
+    if update_cache or the_providers is None:
+        providers = {name: driver.DriverManager('quest.services', name, invoke_on_load=True, invoke_kwds={'name': name}).driver for name in web_services}
 
-    if len(settings.get('USER_SERVICES', [])) > 0:
-        for uri in settings.get('USER_SERVICES', []):
-            try:
-                drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver
-                providers['user-' + drv.name] = drv
-            except Exception as e:
-                logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e)))
+        if len(settings.get('USER_SERVICES', [])) > 0:
+            for uri in settings.get('USER_SERVICES', []):
+                try:
+                    drv = driver.DriverManager('quest.services', 'user', invoke_on_load=True, invoke_kwds={'uri': uri}).driver
+                    providers['user-' + drv.name] = drv
+                except Exception as e:
+                    logger.error('Failed to load local service from %s, with exception: %s' % (uri, str(e)))
+
+        the_providers = providers
+    else:
+        providers = the_providers
 
     return providers
 

setup.cfg

@@ -37,7 +37,7 @@ quest.services =
     usgs-nlcd = quest.services.usgs_nlcd:UsgsNlcdProvider
     cuahsi-hydroshare = quest.services.cuahsi_hs:HSProvider
     kitware-girder = quest.services.kitware_girder:GirderProvider
-    # nasa = quest.services.nasa:NasaProvider
+    nasa = quest.services.nasa:NasaProvider
 
 quest.filters =
     ts-unit-conversion = quest.filters.timeseries.timeseries:TsUnitConversion

test/data.py

@@ -3,8 +3,8 @@
 """
 
 DOWNLOAD_OPTIONS_FROM_ALL_SERVICES = {
- # 'svc://nasa:srtm-3-arc-second': {},
- # 'svc://nasa:srtm-30-arc-second': {},
+ 'svc://nasa:srtm-3-arc-second': {},
+ 'svc://nasa:srtm-30-arc-second': {},
  'svc://noaa-ncdc:ghcn-daily': {'properties': [{'default': None,
     'description': 'parameter',
     'name': 'parameter',