Adding Secrets functionality

.github/workflows/deploy-kubernetes.yml

@@ -31,7 +31,7 @@ jobs:
  kubectl delete secret exosphere-api-server-secrets --ignore-not-found
  kubectl create secret generic exosphere-api-server-secrets --from-literal=MONGO_URI=${{ secrets.API_SERVER_MONGO_URI }} --from-literal=MONGO_DATABASE_NAME=${{ secrets.API_SERVER_MONGO_DATABASE_NAME }} --from-literal=JWT_SECRET_KEY=${{ secrets.JWT_SECRET }}
 
- kubectl create secret generic exosphere-state-manager-secrets --from-literal=MONGO_URI=${{ secrets.STATE_MANAGER_MONGO_URI }} --from-literal=MONGO_DATABASE_NAME=${{ secrets.STATE_MANAGER_MONGO_DATABASE_NAME }} --from-literal=STATE_MANAGER_SECRET=${{ secrets.STATE_MANAGER_SECRET }}
+ kubectl create secret generic exosphere-state-manager-secrets --from-literal=MONGO_URI=${{ secrets.STATE_MANAGER_MONGO_URI }} --from-literal=MONGO_DATABASE_NAME=${{ secrets.STATE_MANAGER_MONGO_DATABASE_NAME }} --from-literal=STATE_MANAGER_SECRET=${{ secrets.STATE_MANAGER_SECRET }} --from-literal=SECRETS_ENCRYPTION_KEY=${{ secrets.SECRETS_ENCRYPTION_KEY }}
 
  kubectl apply -f ./api-server-deployment.yaml
  kubectl apply -f ./api-server-service.yaml

python-sdk/README.md

@@ -130,6 +130,55 @@ class RobustNode(BaseNode):
 ```
 Error handling is automatically handled by the runtime and the state manager.
 
+### Working with Secrets
+
+Secrets allow you to securely manage sensitive configuration data like API keys, database credentials, and authentication tokens. Here's how to use secrets in your nodes:
+
+```python
+from exospherehost import Runtime, BaseNode
+from pydantic import BaseModel
+
+class APINode(BaseNode):
+ class Inputs(BaseModel):
+ user_id: str
+ query: str
+
+ class Outputs(BaseModel):
+ response: dict
+ status: str
+
+ class Secrets(BaseModel):
+ api_key: str
+ api_endpoint: str
+ database_url: str
+
+ async def execute(self) -> Outputs:
+ # Access secrets via self.secrets
+ headers = {"Authorization": f"Bearer {self.secrets.api_key}"}
+
+ # Use secrets for API calls
+ import httpx
+ async with httpx.AsyncClient() as client:
+ response = await client.post(
+ f"{self.secrets.api_endpoint}/process",
+ headers=headers,
+ json={"user_id": self.inputs.user_id, "query": self.inputs.query}
+ )
+
+ return self.Outputs(
+ response=response.json(),
+ status="success"
+ )
+```
+
+**Key points about secrets:**
+
+- **Security**: Secrets are stored securely by the ExosphereHost Runtime and are never exposed in logs or error messages
+- **Validation**: The `Secrets` class uses Pydantic for automatic validation of secret values
+- **Access**: Secrets are available via `self.secrets` during node execution
+- **Types**: Common secret types include API keys, database credentials, encryption keys, and authentication tokens
+- **Injection**: Secrets are injected by the Runtime at execution time, so you don't need to handle them manually
+
 ## Integration with ExosphereHost Platform
 
 The Python SDK integrates seamlessly with the ExosphereHost platform, providing:

python-sdk/exospherehost/_version.py

@@ -1 +1 @@
-version = "0.0.7b2"
+version = "0.0.7b3"

python-sdk/exospherehost/node/BaseNode.py

@@ -42,17 +42,44 @@ class Outputs(BaseModel):
  """
  pass
 
-async def _execute(self, inputs: Inputs) -> Outputs | List[Outputs]:
+class Secrets(BaseModel):
  """
- Internal method to execute the node with validated inputs.
+ Secrets schema for the node.
+
+ This class defines the structure for sensitive configuration data that nodes may require
+ for their execution. Secrets typically include authentication credentials, API keys,
+ database connection strings, encryption keys, and other sensitive information that
+ should not be exposed in regular input parameters.
+
+ Subclasses should override this class to define the specific secret fields their node
+ requires. The secrets are validated against this schema before node execution and
+ are made available to the node via the `self.secrets` attribute during execution.
+
+ Examples of secrets that might be defined:
+ - API keys and tokens for external services
+ - Database credentials and connection strings
+ - Encryption/decryption keys
+ - Authentication tokens and certificates
+ - Private keys for cryptographic operations
+
+ Note: The actual secret values are managed securely by the Exosphere Runtime
+ and are injected into the node at execution time.
+ """
+ pass
+
+ async def _execute(self, inputs: Inputs, secrets: Secrets) -> Outputs | List[Outputs]:
+ """
+ Internal method to execute the node with validated inputs and secrets.
 
  Args:
  inputs (Inputs): The validated input data for this execution.
+ secrets (Secrets): The validated secrets data for this execution.
 
  Returns:
  Outputs | List[Outputs]: The output(s) produced by the node.
  """
  self.inputs = inputs
+ self.secrets = secrets
  return await self.execute()
 
  @abstractmethod

python-sdk/exospherehost/runtime.py

@@ -1,7 +1,7 @@
 import asyncio
 import os
 from asyncio import Queue, sleep
-from typing import List
+from typing import List, Dict
 
 from pydantic import BaseModel, ValidationError
 from .node.BaseNode import BaseNode
@@ -117,6 +117,12 @@ def _get_register_endpoint(self):
  """
  return f"{self._state_manager_uri}/{str(self._state_manager_version)}/namespace/{self._namespace}/nodes/"
 
+ def _get_secrets_endpoint(self, state_id: str):
+ """
+ Construct the endpoint URL for getting secrets.
+ """
+ return f"{self._state_manager_uri}/{str(self._state_manager_version)}/namespace/{self._namespace}/state/{state_id}/secrets"
+
  async def _register(self):
  """
  Register node schemas and runtime metadata with the state manager.
@@ -135,6 +141,9 @@ async def _register(self):
  "namespace": self._namespace,
  "inputs_schema": node.Inputs.model_json_schema(),
  "outputs_schema": node.Outputs.model_json_schema(),
+ "secrets": [
+ secret_name for secret_name in node.Secrets.model_fields.keys()
+ ]
  } for node in self._nodes
  ]
  }
@@ -223,6 +232,23 @@ async def _notify_errored(self, state_id: str, error: str):
  if response.status != 200:
  logger.error(f"Failed to notify errored state {state_id}: {res}")
 
+ async def _get_secrets(self, state_id: str) -> Dict[str, str]:
+ """
+ Get secrets for a state.
+ """
+ async with ClientSession() as session:
+ endpoint = self._get_secrets_endpoint(state_id)
+ headers = {"x-api-key": self._key}
+
+ async with session.get(endpoint, headers=headers) as response: # type: ignore
+ res = await response.json()
+
+ if response.status != 200:
+ logger.error(f"Failed to get secrets for state {state_id}: {res}")
+ return {}
+
+ return res
+
  def _validate_nodes(self):
  """
  Validate that all provided nodes are valid BaseNode subclasses.
@@ -246,9 +272,17 @@ def _validate_nodes(self):
  if not hasattr(node, "Outputs"):
  errors.append(f"{node.__name__} does not have an Outputs class")
  if not issubclass(node.Inputs, BaseModel):
- errors.append(f"{node.__name__} does not have an Inputs class that inherits from pydantic.BaseModel")
+ errors.append(f"{node.__name__} does not have an Inputs class that inherits from pydantic.BaseModel")
  if not issubclass(node.Outputs, BaseModel):
  errors.append(f"{node.__name__} does not have an Outputs class that inherits from pydantic.BaseModel")
+ if not hasattr(node, "Secrets"):
+ errors.append(f"{node.__name__} does not have an Secrets class")
+ if not issubclass(node.Secrets, BaseModel):
+ errors.append(f"{node.__name__} does not have an Secrets class that inherits from pydantic.BaseModel")
+
+ for field_name, field_info in node.Secrets.model_fields.items():
+ if field_info.annotation is not str:
+ errors.append(f"{node.__name__}.Secrets field '{field_name}' must be of type str, got {field_info.annotation}")
 
  # Find nodes with the same __class__.__name__
  class_names = [node.__name__ for node in self._nodes]
@@ -271,7 +305,8 @@ async def _worker(self):
 
  try:
  node = self._node_mapping[state["node_name"]]
- outputs = await node()._execute(node.Inputs(**state["inputs"]))
+ secrets = await self._get_secrets(state["state_id"])
+ outputs = await node()._execute(node.Inputs(**state["inputs"]), node.Secrets(**secrets))
 
  if outputs is None:
  outputs = []

state-manager/app/controller/create_states.py

@@ -18,6 +18,7 @@ async def create_states(namespace_name: str, body: CreateRequestModel, x_exosphe
  State(
  node_name=state.node_name,
  namespace_name=namespace_name,
+ graph_name=state.graph_name,
  status=StateStatusEnum.CREATED,
  inputs=state.inputs,
  outputs={},
@@ -35,7 +36,7 @@ async def create_states(namespace_name: str, body: CreateRequestModel, x_exosphe
 
  return CreateResponseModel(
  status=StateStatusEnum.CREATED,
- states=[ResponseStateModel(state_id=str(state.id), node_name=state.node_name, inputs=state.inputs, created_at=state.created_at) for state in newStates]
+ states=[ResponseStateModel(state_id=str(state.id), node_name=state.node_name, graph_name=state.graph_name, inputs=state.inputs, created_at=state.created_at) for state in newStates]
  )
 
  except Exception as e:

state-manager/app/controller/get_secrets.py

@@ -0,0 +1,56 @@
+from app.singletons.logs_manager import LogsManager
+from app.models.secrets_response import SecretsResponseModel
+from app.models.db.state import State
+from app.models.db.graph_template_model import GraphTemplate
+from bson import ObjectId
+
+logger = LogsManager().get_logger()
+
+
+async def get_secrets(namespace_name: str, state_id: str, x_exosphere_request_id: str) -> SecretsResponseModel:
+ """
+ Get secrets for a specific state.
+
+ Args:
+ namespace_name (str): The namespace name
+ state_id (str): The state ID
+ x_exosphere_request_id (str): Request ID for logging
+
+ Returns:
+ SecretsResponseModel: The secrets for the state
+
+ Raises:
+ ValueError: If state is not found or graph template is not found
+ """
+ try:
+ # Get the state
+ state = await State.get(ObjectId(state_id))
+ if not state:
+ logger.error(f"State {state_id} not found", x_exosphere_request_id=x_exosphere_request_id)
+ raise ValueError(f"State {state_id} not found")
+
+ # Verify the state belongs to the namespace
+ if state.namespace_name != namespace_name:
+ logger.error(f"State {state_id} does not belong to namespace {namespace_name}", x_exosphere_request_id=x_exosphere_request_id)
+ raise ValueError(f"State {state_id} does not belong to namespace {namespace_name}")
+
+ # Get the graph template to retrieve secrets
+ graph_template = await GraphTemplate.find_one(
+ GraphTemplate.name == state.graph_name,
+ GraphTemplate.namespace == namespace_name
+ )
+
+ if not graph_template:
+ logger.error(f"Graph template {state.graph_name} not found in namespace {namespace_name}", x_exosphere_request_id=x_exosphere_request_id)
+ raise ValueError(f"Graph template {state.graph_name} not found in namespace {namespace_name}")
+
+ # Get the secrets from the graph template
+ secrets_dict = graph_template.get_secrets()
+
+ logger.info(f"Retrieved {len(secrets_dict)} secrets for state {state_id}", x_exosphere_request_id=x_exosphere_request_id)
+
+ return SecretsResponseModel(secrets=secrets_dict)
+
+ except Exception as e:
+ logger.error(f"Error getting secrets for state {state_id}: {str(e)}", x_exosphere_request_id=x_exosphere_request_id)
+ raise e 

state-manager/app/controller/register_nodes.py

@@ -30,7 +30,8 @@ async def register_nodes(namespace_name: str, body: RegisterNodesRequestModel, x
  RegisteredNode.runtime_name: body.runtime_name,
  RegisteredNode.runtime_namespace: namespace_name,
  RegisteredNode.inputs_schema: node_data.inputs_schema, # type: ignore
- RegisteredNode.outputs_schema: node_data.outputs_schema # type: ignore
+ RegisteredNode.outputs_schema: node_data.outputs_schema, # type: ignore
+ RegisteredNode.secrets: node_data.secrets # type: ignore
  }))
  logger.info(f"Updated existing node {node_data.name} in namespace {namespace_name}", x_exosphere_request_id=x_exosphere_request_id)
 
@@ -42,7 +43,8 @@ async def register_nodes(namespace_name: str, body: RegisterNodesRequestModel, x
  runtime_name=body.runtime_name,
  runtime_namespace=namespace_name,
  inputs_schema=node_data.inputs_schema,
- outputs_schema=node_data.outputs_schema
+ outputs_schema=node_data.outputs_schema,
+ secrets=node_data.secrets
  )
  await new_node.insert()
  logger.info(f"Created new node {node_data.name} in namespace {namespace_name}", x_exosphere_request_id=x_exosphere_request_id)
@@ -51,7 +53,8 @@ async def register_nodes(namespace_name: str, body: RegisterNodesRequestModel, x
  RegisteredNodeModel(
  name=node_data.name,
  inputs_schema=node_data.inputs_schema,
- outputs_schema=node_data.outputs_schema
+ outputs_schema=node_data.outputs_schema,
+ secrets=node_data.secrets
  )
  )
 

state-manager/app/controller/upsert_graph_template.py

@@ -18,7 +18,7 @@ async def upsert_graph_template(namespace_name: str, graph_name: str, body: Upse
  namespace_name=namespace_name,
  x_exosphere_request_id=x_exosphere_request_id)
 
- await graph_template.update(
+ await graph_template.set_secrets(body.secrets).update(
  Set({
  GraphTemplate.nodes: body.nodes, # type: ignore
  GraphTemplate.validation_status: GraphTemplateValidationStatus.PENDING, # type: ignore
@@ -40,15 +40,14 @@ async def upsert_graph_template(namespace_name: str, graph_name: str, body: Upse
  nodes=body.nodes,
  validation_status=GraphTemplateValidationStatus.PENDING,
  validation_errors=[]
- )
+ ).set_secrets(body.secrets)
  )
 
  return UpsertGraphTemplateResponse(
- name=graph_template.name,
- namespace=graph_template.namespace,
  nodes=graph_template.nodes,
  validation_status=graph_template.validation_status,
  validation_errors=graph_template.validation_errors,
+ secrets={secret_name: True for secret_name in graph_template.get_secrets().keys()},
  created_at=graph_template.created_at,
  updated_at=graph_template.updated_at
  )

state-manager/app/models/create_models.py

@@ -6,12 +6,14 @@
 
 class RequestStateModel(BaseModel):
  node_name: str = Field(..., description="Name of the node of the state")
+ graph_name: str = Field(..., description="Name of the graph template for this state")
  inputs: dict[str, Any] = Field(..., description="Inputs of the state")
 
 
 class ResponseStateModel(BaseModel):
  state_id: str = Field(..., description="ID of the state")
  node_name: str = Field(..., description="Name of the node of the state")
+ graph_name: str = Field(..., description="Name of the graph template for this state")
  inputs: dict[str, Any] = Field(..., description="Inputs of the state")
  created_at: datetime = Field(..., description="Date and time when the state was created")