Add a graph view, document and URL intake, and a namespace convention

README.md

@@ -43,6 +43,10 @@ crates/
 │ them
 ├── tinymemory-sources/ memory-source contracts and readers — local folders
 │ always, GitHub/RSS/web pages behind `network`
+├── tinymemory-documents/ document and URL intake: sniff a format, convert it
+│ to markdown, and write it into whichever engine is
+│ bound. The URL half is behind `network` and reuses the
+│ source readers' SSRF guard rather than growing a second
 ├── tinymemory-tinycortex/ the TinyCortex engine seen through the contract
 ├── tinymemory-remote/ native HTTP dialects for Supermemory, Mem0, and Cognee
 ├── tinymemory-conformance/ the behavioural suite every driver must pass
@@ -79,6 +83,8 @@ composition — no storage engine, no HTTP stack, no native library.
 | `sync` | `tinymemory::sync` — the Composio normalisers |
 | `sources` | `tinymemory::sources` — source contracts and local readers |
 | `sources-network` | `sources`, plus the GitHub/RSS/web-page readers |
+| `documents` | `tinymemory::documents` — document intake and markdown conversion |
+| `documents-network` | `documents`, plus the URL fetch path |
 | `conformance` | `tinymemory::conformance` — the driver contract suite |
 | `memory-git` | git-backed diff snapshots (implies `tinycortex`; links libgit2) |
 | `contacts` | the macOS address-book seeding path (implies `core`) |

crates/tinymemory-api/src/lib.rs

@@ -70,6 +70,13 @@
 //! - [`traits`]: the [`traits::Memory`] storage-backend trait.
 //! - [`chunks`]: the persisted chunk model ([`chunks::Chunk`], [`chunks::Metadata`],
 //! [`chunks::SourceRef`], …) and the deterministic [`chunks::chunk_id`].
+//! - [`graph`]: the bounded graph-view model ([`graph::GraphView`],
+//! [`graph::GraphViewQuery`], [`graph::GraphNode`], [`graph::GraphEdge`]) —
+//! the graph counterpart of [`tree`], and what
+//! [`provider::MemoryGraph::graph_view`] returns.
+//! - [`namespace`]: the `<section>:<scope>` namespace convention
+//! ([`namespace::Namespace`], [`namespace::MemorySection`]) and its
+//! validator.
 //! - [`tree`]: the markdown summary-tree node model ([`tree::TreeNode`],
 //! [`tree::NodeLevel`], [`tree::TreeStatus`], …).
 //! - [`tool_memory`]: tool-scoped rule contracts ([`tool_memory::ToolMemoryRule`], …).
@@ -99,7 +106,8 @@ pub mod host;
 // point: a second definition would need a conversion at the module seam that
 // nothing type-checks.
 pub use tinymemory_bus::{
- capabilities, chunks, error, goals, health, recall, tool_memory, tree, types, version, wire,
+ capabilities, chunks, error, goals, graph, health, namespace, recall, tool_memory, tree, types,
+ version, wire,
 };
 /// The mandatory-family composition: wrap any [`traits::Memory`] backend as a
 /// complete [`provider::MemoryProvider`].

crates/tinymemory-api/src/provider/knowledge.rs

@@ -10,12 +10,26 @@
 //! extraction models, hotness decay curves, and snapshot retention are driver
 //! concerns and appear in none of these signatures.
 
+use std::collections::BTreeSet;
+
 use async_trait::async_trait;
 
 use crate::error::MemoryError;
+use crate::graph::{GraphEdge, GraphNode, GraphView, GraphViewQuery};
 use crate::provider::types::{DiffReport, EntityHit, SnapshotRef};
 use crate::types::{GraphRelationRecord, MemoryKvRecord};
 
+/// How many edges the default [`MemoryGraph::graph_view`] traversal scans per
+/// predicate when it has to resolve *inbound* edges.
+///
+/// [`MemoryGraph::relations`] filters by subject and predicate but not by
+/// object, so inbound expansion has no indexed form in this contract and the
+/// default traversal falls back to a bounded scan. The bound exists so a graph
+/// larger than memory cannot be pulled into a view; hitting it sets
+/// [`GraphView::truncated`]. A driver whose store indexes the object column
+/// should override `graph_view` and skip this path entirely.
+pub const INBOUND_SCAN_LIMIT: usize = 4_096;
+
 /// The entity index: who and what the stored memory is about.
 #[async_trait]
 pub trait MemoryEntities: Send + Sync {
@@ -140,6 +154,238 @@ pub trait MemoryGraph: Send + Sync {
  /// [`MemoryError::Invalid`] for a malformed edge, otherwise backend
  /// failures.
  async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError>;
+
+ /// Assemble a bounded, renderable slice of the graph.
+ ///
+ /// This is to [`Self::relations`] what
+ /// [`crate::provider::MemoryTree::drill_down`] is to a raw node read: one
+ /// call returns a node *together with its surroundings*, already joined
+ /// into a node set and an edge set, so navigating a graph is a sequence of
+ /// view calls rather than a client-side reassembly that every caller would
+ /// write differently.
+ ///
+ /// # The default implementation
+ ///
+ /// Provided, not required: it breadth-first expands
+ /// [`GraphViewQuery::seeds`] using [`Self::relations`] alone, so every
+ /// existing driver gains a graph view without writing one, and a driver
+ /// that advertises no [`crate::capabilities::Capability::Graph`] family
+ /// still surfaces the same [`MemoryError::Unsupported`] its `relations`
+ /// returns.
+ ///
+ /// It costs one `relations` call per node visited, including one final
+ /// round at the outermost hop that adds no nodes and exists only to close
+ /// edges *between* nodes already in the view — without it the outer ring
+ /// renders as a star rather than as the graph it is. A driver with a native
+ /// multi-hop traversal should override this and use it.
+ ///
+ /// Inbound expansion has no indexed form here — `relations` cannot filter
+ /// by object — so [`crate::graph::GraphDirection::In`] and
+ /// [`crate::graph::GraphDirection::Both`] fall back to a scan capped at
+ /// [`INBOUND_SCAN_LIMIT`] per predicate.
+ ///
+ /// # Errors
+ ///
+ /// Whatever [`Self::relations`] returns. Bounds are never an error: a
+ /// traversal that hits one returns the partial view with
+ /// [`GraphView::truncated`] set.
+ async fn graph_view(&self, query: &GraphViewQuery) -> Result<GraphView, MemoryError> {
+ let namespace = query.namespace.as_deref();
+ let mut view = GraphView {
+ namespace: query.namespace.clone(),
+ seeds: query.seeds.clone(),
+ ..GraphView::default()
+ };
+
+ // Each predicate needs its own call: `relations` takes one, not a set.
+ // An empty filter becomes the single unfiltered call rather than a
+ // special case further down.
+ let predicates: Vec<Option<&str>> = if query.predicates.is_empty() {
+ vec![None]
+ } else {
+ query.predicates.iter().map(|p| Some(p.as_str())).collect()
+ };
+
+ // Nodes reached but never expanded, either because a bound was hit or
+ // because they sit one hop past the requested depth. A set, not a
+ // counter: the same boundary node is commonly reached from several
+ // directions, and counting it twice would overstate what is left.
+ let mut unexpanded: BTreeSet<String> = BTreeSet::new();
+
+ // Unseeded: an overview, not a traversal. One bounded scan of the
+ // slice, and the node set is whatever the returned edges touch.
+ if query.seeds.is_empty() {
+ for predicate in &predicates {
+ // One past the ceiling: a call that asked for exactly
+ // `max_edges` and got them cannot tell a full slice from a
+ // truncated one.
+ let records = self
+ .relations(
+ namespace,
+ None,
+ *predicate,
+ query.max_edges.saturating_add(1),
+ )
+ .await?;
+ for record in records {
+ push_view_edge(&mut view, record, &mut unexpanded, query, 0);
+ }
+ }
+ view.stats.frontier_remaining = unexpanded.len();
+ view.recompute_stats();
+ return Ok(view);
+ }
+
+ // Inbound edges are resolved from one scan per predicate rather than
+ // one per node: the scan is the expensive part, and repeating it for
+ // every node visited would multiply it by `max_nodes`.
+ let mut inbound: Vec<GraphRelationRecord> = Vec::new();
+ if query.direction.follows_in() {
+ for predicate in &predicates {
+ let records = self
+ .relations(namespace, None, *predicate, INBOUND_SCAN_LIMIT)
+ .await?;
+ if records.len() >= INBOUND_SCAN_LIMIT {
+ view.truncated = true;
+ }
+ inbound.extend(records);
+ }
+ }
+
+ let mut frontier: Vec<String> = Vec::new();
+ for seed in &query.seeds {
+ if view.nodes.iter().any(|n| &n.id == seed) {
+ continue;
+ }
+ if view.nodes.len() >= query.max_nodes {
+ unexpanded.insert(seed.clone());
+ view.truncated = true;
+ continue;
+ }
+ view.nodes.push(GraphNode::bare(seed.clone(), 0));
+ frontier.push(seed.clone());
+ }
+
+ for hop in 0..=query.depth {
+ if frontier.is_empty() {
+ break;
+ }
+ let mut next: Vec<String> = Vec::new();
+ for node_id in &frontier {
+ let mut incident: Vec<GraphRelationRecord> = Vec::new();
+ if query.direction.follows_out() {
+ for predicate in &predicates {
+ incident.extend(
+ self.relations(
+ namespace,
+ Some(node_id),
+ *predicate,
+ query.max_edges.saturating_add(1),
+ )
+ .await?,
+ );
+ }
+ }
+ if query.direction.follows_in() {
+ incident.extend(
+ inbound
+ .iter()
+ .filter(|record| &record.object == node_id)
+ .cloned(),
+ );
+ }
+
+ for record in incident {
+ if !query.accepts_predicate(&record.predicate) {
+ continue;
+ }
+ let other = if &record.subject == node_id {
+ record.object.clone()
+ } else {
+ record.subject.clone()
+ };
+ let known = view.nodes.iter().any(|n| n.id == other);
+ if !known {
+ // An edge to a node the view will not hold would
+ // dangle, so it is dropped either way — but *why* it
+ // was dropped matters. Reaching the requested depth is
+ // the caller getting what they asked for; hitting the
+ // node ceiling is not, and only the second makes the
+ // view truncated. Conflating them would set the flag on
+ // every finite traversal of a connected graph and leave
+ // it saying nothing.
+ if hop >= query.depth {
+ unexpanded.insert(other);
+ continue;
+ }
+ if view.nodes.len() >= query.max_nodes {
+ unexpanded.insert(other);
+ view.truncated = true;
+ continue;
+ }
+ view.nodes.push(GraphNode::bare(other.clone(), hop + 1));
+ next.push(other);
+ }
+ push_view_edge(&mut view, record, &mut unexpanded, query, hop);
+ }
+ }
+ frontier = next;
+ }
+
+ view.stats.frontier_remaining = unexpanded.len();
+ view.recompute_stats();
+ Ok(view)
+ }
+}
+
+/// Add one relation to a view, deduplicating by triple and honouring
+/// [`GraphViewQuery::max_edges`].
+///
+/// A separate function rather than a closure so the borrow of `view` ends
+/// between calls, which the traversal above needs while it is also pushing
+/// nodes.
+fn push_view_edge(
+ view: &mut GraphView,
+ record: GraphRelationRecord,
+ unexpanded: &mut BTreeSet<String>,
+ query: &GraphViewQuery,
+ depth: u32,
+) {
+ if !query.accepts_predicate(&record.predicate) {
+ return;
+ }
+ let triple = (
+ record.subject.clone(),
+ record.predicate.clone(),
+ record.object.clone(),
+ );
+ if view
+ .edges
+ .iter()
+ .any(|e| e.key() == (&triple.0, &triple.1, &triple.2))
+ {
+ return;
+ }
+ if view.edges.len() >= query.max_edges {
+ unexpanded.insert(triple.0);
+ unexpanded.insert(triple.2);
+ view.truncated = true;
+ return;
+ }
+ // The unseeded overview derives its node set from the edges it found; the
+ // seeded traversal has already placed both endpoints.
+ for id in [triple.0.clone(), triple.2.clone()] {
+ if view.nodes.iter().any(|n| n.id == id) {
+ continue;
+ }
+ if view.nodes.len() >= query.max_nodes {
+ unexpanded.insert(id);
+ view.truncated = true;
+ return;
+ }
+ view.nodes.push(GraphNode::bare(id, depth));
+ }
+ view.edges.push(GraphEdge::from(record));
 }
 
 /// Snapshot capture and change computation over synced sources.

crates/tinymemory-api/src/provider/mod.rs

@@ -78,7 +78,7 @@ pub use chunks::{ChunkDetail, ChunkEmbedding, ChunkQuery, MemoryChunks};
 pub use content::{MemoryDocuments, MemoryIngest, MemoryTree};
 pub use driver::MemoryProvider;
 pub use episodic::{ConversationSegment, EpisodicTurn, MemoryEpisodic};
-pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph};
+pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph, INBOUND_SCAN_LIMIT};
 pub use mandatory::{MemoryCore, MemoryPortability, MemoryRecall};
 pub use people::{
  AddressBookSeedOutcome, MemoryPeople, PersonHandle, PersonInteraction, PersonRecord, PersonRef,