feat(sqlite): vector search index

.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md

@@ -94,8 +94,8 @@ Stream cleanup worker: <error>
 GSI query returns stale data after a write
 ```
 
-**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `gsi_propagation_delay_ms` runtime setting.
+**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `index_propagation_delay_ms` runtime setting.
 
-**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set gsi_propagation_delay_ms 0`. For production-like testing, keep the default async delay.
+**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set index_propagation_delay_ms 0`. For production-like testing, keep the default async delay.
 
 **Source:** `docs/troubleshooting.md`, section "GSI Async Update Behavior", last synced 2026-05-12.

.github/workflows/integration.yml

@@ -232,44 +232,88 @@ jobs:
  echo "Server failed to start"
  exit 1
 
- - name: Provision IAM test user and access key
- id: creds
+ # Driven through devtools/run-tests rather than a bare `cargo test`, because the
+ # harness provisions what several suites require and a raw invocation does not:
+ # devtools/provision-test-credentials creates account 123456789012 with an IAM
+ # user, access key and full-access policy, and exports the credentials.
+ #
+ # batch_transact_authz targets that account and skips itself when
+ # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its
+ # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232.
+ # The vector backfill tests need the same access to set the batch delay.
+ - name: Run Rust integration tests
  env:
- EXTENDDB_PASSWORD: ${{ steps.init.outputs.admin_password }}
+ EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
+ AWS_DEFAULT_REGION: us-east-1
+ EXTENDDB_ADMIN_USER: admin
+ EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+ # PostgreSQL implements no vector search, so this is the job where the
+ # wire refusal tests must actually run. Both vector suites adapt to
+ # whatever the backend reports, so the refusal suite could skip every
+ # assertion here and still report green. "0" asserts the backend
+ # refuses vector indexes, making those tests mandatory rather than
+ # optional, and turns a silent skip into a failure.
+ EXTENDDB_EXPECT_VECTORS: "0"
+ run: devtools/run-tests --extenddb --rust-integration --release
+ run-rust-integration-sqlite:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v6
+
+ - uses: dtolnay/rust-toolchain@stable
+
+ - uses: Swatinem/rust-cache@v2
+ with:
+ cache-on-failure: true
+
+ - name: Build release (SQLite backend)
+ run: cargo build --release -p extenddb --no-default-features --features sqlite
+
+ - name: Initialize ExtendDB
+ id: init
  run: |
- acc='${{ steps.init.outputs.account_id }}'
- ./target/release/extenddb manage --user admin create-user \
- --account-id "$acc" --user-name tester
- ./target/release/extenddb manage --user admin put-user-policy \
- --account-id "$acc" --user-name tester --policy-name ddbfull \
- --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}'
- ./target/release/extenddb manage --user admin create-access-key \
- --account-id "$acc" --user-name tester > /tmp/key.json
- echo "akid=$(jq -r .access_key_id /tmp/key.json)" >> "$GITHUB_OUTPUT"
- secret=$(jq -r .secret_access_key /tmp/key.json)
- echo "::add-mask::$secret"
- echo "secret=$secret" >> "$GITHUB_OUTPUT"
+ output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1)
+ echo "$output"
+ echo "admin_password=$(echo "$output" | grep -oP 'Password: \K\S+')" >> "$GITHUB_OUTPUT"
+
+ - name: Enable provisioned-capacity throttling enforcement
+ run: ./target/release/extenddb settings set throttling_enabled true
+
+ - name: Start ExtendDB
+ run: |
+ ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file &
+ for i in $(seq 1 30); do
+ if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then
+ echo "Server ready"
+ exit 0
+ fi
+ sleep 1
+ done
+ echo "Server failed to start"
+ exit 1
 
+ # Driven through devtools/run-tests rather than a bare `cargo test`, because the
+ # harness provisions what several suites require and a raw invocation does not:
+ # devtools/provision-test-credentials creates account 123456789012 with an IAM
+ # user, access key and full-access policy, and exports the credentials.
+ #
+ # batch_transact_authz targets that account and skips itself when
+ # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its
+ # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232.
+ # The vector backfill tests need the same access to set the batch delay.
  - name: Run Rust integration tests
  env:
  EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
  AWS_DEFAULT_REGION: us-east-1
- AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }}
- AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }}
- # No in-tree backend implements vector search, so every vector request
- # must be refused and the refusal suite must actually run. Without this
- # the suite self-skips the moment a backend gains vector support, and the
- # contract that non-participating backends refuse would stop being
- # checked anywhere while still reporting green. Pinning the expectation
- # turns that skip into a failure. The first backend to implement vector
- # search sets this to 1 in its own job.
- EXTENDDB_EXPECT_VECTORS: "0"
- run: |
- # Self-signed cert generated by init; trust it for the SDK client.
- export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)"
- # The suite shares one SDK client across tests; the hyper connection
- # pool is bound to the first test's runtime, so run serially.
- cd tests/rust && cargo test -- --test-threads=1
+ EXTENDDB_ADMIN_USER: admin
+ EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+ # Both vector suites self-skip when the backend is the wrong kind, so without
+ # this the positive suite could skip every assertion and still report green,
+ # which is what would happen if the backend lost the capability. Pinning the
+ # expectation turns that skip into a failure.
+ EXTENDDB_EXPECT_VECTORS: "1"
+ run: devtools/run-tests --extenddb --rust-integration --release
 
  integration:
  runs-on: ubuntu-latest
@@ -279,13 +323,15 @@ jobs:
  run-integration-sqlite,
  run-integration-dev-mode,
  run-rust-integration,
+ run-rust-integration-sqlite,
  ]
  if: always()
  steps:
  - run: |
  if [ "${{ needs.run-integration.result }}" != "success" ] || \
  [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \
  [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \
- [ "${{ needs.run-rust-integration.result }}" != "success" ]; then
+ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \
+ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then
  exit 1
  fi

crates/app/src/cmd_settings.rs

@@ -112,6 +112,9 @@ async fn set(store: &dyn SettingsStore, key: &str, value: &str) -> anyhow::Resul
  );
  }
 
+ // Write under the canonical name so the deprecated alias updates the row the read
+ // path consults, rather than adding a second one that is silently ignored.
+ let key = extenddb_core::settings_keys::canonical_key(key);
  store
  .set_setting(key, value)
  .await

crates/core/src/expression/search_condition.rs

@@ -130,6 +130,21 @@ pub fn validate_search_condition_expression(
  ))
  })?
  } else {
+ // A bare identifier must not be a reserved word. This check existed for
+ // ProjectionExpression, where its message is byte-identical to the
+ // service, but was never wired into this expression type, so
+ // `cat = :c AND bucket = :b` was accepted and returned results where the
+ // service refuses it. Measured on 2026-08-11.
+ //
+ // Applied to the bare form only, which is the point of the rule: the
+ // `#alias` branch above exists precisely so a reserved word can be used
+ // by aliasing it.
+ if crate::expression::reserved_words::is_reserved(lhs) {
+ return Err(invalid(format!(
+ "Invalid SearchConditionExpression: Attribute name is a reserved \
+ keyword; reserved keyword: {lhs}"
+ )));
+ }
  (*lhs).to_owned()
  };
  if attr_name.contains('.') || attr_name.contains('[') {
@@ -219,20 +234,12 @@ pub fn validate_conditions_against_search_schema(
 ) -> Result<(), DynamoDbError> {
  let schema = search_schema.unwrap_or(&[]);
 
- // Every referenced attribute must be part of the index search schema.
- for condition in conditions {
- let in_schema = schema
- .iter()
- .any(|element| element.attribute_name == condition.attribute_name);
- if !in_schema {
- return Err(invalid(
- "SearchConditionExpression must not contain any attributes outside the vector \
- index search schema",
- ));
- }
- }
-
  // Every partition-key (HASH) element must be present in the conditions.
+ //
+ // Ordered BEFORE the out-of-schema check deliberately: measured against the
+ // service 2026-08-19 (us-east-1), an expression whose only attribute is
+ // out-of-schema AND which omits the HASH is refused for the missing HASH,
+ // not the unknown attribute. Missing-HASH takes precedence.
  for element in schema
  .iter()
  .filter(|element| element.element_type == SearchSchemaElementType::Hash)
@@ -242,11 +249,34 @@ pub fn validate_conditions_against_search_schema(
  .any(|condition| condition.attribute_name == element.attribute_name);
  if !present {
  return Err(invalid(
- "SearchConditionExpression must have all HASH attributes",
+ // Measured against the service 2026-08-19 (us-east-1): an
+ // expression that omits a declared HASH attribute is refused
+ // with exactly this text.
+ "SearchConditionExpression must have all HASH attributes in configured SearchSchema",
  ));
  }
  }
 
+ // Every referenced attribute must be part of the index search schema.
+ //
+ // Wording measured against the service on 2026-08-11. It names the offending
+ // attribute, which the previous text did not, so a caller with several
+ // conditions had to work out which one was at fault. The service's own grammar
+ // slip ("attributes that is not") is reproduced deliberately: parity means
+ // matching what clients actually receive, not correcting it.
+ for condition in conditions {
+ let in_schema = schema
+ .iter()
+ .any(|element| element.attribute_name == condition.attribute_name);
+ if !in_schema {
+ return Err(invalid(format!(
+ "SearchConditionExpression must not contain any attributes that is not in \
+ SearchSchema. Invalid attribute: {}",
+ condition.attribute_name
+ )));
+ }
+ }
+
  // Each value type must agree with the declared attribute type.
  for condition in conditions {
  if let Some(definition) = attribute_definitions
@@ -299,6 +329,71 @@ mod tests {
  }
  }
 
+ /// A bare reserved word is refused, with the service's wording.
+ ///
+ /// The check already existed for `ProjectionExpression`, where its message is
+ /// byte-identical to the service, but was never applied here, so
+ /// `bucket = :b` was accepted and returned results for a request the service
+ /// refuses. `bucket` is the specific word that exposed it against the live
+ /// service on 2026-08-11.
+ #[test]
+ fn a_bare_reserved_keyword_is_refused() {
+ let v = values(&[(":b", "4")]);
+ let message = err(validate_search_condition_expression(
+ "bucket = :b",
+ None,
+ Some(&v),
+ ));
+ assert_eq!(
+ message,
+ "Invalid SearchConditionExpression: Attribute name is a reserved keyword; \
+ reserved keyword: bucket"
+ );
+ }
+
+ /// Aliasing is the documented escape hatch, so the rule must apply to the bare
+ /// form only. Without this, the fix above would make reserved-word attributes
+ /// unusable rather than merely requiring an alias.
+ #[test]
+ fn an_aliased_reserved_keyword_is_accepted() {
+ let n = names(&[("#b", "bucket")]);
+ let v = values(&[(":b", "4")]);
+ let conditions = validate_search_condition_expression("#b = :b", Some(&n), Some(&v))
+ .expect("an aliased reserved word must be allowed");
+ assert_eq!(conditions.len(), 1);
+ assert_eq!(conditions[0].attribute_name, "bucket");
+ }
+
+ /// The condition is rejected when it names an attribute outside the schema, and
+ /// the message names WHICH attribute, as the service's does.
+ #[test]
+ fn an_attribute_outside_the_search_schema_is_named_in_the_error() {
+ let conditions = vec![
+ SearchCondition {
+ attribute_name: "cat".to_owned(),
+ value: AttributeValue::S("alpha".to_owned()),
+ },
+ SearchCondition {
+ attribute_name: "payload".to_owned(),
+ value: AttributeValue::S("nope".to_owned()),
+ },
+ ];
+ let schema = [SearchSchemaElement {
+ attribute_name: "cat".to_owned(),
+ element_type: SearchSchemaElementType::Hash,
+ }];
+ let message = err(validate_conditions_against_search_schema(
+ &conditions,
+ Some(&schema),
+ &[],
+ ));
+ assert_eq!(
+ message,
+ "SearchConditionExpression must not contain any attributes that is not in \
+ SearchSchema. Invalid attribute: payload"
+ );
+ }
+
  #[test]
  fn single_literal_equality_resolves() {
  let v = values(&[(":cat", "Electronics")]);

crates/core/src/expression/update_evaluator_tests.rs

@@ -5,6 +5,7 @@ use super::*;
 use crate::expression::resolver::ExpressionMaps;
 use crate::expression::tokenizer::tokenize;
 use crate::expression::update_parser::parse_update;
+use crate::types::{Projection, ProjectionType};
 use std::collections::HashMap;
 
 fn apply(
@@ -34,6 +35,10 @@ mod vector_validated {
  dimensions: DIMS,
  vector_attribute_name: "emb".to_owned(),
  search_schema: Vec::new(),
+ projection: Projection {
+ projection_type: ProjectionType::All,
+ non_key_attributes: None,
+ },
  }
  }
 
@@ -125,7 +130,7 @@ mod vector_validated {
  item.insert("other".into(), AttributeValue::S("not-a-vector".into()));
  let err = apply_validated("SET emb = other", &mut item, HashMap::new()).unwrap_err();
  assert!(
- matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Expected: a list of numbers")),
+ matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Invalid type for parameter emb")),
  "unexpected error: {err:?}"
  );
  }
@@ -181,6 +186,10 @@ mod vector_validated {
  attribute_name: "tenant".to_owned(),
  element_type: crate::types::SearchSchemaElementType::Hash,
  }],
+ projection: Projection {
+ projection_type: ProjectionType::All,
+ non_key_attributes: None,
+ },
  };
  let defs = [crate::types::AttributeDefinition {
  attribute_name: "tenant".to_owned(),
@@ -212,6 +221,10 @@ mod vector_validated {
  attribute_name: "tenant".to_owned(),
  element_type: crate::types::SearchSchemaElementType::Hash,
  }],
+ projection: Projection {
+ projection_type: ProjectionType::All,
+ non_key_attributes: None,
+ },
  };
  let defs = [crate::types::AttributeDefinition {
  attribute_name: "tenant".to_owned(),
@@ -246,6 +259,10 @@ mod vector_validated {
  attribute_name: "tenant".to_owned(),
  element_type: crate::types::SearchSchemaElementType::Hash,
  }],
+ projection: Projection {
+ projection_type: ProjectionType::All,
+ non_key_attributes: None,
+ },
  };
  let defs = [crate::types::AttributeDefinition {
  attribute_name: "tenant".to_owned(),

crates/core/src/lib.rs

@@ -12,6 +12,7 @@ pub mod expression;
 pub mod limits;
 pub mod metrics;
 pub mod serde_helpers;
+pub mod settings_keys;
 pub mod throttle;
 pub mod types;
 pub mod validation;