Skip to content
82 changes: 80 additions & 2 deletions crates/utopia-extract/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ pub struct ExtractedFact {
/// 同上,宾语那一侧
#[serde(default)]
pub object_span: Option<String>,
/// 日期属性的值只相对一件事给出(「触发日后 45 天」),没有日历上的日期(#681 §4)。
/// 模型判断、模型标;服务端不认这类说法的词,只看这个标记决定收不收
#[serde(default)]
pub relative: bool,
}

/// 提示词里的一条关系。
Expand Down Expand Up @@ -291,7 +295,8 @@ pub fn build_messages_with_opening(
let attr_rules = if attributes.is_empty() {
String::new()
} else {
"\n10. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one); bool = true/false; \
"\n10. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one) — a date the text gives only relative to an event \
(\"45 days after the Trigger Date\", \"within 30 days of closing\") has no calendar date to convert: write it as the text writes it and add \"relative\": true; bool = true/false; \
text = a short string. Only attach an attribute to a subject of its listed class. \
valid_from = when this value took effect, if the text or the opening of the document says so. \
A document that changes a value set earlier — amends, extends or replaces it — makes the new value \
Expand Down Expand Up @@ -377,7 +382,8 @@ pub fn build_messages_with_opening(
units and all — except a date, which is always written in the format of rule 3 \
(\"June 23, 2020\" is \"2020-06-23\"): the server keeps a date only in that \
format, and a date written any other way is lost. A deadline or a period stated \
relative to an event, with no calendar date, is not a date. \
relative to an event, with no calendar date, is not a date: keep it as written \
and mark it \"relative\" as rule 10 says. \
**A stated figure left out is the loss that costs most**: the reader \
came for those numbers, and no later step can recover one that was never written \
down.\n\
Expand Down Expand Up @@ -1045,6 +1051,25 @@ fn next_token(s: &str) -> (&str, &str) {
(&s[..end], &s[end..])
}

/// 一个属性值落库时的样子:按 datatype 归一成 `{"value": …}`;失败返回 None,调用方记
/// `attr_datatype`。
///
/// 日期属性上一个**相对**的值(#681 §4):解不成日期、模型又标了 `relative`、写的是一段非空
/// 文字时,照原文收下,值里带 `"relative": true`。它不是日期,从不当日期比较或排序。解得成
/// 日期的照日期存,标错了也不当相对;没标的非日期值仍然不收
pub fn attr_object_value(
datatype: &str,
raw: &serde_json::Value,
relative: bool,
) -> Option<serde_json::Value> {
if let Some(value) = normalize_attr_value(datatype, raw) {
return Some(serde_json::json!({ "value": value }));
}
let written = raw.as_str().map(str::trim).filter(|s| !s.is_empty())?;
(datatype == "date" && relative)
.then(|| serde_json::json!({ "value": written, "relative": true }))
}

/// 属性值按 datatype 归一。失败返回 None——宁缺勿脏,调用方跳过并记日志。
/// number 容忍千分位/空格;date 要求 YYYY[-MM[-DD]] 且保留原精度;bool 宽容 yes/no。
pub fn normalize_attr_value(datatype: &str, raw: &serde_json::Value) -> Option<serde_json::Value> {
Expand Down Expand Up @@ -1169,6 +1194,59 @@ fn split_zone(clock: &str) -> Option<(&str, chrono::Duration)> {
mod prompt_shape_tests {
use super::*;

/// 第十份补充协议把截止日改成「触发日后 45 天」:模型标 relative,服务端照原文收;
/// 没标的、不是日期属性的、空的都不走这条
#[test]
fn a_relative_date_is_kept_as_written_only_when_marked() {
let raw = serde_json::json!("45 days after the Trigger Date");
assert_eq!(
attr_object_value("date", &raw, true),
Some(
serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true })
)
);
assert_eq!(attr_object_value("date", &raw, false), None, "没标就不收");
assert_eq!(
attr_object_value("date", &serde_json::json!(" "), true),
None
);
// 解得成日期的照日期存,标了 relative 也不当相对
assert_eq!(
attr_object_value("date", &serde_json::json!("2020-06-23"), true),
Some(serde_json::json!({ "value": "2020-06-23" }))
);
// 不是日期属性:只按它自己的 datatype 归一,relative 不起作用
assert_eq!(
attr_object_value("bool", &serde_json::json!("45 days after"), true),
None
);
assert_eq!(
attr_object_value("number", &serde_json::json!("1,250"), true),
Some(serde_json::json!({ "value": 1250.0 }))
);
let fact: ExtractedFact = serde_json::from_value(serde_json::json!({
"subject": "Lease", "predicate": "expansion_option_deadline",
"value": "45 days after the Trigger Date", "relative": true
}))
.unwrap();
assert!(fact.relative);
let plain: ExtractedFact = serde_json::from_value(serde_json::json!({
"subject": "Lease", "predicate": "expansion_option_deadline", "value": "2020-06-23"
}))
.unwrap();
assert!(!plain.relative, "没写就不是");
let msgs = build_messages(
&[],
&[],
&["lease.deadline (date)".into()],
None,
"a.txt",
&[],
"text",
);
assert!(msgs[0].content.contains("add \"relative\": true"));
}

fn rel(key: &str, description: &str, signature: &str) -> PromptRelation {
PromptRelation {
key: key.into(),
Expand Down
2 changes: 2 additions & 0 deletions crates/utopia-extract/src/normalize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ fn value_fact(
quote: f.quote.clone(),
subject_span: f.subject_span.clone(),
object_span: None,
relative: false,
}
}

Expand Down Expand Up @@ -536,6 +537,7 @@ mod tests {
quote: Some(quote.into()),
subject_span: Some(subject.into()),
object_span: object.map(str::to_string),
relative: false,
}
}
fn valued(subject: &str, predicate: &str, value: &str, quote: &str) -> ExtractedFact {
Expand Down
23 changes: 13 additions & 10 deletions crates/utopia-server/src/extraction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1726,7 +1726,11 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
}
};
let datatype = attr.datatype.as_deref().unwrap_or("text");
let Some(normalized) = utopia_extract::normalize_attr_value(datatype, &raw) else {
// 只相对一件事给出的日期(「触发日后 45 天」,#681 §4)照原文收下、带着标记:
// 它是新的状态值,时态引擎照常用它接替前一个截止日
let Some(mut object_value) =
utopia_extract::attr_object_value(datatype, &raw, f.relative)
else {
tracing::debug!(%document_id, attr = attr.key, ?raw, "属性值不合 datatype,跳过");
drop_signal(
state,
Expand All @@ -1739,7 +1743,6 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
.await;
continue;
};
let mut object_value = serde_json::json!({ "value": normalized });
// 单位随事实落笔:类型上的单位以后改了,旧值仍按记录时的单位读。
// 记哪个单位照 `unit_for`——从前这里无条件盖上声明的单位,实测
//「提供 500 兆瓦的风电」被模型记成金额,再盖上 ¥ 就成了 500 块钱
Expand Down Expand Up @@ -1791,11 +1794,11 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
Some(f.predicate.as_str()),
)
.await?;
if !created {
continue;
if created {
fact_count += 1;
}
fact_count += 1;
// 单值属性 = functional:新值闭合旧值(属性历史由此而来
// 单值属性 = functional:新值闭合旧值(属性历史由此而来)。并进已有断言的也对:
// 这份证据的日期可能更早,时间线的形状跟着变(#679
if attr.functional && attr.temporal == "state" {
let report = utopia_store::temporal::reconcile_new_fact(
&state.pool,
Expand Down Expand Up @@ -2757,12 +2760,12 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
Some(f.predicate.as_str()),
)
.await?;
if !created {
continue;
if created {
fact_count += 1;
}
fact_count += 1;
// 时态对账:带唯一性约束的状态关系落新事实即检测矛盾(纯规则点查,
// 自动闭合走"作废+改写",拿不准进 fact_conflicts 人裁)
// 自动闭合走"作废+改写",拿不准进 fact_conflicts 人裁)。并进已有断言的
// 也对:多了一份证据,时间线的形状可能跟着变(#679)
// 没有谓词就没有关系元数据,也就不参与时态对账——
// 一条说不出是什么关系的边,本来就不可能带唯一性约束
if let Some((pid, (func, inv_func, temporal))) =
Expand Down
43 changes: 42 additions & 1 deletion crates/utopia-server/src/rdf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,10 @@ pub fn emit_fact(
if let Some(o) = &object {
sink.triple(TripleRef::new(stmt.as_ref(), rdf::OBJECT, o.as_ref()))?;
}
// 只相对一件事给出的值(「触发日后 45 天」,#681 §4):字面量是原文,这一行说它不是日期
if f.object_value.as_ref().is_some_and(is_relative) {
sink.l(&stmt, &utopia("relativeValue"), &flag(true))?;
}
emit_validity(
sink,
&stmt,
Expand Down Expand Up @@ -626,7 +630,13 @@ fn emit_validity(
Ok(())
}

/// 属性事实的字面值。`{"value": …, "unit": …}` 或 `{"summary": …}`
/// 值上带着 `"relative": true`:原文只相对一件事给出它,没有日历上的日期
fn is_relative(v: &serde_json::Value) -> bool {
v.get("relative").and_then(|r| r.as_bool()) == Some(true)
}

/// 属性事实的字面值。`{"value": …, "unit": …}` 或 `{"summary": …}`。
/// 相对的值写成普通字符串:`"45 days after the Trigger Date"^^xsd:date` 是个不合法的字面量
fn literal_value(v: &serde_json::Value, datatype: Option<&str>) -> Literal {
let raw = v.get("value").unwrap_or(v);
let as_text = match raw {
Expand All @@ -639,6 +649,7 @@ fn literal_value(v: &serde_json::Value, datatype: Option<&str>) -> Literal {
other => other.to_string(),
};
let ty: NamedNodeRef<'_> = match datatype {
_ if is_relative(v) => xsd::STRING,
Some("number") => xsd::DECIMAL,
Some("date") => xsd::DATE,
Some("bool") => xsd::BOOLEAN,
Expand Down Expand Up @@ -951,6 +962,36 @@ mod tests {
);
}

/// 日期属性上相对的值(#681 §4)导出成普通字符串,陈述上另有一行说它是相对的——
/// 写成 xsd:date 的字面量不合法,严格的解析器会整份拒收
#[test]
fn a_relative_deadline_is_a_string_that_says_it_is_relative() {
let dated = literal_value(&serde_json::json!({ "value": "2020-06-23" }), Some("date"));
assert_eq!(dated.datatype(), xsd::DATE);
let relative = literal_value(
&serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true }),
Some("date"),
);
assert_eq!(relative.datatype(), xsd::STRING);
assert_eq!(relative.value(), "45 days after the Trigger Date");

let mut attr = fact(5);
attr.predicate_id = Some(id(4));
attr.object_id = None;
attr.object_value = Some(
serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true }),
);
let quads = export(Format::Turtle, |sink, names, vocab| {
emit_fact(sink, names, vocab, &attr, at("2026-06-01T00:00:00Z")).unwrap();
});
assert!(has(
&quads,
STMT,
"urn:utopia:ns:relativeValue",
"\"true\"^^<http://www.w3.org/2001/XMLSchema#boolean>"
));
}

/// 业务规则的结论也要出现在导出里,而且宾语是**字面值**。
///
/// 这一条挡的是一次静默丢失:取数那边原本 `JOIN rules`,而业务规则的
Expand Down
86 changes: 85 additions & 1 deletion crates/utopia-store/src/documents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -822,11 +822,15 @@ pub async fn delete(
.bind(id)
.fetch_all(&mut *tx)
.await?;
// 作废的事实可能是别的值的后任:它走了,关在它开始时的前任要重新接上;没作废、只是
// 少了这份证据的,排序用的日期也可能变。牵连的时间线先按固定顺序锁上,再作废任何一行
// (与撤回合并同一个顺序,见 temporal 模块头)
let (cited, timelines) = lock_cited_timelines(&mut tx, kb_id, id, &[]).await?;
// 先打了墓碑再算:这篇文档此刻已经算「已删除」,所以只剩它作出处的事实才落网;
// 另一篇活着的文档里也有证据的一条不动——删一份重复上传不该掀掉半张图
let facts: Vec<(Uuid,)> = sqlx::query_as(
"UPDATE facts f SET invalidated_at = now()
WHERE f.kb_id = $1 AND f.invalidated_at IS NULL
WHERE f.kb_id = $1 AND f.invalidated_at IS NULL AND f.id = ANY($3)
AND EXISTS (SELECT 1 FROM fact_evidence fe
JOIN chunks c ON c.id = fe.chunk_id
WHERE fe.fact_id = f.id AND c.document_id = $2)
Expand All @@ -844,10 +848,13 @@ pub async fn delete(
)
.bind(kb_id)
.bind(id)
.bind(&cited)
.fetch_all(&mut *tx)
.await?;
let chunk_ids: Vec<Uuid> = chunks.into_iter().map(|(c,)| c).collect();
let fact_ids: Vec<Uuid> = facts.into_iter().map(|(f,)| f).collect();
reattest_tx(&mut tx, &cited).await?;
crate::temporal::tidy_timelines_tx(&mut tx, kb_id, &timelines).await?;
let deletion_id = Uuid::now_v7();
sqlx::query(
"INSERT INTO document_deletions
Expand All @@ -872,6 +879,77 @@ pub async fn delete(
})
}

/// 证据文档变了(删了一篇、撤销了删除),把这些事实的「最早证据日期」按还在的文档重算。
///
/// 读路径拿它当没起点的事实从哪天起成立(`facts_holds_from`),引擎排时间线用的是同一个
/// 日期(`temporal::DATED_AT`)。删掉最早那份文档之后引擎的锚点挪到了第二份,这里不跟着
/// 挪的话,前任读到新锚点为止、它却还从旧日期读起,两段叠了一年(#679 第四轮评审)。
/// 一份带日期的文档都不剩的,保留原值——那是别的来源(人写的)给的日期
async fn reattest_tx(tx: &mut Transaction<'_, Postgres>, fact_ids: &[Uuid]) -> AppResult<()> {
if fact_ids.is_empty() {
return Ok(());
}
sqlx::query(
"UPDATE facts f SET attested_from = e.first
FROM (SELECT fe.fact_id, min(d.doc_time) AS first
FROM fact_evidence fe
JOIN chunks c ON c.id = fe.chunk_id
JOIN documents d ON d.id = c.document_id
WHERE fe.fact_id = ANY($1) AND d.deleted_at IS NULL AND d.doc_time IS NOT NULL
GROUP BY fe.fact_id) e
WHERE f.id = e.fact_id AND f.invalidated_at IS NULL
AND f.attested_from IS DISTINCT FROM e.first",
)
.bind(fact_ids)
.execute(&mut **tx)
.await?;
Ok(())
}

/// 引用这篇文档的现存事实,连同它们所在的唯一性时间线,锁上之后返回。
///
/// **锁上之后再读一遍**(#679 第三轮评审):等锁的时候,时间线重算可能把其中一行改写成了
/// 新的一行,头一遍读到的是旧 id——拿旧名单去作废,新的那一行就活下来,出处却已经删了。
/// 改写只在时间线的锁里发生,锁上之后名单不会再变;再读出来的行若落在没锁上的时间线上
/// (撤回合并把它送回了源实体),把那几条也锁上
///
/// `also`:一并要锁的别的事实(撤销删除时要复活的那些)。**一次锁齐**:先锁引用的、
/// 再锁复活的,分两轮拿锁的话,另一个按顺序拿同样两把锁的事务会和它互相等死
async fn lock_cited_timelines(
tx: &mut Transaction<'_, Postgres>,
kb_id: Uuid,
document_id: Uuid,
also: &[Uuid],
) -> AppResult<(Vec<Uuid>, Vec<crate::temporal::Timeline>)> {
let cited_sql = "SELECT DISTINCT f.id FROM facts f
JOIN fact_evidence fe ON fe.fact_id = f.id
JOIN chunks c ON c.id = fe.chunk_id
WHERE f.kb_id = $1 AND f.invalidated_at IS NULL AND c.document_id = $2";
let cited: Vec<Uuid> = sqlx::query_scalar(cited_sql)
.bind(kb_id)
.bind(document_id)
.fetch_all(&mut **tx)
.await?;
let first: Vec<Uuid> = cited.iter().chain(also).copied().collect();
let mut timelines = crate::temporal::timelines_of(&mut **tx, kb_id, &first, None).await?;
crate::temporal::lock_timelines(tx, kb_id, &timelines).await?;
let cited: Vec<Uuid> = sqlx::query_scalar(cited_sql)
.bind(kb_id)
.bind(document_id)
.fetch_all(&mut **tx)
.await?;
let late: Vec<_> = crate::temporal::timelines_of(&mut **tx, kb_id, &cited, None)
.await?
.into_iter()
.filter(|t| !timelines.contains(t))
.collect();
if !late.is_empty() {
crate::temporal::lock_timelines(tx, kb_id, &late).await?;
timelines.extend(late);
}
Ok((cited, timelines))
}

/// 撤销一次删除:文档、这次打标的分块、这次作废的事实原路复活,形状照 `revert_merge`。
///
/// 只救 `document_deletions` 名单上的——更早版本的旧分块、删除之前就作废的事实
Expand Down Expand Up @@ -931,13 +1009,19 @@ async fn restore_tx(
.bind(&chunk_ids)
.execute(&mut **tx)
.await?;
// 复活的事实回到各自的时间线上;一直引用着这篇文档的事实,排序用的日期也回来了。
// 先锁、再复活、再重算(同删除)
let (cited, timelines) = lock_cited_timelines(tx, kb_id, id, &fact_ids).await?;
sqlx::query(
"UPDATE facts SET invalidated_at = NULL
WHERE id = ANY($1) AND invalidated_at IS NOT NULL",
)
.bind(&fact_ids)
.execute(&mut **tx)
.await?;
let touched: Vec<Uuid> = cited.iter().chain(&fact_ids).copied().collect();
reattest_tx(tx, &touched).await?;
crate::temporal::tidy_timelines_tx(tx, kb_id, &timelines).await?;
sqlx::query("UPDATE document_deletions SET reverted_at = now() WHERE id = $1")
.bind(deletion_id)
.execute(&mut **tx)
Expand Down
Loading
Loading