diff --git a/Cargo.lock b/Cargo.lock index bf6859523..66de830c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5414,7 +5414,6 @@ dependencies = [ "serde", "serde_json", "tracing", - "unicode-segmentation", "utopia-llm", ] diff --git a/Cargo.toml b/Cargo.toml index b0e967936..bb197b5d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,6 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] } # 分块预算按 token 数：cl100k 的排名表随 crate 内嵌，离线可用 tiktoken-rs = "0.12" # 句子边界按 Unicode 标准（UAX #29）切，不自己写标点规则 -unicode-segmentation = "1.13" pdf-extract = "0.12" calamine = "0.36" pgvector = { version = "0.4", features = ["sqlx"] } diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index aa8fa8f99..d90fa4a83 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -1077,10 +1077,6 @@ pub struct KnowledgeBase { /// 这次打开治理的时刻；保险丝只数它之后的撤回（0025 决定 9）。 /// 新库生下来就开着治理，这一格于是等于建库的时刻（0050） pub governance_since: Option>, - /// 开放抽取（0044 第一刀，#729）：开着，抽取只写开放图谱——陈述照文档的字落库 - /// （`facts.layer = 'open'`），不读本体、不问日期。**缺省开**（0044 决定 2）；关掉走 - /// 带本体的老路，它按决定 3 只是已批准本体下的可选第二路，等对齐追平就退场 - pub open_extraction: bool, /// 多久重推一次（分钟）。见 `knowledge_bases.inference_interval_minutes` pub inference_interval_minutes: i32, /// 上次推完的时间。**答的是「上次看过没有」，不是「上次改过没有」** diff --git a/crates/utopia-extract/Cargo.toml b/crates/utopia-extract/Cargo.toml index 5a0840ce0..ee407ae06 100644 --- a/crates/utopia-extract/Cargo.toml +++ b/crates/utopia-extract/Cargo.toml @@ -11,4 +11,3 @@ anyhow.workspace = true chrono.workspace = true utopia-llm.workspace = true tracing.workspace = true -unicode-segmentation.workspace = true diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index ad4415735..58fe15855 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -1,5 +1,9 @@ -//! utopia-extract: LLM 抽取（实体/关系/时间归一化）。 -//! 提示词注入本体类型与文档元时间；输出严格 JSON；证据引句强制（无引句降置信度）。 +//! utopia-extract：抽取用的提示词与解析。 +//! +//! 开放抽取（[`open`]）是唯一的抽取契约（0044 决定 2）：模型用文档自己的话写陈述， +//! 不读本体、不算日期。这里留的是它借用的零件（已知实体的句柄、文件开头的预算、 +//! JSON 取块与补括号）、实体消解的攒批裁决，以及量与时间的读法——本体属性的采纳、 +//! 工具参数、连接器都用它们。 use chrono::{DateTime, NaiveDate, TimeZone, Utc}; use serde::Deserialize; @@ -8,122 +12,6 @@ use utopia_llm::ChatMessage; pub mod governor; pub mod open; -pub mod normalize; -pub use normalize::{drop_quotes_from_opening, normalize_facts, Normalization}; - -#[derive(Debug, Deserialize)] -pub struct Extraction { - #[serde(default)] - pub entities: Vec, - #[serde(default)] - pub facts: Vec, - /// 实体在这段文字里的**别的名字**（0041 决定 2）：简称、曾用名、另一种文字的写法。 - /// 模型报，服务端只核对名字与引文确实在原文里——认不认「简称」「又名」这些词是模型的事 - #[serde(default)] - pub names: Vec, - /// 逐项解析时被跳过的条目数。**必须报给调用方**——不报就是一次静默丢弃， - /// 与 #108「部分抽取报告成完成」同一类错 - #[serde(skip)] - pub skipped_entities: usize, - #[serde(skip)] - pub skipped_facts: usize, - /// 模型的输出被截断，这里是修补后解析的 - #[serde(skip)] - pub truncated: bool, -} - -#[derive(Debug, Deserialize)] -pub struct ExtractedEntity { - /// Identifier scoped to this extraction response. It preserves mention identity while - /// facts are bound; it is not a persistent entity id or a resolution verdict. - #[serde(default)] - pub local_id: Option, - pub name: String, - #[serde(rename = "type")] - pub type_key: String, - /// 模型自己的说法：它认为这最具体是个什么。**不校验、不入本体**。 - /// - /// 存在的理由是清单里总有个"差不多"的：本体有 product，模型觉得够用就选了， - /// 心里那个"向量数据库软件"就此丢失。实测 17 个实体的 proposed_type - /// 全是空的，正是这个原因——而事后消解最需要的恰是这个名字： - /// 短名字对短标签，比拿一段中文散文去匹配 "A software application." 近得多。 - #[serde(default)] - pub specific_type: Option, -} - -/// 一个实体的一个别的名字。`ref` 是这次回复里的 local_id，或者提示词给的 k 句柄 -#[derive(Debug, Deserialize)] -pub struct ExtractedName { - #[serde(rename = "ref")] - pub entity_ref: String, - pub name: String, - /// 名字出现在里面的那段原文，逐字抄 - #[serde(default)] - pub quote: Option, -} - -#[derive(Debug, Deserialize)] -pub struct ExtractedFact { - pub subject: String, - /// Response-local entity handle. When present, callers must bind through it rather than - /// guessing from the surface name. - #[serde(default)] - pub subject_ref: Option, - pub predicate: String, - /// 关系事实的宾语实体名；属性事实为空 - #[serde(default)] - pub object: Option, - #[serde(default)] - pub object_ref: Option, - /// 属性事实的字面值（谓词是 attribute 时） - #[serde(default)] - pub value: Option, - /// **边上的属性**（0037）：`{"amount": "$5 billion", "stake": "20%"}`。 - /// 只对关系事实有意义，key 必须是清单里这条关系声明过的；值照原文写，换算在服务端 - #[serde(default)] - pub qualifiers: Option>, - #[serde(default)] - pub valid_from: Option, - #[serde(default)] - pub valid_to: Option, - #[serde(default)] - pub confidence: Option, - #[serde(default)] - pub quote: Option, - /// 引文里逐字点名主语的那几个字（#582）。模型抄，不判断；落库时机器核对它 - /// 是不是 `subject` 那个名字——"Former OpenAI personnel" 不是 OpenAI - #[serde(default)] - pub subject_span: Option, - /// 同上，宾语那一侧 - #[serde(default)] - pub object_span: Option, - /// 日期属性的值只相对一件事给出（「触发日后 45 天」），没有日历上的日期（#681 §4）。 - /// 模型判断、模型标；服务端不认这类说法的词，只看这个标记决定收不收 - #[serde(default)] - pub relative: bool, -} - -/// 提示词里的一条关系。 -/// -/// 比类多一样东西：**类型签名**。它是给模型的签名，不是闸门——在模型落笔那一刻 -/// 减少"Alice works_at 西雅图"，而不是等错了再拦。事后校验面对的是既成事实 -///（丢掉可惜、留着是脏数据），签名是在写出来之前掰正。本体写错时模型看到原文 -/// 说了别的仍可覆盖；硬闸门会系统性丢数据，`part_of` 烧我们的正是那种方式。 -pub struct PromptRelation { - pub key: String, - pub label: String, - pub description: String, - /// 形如 `person|organization → vendor`，`*` 表示那一侧不限。空串 = 两侧都不限。 - /// **一律用 key**：模型要输出的就是 key，中文库里 person 的 label 是"人物"， - /// 写进签名等于教它输出一个不存在的类型（docs/decisions/0004） - pub signature: String, - /// 时间语义（`relation_types.temporal`）：`state` / `event` / `eternal`（0031）。 - /// 只有 event 与 eternal 会在清单里带标记——状态是默认，写出来只多花 token - pub temporal: String, - /// 这条关系的边能带的属性，已排好版：`amount: number $`（0037）。空 = 不带 - pub qualifiers: Vec, -} - /// Response-scoped reference to a persistent entity; database UUIDs must never enter prompts. pub struct KnownEntity { pub handle: String, @@ -131,322 +19,9 @@ pub struct KnownEntity { pub name: String, } -/// 构造抽取提示词。`types` 为 (key, label, description) 三元组； -/// description 非空时按行列出——本体里的语义指引直接决定抽取质量。 -/// `attributes` 为调用方预排版的属性行（"person.salary (number, CNY): 月薪"）； -/// 为空时提示词一字不变——没定义属性的库零成本。 -pub fn build_messages( - types: &[(String, String, String)], - relations: &[PromptRelation], - attributes: &[String], - doc_time: Option<&str>, - filename: &str, - // 本文档前面几块已经认下的实体，按首次出现排序。handle 只对这次回复有效。 - // 第一块为空——那时还没有"前面" - known: &[KnownEntity], - chunk_text: &str, -) -> Vec { - build_messages_with_opening( - types, relations, attributes, doc_time, filename, known, None, chunk_text, - ) -} - /// 文件开头进提示词的字符预算。一份补充协议的标题、生效日、当事方和「修订的是哪份 /// 协议」通常在头一千字符里；新闻稿的电头与导语也是 -pub const OPENING_BUDGET_CHARS: usize = 1500; - -/// 同 [`build_messages`]，另带**这份文件的开头**（第一块的原文），给第二块往后用。 -/// -/// **一块是孤立抽取的，它看不见自己属于什么。** 补充协议把截止日写在第三块的表格里， -/// 那一块只说「Article 13 的日期延至……」：改的是哪份租约、从哪天起改，都写在第一块。 -/// 模型拿不到，就只能把「Phase 2 Exercise Deadline」本身当主语（服务端按主语未声明丢掉）， -/// 或者抽出一个没有起点的日期（时态引擎没法据此关闭旧值）——Blackbaud 总部租约链 -/// 上五次改期丢了两次，抽到的三次一次都没关上旧值。开头只作背景，不从里面抽事实： -/// 它自己那一块会抽，重复抽只会多出重复的事实 -#[allow(clippy::too_many_arguments)] -pub fn build_messages_with_opening( - types: &[(String, String, String)], - relations: &[PromptRelation], - attributes: &[String], - doc_time: Option<&str>, - filename: &str, - known: &[KnownEntity], - opening: Option<&str>, - chunk_text: &str, -) -> Vec { - // **有描述时不送 label**。label 是给人看的显示名，而且它跟界面无关、 - // 跟这个库的语料语言走——中文库里 person 的 label 是"人物"。 - // `- person (人物): 有名有姓的具体的人…` 里那个"人物"相对 key 近乎零信息量， - // 却让提示词在语料语言与标识符之间来回跳。描述为空时才拿它兜底： - // 光一个 key 太单薄。见 docs/decisions/0004 - let fmt_list = |items: &[(String, String, String)]| { - items - .iter() - .map(|(k, l, d)| { - let d = d.trim(); - if d.is_empty() { - format!("- {k} ({l})") - } else { - format!("- {k}: {d}") - } - }) - .collect::>() - .join("\n") - }; - let type_list = fmt_list(types); - // 关系行：有签名时括号里放签名，没有才退回 label。 - // `- works_at (person → organization): 一个人受雇于某个组织。` - let rel_list = relations - .iter() - .map(|r| { - let d = r.description.trim(); - let paren = if !r.signature.is_empty() { - r.signature.clone() - } else if d.is_empty() { - r.label.clone() - } else { - String::new() - }; - // 事件与恒常带方括号标记；状态是默认，不标（0031） - let mark = temporal_mark(&r.temporal) - .map(|m| format!(" [{m}]")) - .unwrap_or_default(); - // 边上能带的属性跟在标记后面：`{amount: number $, stake: number %}` - let mark = if r.qualifiers.is_empty() { - mark - } else { - format!("{mark} {{{}}}", r.qualifiers.join(", ")) - }; - match (paren.is_empty(), d.is_empty()) { - (false, false) => format!("- {} ({paren}){mark}: {d}", r.key), - (false, true) => format!("- {} ({paren}){mark}", r.key), - (true, false) => format!("- {}{mark}: {d}", r.key), - (true, true) => format!("- {}{mark}", r.key), - } - }) - .collect::>() - .join("\n"); - // 标记也只在真有事件或恒常关系时解释一次；全是状态的库，提示词一字不变。 - // 说的是**写什么**而不是「它是什么」：事件的那一刻进 valid_from、valid_to 留空 - // ——不然模型照状态的样子填一个起点，账本就把一次收购读成从那天起一直持续 - let temporal_note = if relations - .iter() - .any(|r| temporal_mark(&r.temporal).is_some()) - { - "\n 3b. A relation marked [event] happens at one moment: put the date it happened in \ - valid_from and leave valid_to null — it has no span and does not end. A relation \ - marked [eternal] holds regardless of time: leave both dates null." - } else { - "" - }; - // 记号只在真有签名时解释一次；没有签名的库，提示词一字不变。 - // 说明用英文——提示词的**指令语言**是英文，只有 description 跟语料走 - // - // **签名管两件事，而它们的可覆盖性不同。** 第一版把两件事混成了一句 - // "It is a hint, not a rule — when the text says otherwise, write what the - // text says"，于是模型连参数顺序也一并按原文的说法写： - // - // Elon Musk (person) --employee--> Microsoft - // - // 而 schema.org 声明的是 employee (organization → person)。实测一次跑里 - // 130 条可校验的事实有 102 条这样反着落库——**恰恰是本体包最主要的卖点失效**， - // 选 schema.org 的理由就是「方向是声明的不是描述的」。 - // - // 两件事分开说： - // - // - **哪些类型能参与**：提示不是闸门。本体可能写错，原文说西雅图就写西雅图。 - // 0001 的判断在这里不变——硬闸门会系统性丢数据，part_of 烧我们的正是那样。 - // - **参数顺序**：由签名定。顺序不是关于世界的断言，是这个 key 的编码约定； - // 原文从来没有「说了别的方向」，它只说两个实体之间存在某种关系。 - // 反着说时该交换主宾，而不是反过来用这个关系。 - let sig_note = if relations.iter().any(|r| !r.signature.is_empty()) { - ". A parenthesis after the key is the type signature, subject then object; \ - \"|\" means or, \"*\" means unconstrained. Which kinds of things may take part \ - is a hint, not a rule — when the text says otherwise, write what the text says. \ - The order is not a hint: the signature fixes which side is the subject. If the \ - text puts them the other way round, swap subject and object so that the subject \ - matches the left side — do not reverse the relation. For example, given \ - \"employee (organization → person)\" and a text saying \"X is an employee of Y\", \ - write Y as the subject and X as the object" - } else { - "" - }; - let time_ctx = doc_time - .map(|t| { - format!( - "Document date: {t}. Resolve relative time expressions (e.g. \"last year\", \ - \"this March\") to absolute dates using it as the reference." - ) - }) - .unwrap_or_else(|| { - "Document date unknown — only output dates explicitly written in the text.".into() - }); - - // 属性段按需注入：清单 + 输出说明 + 取值规则。没定义属性时完全不出现 - let attr_section = if attributes.is_empty() { - String::new() - } else { - format!( - "\nAttributes (literal-valued fields, listed as class.attribute_key; as \"predicate\" \ - use the attribute_key alone — e.g. \"salary\", not \"person.salary\" — with a \ - \"value\" instead of \"object\"):\n{}\n", - attributes.join("\n") - ) - }; - let attr_rules = if attributes.is_empty() { - String::new() - } else { - "\n11. Attribute facts carry \"value\" (no \"object\"): number = the figure **as the text writes it, magnitude and currency included** \n (\"86亿元\", \"$5 billion\", \"4,300 人\") — never reduce it to a bare number, the server converts; date = \"YYYY[-MM[-DD]]\" (a zoned clock time only when the text gives one) — a date the text gives only relative to an event \ - (\"45 days after the Trigger Date\", \"within 30 days of closing\") has no calendar date to convert: write it as the text writes it and add \"relative\": true; bool = true/false; \ - text = a short string. Only attach an attribute to a subject of its listed class. \ - valid_from = when this value took effect, if the text or the opening of the document says so. \ - A document that changes a value set earlier — amends, extends or replaces it — makes the new value \ - hold from the date the change takes effect, which is the document's own effective date unless the text gives another." - .to_string() - }; - let system = format!( - "You are a knowledge-graph extraction engine. Extract entities and factual relations \ - from the given text. Output exactly one JSON object and nothing else.\n\ - \n\ - Entity types (prefer these keys):\n{type_list}\n\ - \n\ - Relation types (prefer these keys){sig_note}:\n{rel_list}\n\ - {attr_section}\ - \n\ - Output format:\n\ - {{\"entities\":[{{\"local_id\":\"e1\",\"name\":\"entity name\",\"type\":\"type key\",\"specific_type\":\"what you would call it\"}}],\n\ - \"facts\":[{{\"subject\":\"subject entity name\",\"subject_ref\":\"e1\",\"subject_span\":\"the words in quote that name the subject\",\"predicate\":\"relation key\",\"object\":\"object entity name\",\"object_ref\":\"e2\",\"object_span\":\"the words in quote that name the object\",\n\ - \"valid_from\":\"2023-01\",\"valid_to\":null,\"confidence\":0.9,\"quote\":\"verbatim supporting quote\"}}],\n\ - \"names\":[{{\"ref\":\"e1\",\"name\":\"another name the text uses for it\",\"quote\":\"verbatim text containing that name\"}}]}}\n\ - \n\ - Rules:\n\ - 1. Give every newly listed entity a local_id unique within this response (e1, e2, ...). \ - A local_id may define at most one entity. Reuse the same local_id when this response \ - mentions the same entity again, including abbreviations; never allocate a new handle \ - merely because a mention repeats. Different handles mean the mentions should be \ - tracked separately for attribution, not that their permanent identity is proven. \ - When the text clearly describes two different entities with the same surface name, \ - list both under different local_ids. A shared name alone neither proves sameness nor \ - requires a split.\n\ - 1a. Use the canonical full name as written in the text, in the text's original language; \ - list each entity once. Text introduces a full name and then shortens it — \ - \"星云科技上海研究院\" becomes \"上海研究院\", \"Nebula Technologies Inc.\" becomes \ - \"Nebula\" — and both forms mean one entity, listed once under the fuller form. \ - Two names are two entities only when the text is talking about two things. \ - A name identifies the thing; it is not a description of its history. When the text \ - names something and then describes what happened to it, the name ends where the \ - description begins.\n\ - 1b. Every other name the text gives an entity goes into \"names\", once per name: the \ - shortened form it introduces or uses (\"上海研究院\" for \"星云科技上海研究院\"), a \ - former name, the name in another language. \"ref\" is the entity's local_id or its \ - known handle, and \"quote\" is a verbatim excerpt that contains the name. Only \ - names belong there — never a pronoun or a description (\"该公司\", \"the company\", \ - \"former employees\") — and never the name already written in entities. A name \ - must name the entity itself, not something that belongs to it: \"星云科技研发团队\" \ - names a team, not 星云科技.\n\ - 2. Every fact keeps its name fields and uses subject_ref; relation facts also use \ - object_ref. Each ref must be either a local_id defined exactly once in \ - entities or a known handle supplied with this text. An entity referenced by a known \ - handle must not be copied into entities.\n\ - 3. Dates must be \"YYYY\", \"YYYY-MM\", \"YYYY-MM-DD\", or null — never invent dates. \ - A clock time is allowed only together with its zone, as \"YYYY-MM-DDTHH:MM[:SS]Z\" \ - or with a \"+HH:MM\" offset, and only when the text or the document states that \ - zone; a time of day without a zone stays a plain date — never guess a zone.\n\ - 3a. valid_to takes a third value: \"unknown\". Use it when the text says the relation \ - has ended but does not say when — \"former CEO of X\", \"stepped down\", \"left the \ - company\", \"no longer available\", \"until recently\". Use null only for something \ - still going on. These are not interchangeable: null asserts it still holds, and \ - writing null for a relation the text says is over makes us claim the opposite of \ - the source.\n\ - 3c. A period is when a fact holds, never what it is about. A quarter, a half, a \ - fiscal or calendar year, a month, \"the three months ended July 26, 2026\" — \ - none of these is an entity and none is an object. Put the period's dates in \ - valid_from and valid_to (a fiscal period resolves to the dates the document \ - states for it) and write the figure as the fact's \"value\" — the figure alone, as it stands in the \ - quote, with nothing appended. A column of a table headed by a period is a column \ - of values that hold in that period.\n\ - {temporal_note}\n\ - 4. {time_ctx}\n\ - 5. quote must be a contiguous excerpt from the Text block; never quote the opening of the document. Every fact needs one.\n\ - 6. confidence in 0~1: 0.9 explicitly stated, 0.7 inferred, 0.5 uncertain. A value \ - the text writes out is stated whatever the layout — a sentence, a list, a table \ - cell, a schedule, the new column of an amendment that replaces an earlier term. \ - Inferred means the text does not write the value and you worked it out.\n\ - 7. If nothing can be extracted, output {{\"entities\":[],\"facts\":[]}}.\n\ - 8. If no listed relation fits, do not force the nearest one — write the predicate the \ - text itself uses, in snake_case (e.g. \"available_on\", \"runs_on\"). A relation \ - named after the text is worth more than a listed one that says something false.\n\ - 8a. The same holds for a literal the text states outright — an amount, a share count, \ - a percentage, a capacity, a date, a job title, a ticker. Write it as a fact with \ - \"value\" and no \"object\": {{\"subject\":\"NVIDIA\",\"subject_ref\":\"e1\",\ - \"predicate\":\"purchase_price\",\"value\":\"$11.9 billion\",\"confidence\":0.9,\ - \"quote\":\"...\"}}. Name the predicate after the text when no listed attribute \ - fits — \"purchase_price\", \"job_title\", \"generation_capacity\", \"record_date\". \ - Attach it to the entity the text attaches it to, and keep the literal as written, \ - units and all — except a date, which is always written in the format of rule 3 \ - (\"June 23, 2020\" is \"2020-06-23\"). A deadline or a period stated \ - relative to an event, with no calendar date, is not a date: keep it as written \ - and mark it \"relative\" as rule 11 says. \ - **A stated figure left out is the loss that costs most**: the reader \ - came for those numbers, and no later step can recover one that was never written \ - down.\n\ - 8b. A listed relation followed by {{…}} can carry those **qualifiers on the edge**: when the same sentence gives both the other entity and a figure for it — an amount, a stake, a price, a share count — write the relation with its \"object\" and put the figure in \"qualifiers\" keyed exactly as listed, **as written in the text, currency and all** (\"€30 million\", \"15亿元人民币\", never a bare number) — except a date, which takes the format of rule 3: {{\"subject\":\"Vega Capital\",\"predicate\":\"invested_in\",\"object\":\"Northwind\", \"qualifiers\":{{\"amount\":\"$5 billion\"}},…}}. Never invent a key that is not listed for that relation, and never drop the figure to keep the edge — a relation without its amount is half the sentence. A relation you name after the text (rule 8) carries its figure the same way — keyed by the listed attribute that fits it, or by the plainest word for it (\"amount\", \"stake\", \"price\") when none does. - 8c. A **listed** relation also takes \"value\" when what the text gives is a \ - string rather than another entity — a job title, a designation, a ticker, a \ - model number. Never invent an entity for a string. And when the text introduces \ - someone by their role — \"X, founder and CEO of Y\", \"Z, co-CEO of W\", \ - \"Y's vice president of research\", \"the president of OpenAI\", \ - \"chief executive of Quora\", \"OpenAI's chief technology officer of \ - applications\" — write both facts: the tie to the organization, and \ - the role itself as a value on the person. The tie alone says they \ - are connected; the role is what the sentence was actually telling \ - you. The possessive (\"Y's \", \" of Y\", \" at Y\"), the past \ - tense (\"was Y's \", \"former of Y\"), and the implied form \ - (\"appointed … as OpenAI's CTO of applications\") all carry the same \ - shape — the role is the value, the organization is the other \ - entity. Past tense and \"former\" give the tie valid_to: \"unknown\".\n\ - 8d. A list of named parties is a list of facts — one per name. \"partners \ - including A, B, C and D\" is four facts, not one; \"advisors A and B\" is two. \ - Do not collapse an enumeration into a summary or into its first member. \ - The same applies to the entities: each named party is its own entity.\n\ - 8e. subject_span and object_span are the exact words in quote that name each side. \ - Copy them; never paraphrase. When the words that do the thing are a description \ - rather than a name — \"former X employees\", \"companies using X\" — the span \ - is that description, whatever you wrote in subject.\n\ - 8f. An obligation, a deadline or a right belongs to the agreement, law or decision \ - that imposes it, even when it concerns another agreement or thing. A lease that \ - sets the last day to sign a second lease gives that deadline to the first lease; \ - the second lease is only what the deadline is about.\n\ - 9. The same holds for entity types: if none of the listed types fits, write the type \ - the text implies, in snake_case (e.g. \"model\", \"technology\"). Do not fall back \ - to a broad listed type such as \"thing\" or \"creative_work\" merely because \ - nothing specific matched — that hides the gap instead of reporting it.\n\ - 10. specific_type is required on every entity and is never checked against the list. \ - Name the most specific kind the thing is, in the words you would use for it. Write \ - it even when \"type\" already fits, and make it narrower than \"type\" wherever the \ - text supports it — type \"product\", specific_type \"vector database software\". \ - Repeat the listed type only when the text genuinely says nothing more precise.\ - {attr_rules}" - ); - - // 已知实体紧挨着正文：服从性靠位置，理由见 known_block 的注释 - let user = format!( - "Source file: \"{filename}\"\n{}{}\nText:\n{chunk_text}", - opening_block(opening), - known_block(known) - ); - - vec![ - ChatMessage { - role: "system".into(), - content: system, - }, - ChatMessage { - role: "user".into(), - content: user, - }, - ] -} +pub(crate) const OPENING_BUDGET_CHARS: usize = 1500; /// 文件开头排版成提示词里的一段。开头为空（或只有空白）时返回空串； /// 「这一块就是开头本身」由调用方判断，那时它传 `None`。 @@ -471,75 +46,12 @@ pub(crate) fn opening_block(opening: Option<&str>) -> String { ) } -/// 一段描述的第一句（句子边界按 UAX #29）。按块检索出的清单只带这一句：schema.org 的 -/// 描述后半截多是用法说明与示例，一块铺上百行时它们占了清单的八成 -pub fn first_sentence(text: &str) -> &str { - use unicode_segmentation::UnicodeSegmentation; - text.trim().unicode_sentences().next().map_or("", str::trim) -} - -/// 清单里给关系带的标记：事件 `[event]`、恒常 `[eternal]`；状态不标。 -/// 认不出的值当状态——数据库的 CHECK 只放这三个进来，这里不再报错 -fn temporal_mark(temporal: &str) -> Option<&'static str> { - match temporal { - "event" => Some("event"), - "eternal" => Some("eternal"), - _ => None, - } -} - /// 已在本文档中出现过的实体，放进提示词的字符预算。 /// /// 超出就截断（保留先出现的）。中文商业文本先出全称、主角先出场，所以 /// **首次出现顺序天然偏向那些后面会被简称的名字**。 pub(crate) const KNOWN_BUDGET_CHARS: usize = 1200; -/// 把「本文档已经认下的实体」排版成提示词里的一段。空则返回空串。 -/// -/// **为什么在正文之前、指令贴着清单**：抽象规则打不过挨着它的具体块——本体建议 -/// 那次，语言要求就输给了紧随其后的英文 JSON 骨架，挪到骨架之后并点名它才生效。 -/// 服从性靠位置，所以指令挨着它管的数据放，两者一起挨着正文。 -/// -/// **顺带一条与放哪条消息无关的规矩：逐块变化的内容一律放最后。** 前缀缓存匹配的是 -/// token 前缀，而消息按 system→user 拼接，所以「system 末尾」与「user 开头」几乎等价； -/// 真正会打碎缓存的是把它塞在**中间**（本体之后、规则之前），那会把规则挤出前缀。 -/// 缓存本身不归我们管——供应商开不开、报不报都由它，本部署实测 `cached=0`—— -/// 我们只负责别把它弄碎。自部署 vLLM 默认开着自动前缀缓存，那省的是算力不是钱。 -fn known_block(known: &[KnownEntity]) -> String { - if known.is_empty() { - return String::new(); - } - let mut lines = Vec::new(); - let mut used = 0usize; - for entity in known { - used += entity.handle.chars().count() - + entity.type_key.chars().count() - + entity.name.chars().count() - + 6; - if used > KNOWN_BUDGET_CHARS { - break; - } - lines.push(format!( - " {} [{}]: {}", - entity.handle, entity.type_key, entity.name - )); - } - if lines.is_empty() { - return String::new(); - } - let lines = lines.join("\n"); - format!( - "\nAlready recorded from earlier parts of this same document:\n{lines}\n\ - \n\ - If something in the text below refers to one of these, use its k-handle in the fact's \ - subject_ref/object_ref, write that exact string as the name, and give it that same \ - type — documents abbreviate after first mention \ - (\"星云科技上海研究院\" later becomes \"上海研究院\"), and the shortened form must \ - not become a second entity. If it is a different thing, name it as the text does; \ - do not force it onto this list.\n" - ) -} - /// 回复里可能是 JSON 的那段文字：切掉思考过程与代码围栏。前后的废话留给调用方按 /// 括号定位——[`json_block`] 取第一个 `{` 到最后一个 `}`；开放抽取的截断修补则从 /// 第一个 `{` 取到结尾，那边的记录是数组，最后一个 `}` 不是可靠的结尾 @@ -611,102 +123,6 @@ pub(crate) fn close_brackets(head: &str) -> Option { Some(out) } -/// 输出被截断时，退到**最后一个完整对象**的结尾再把括号补齐。 -/// -/// 模型写到一半没了（撞上 max_tokens）时，前面那些对象是完整且正确的。 -/// 整块作废等于把已经抽对的十几条事实一起扔掉——实测 246 次调用里 4 次是这种。 -fn repair_truncated(json: &str) -> Option { - let mut cut = json.len(); - for _ in 0..64 { - let idx = json[..cut].rfind('}')?; - if let Some(closed) = close_brackets(&json[..=idx]) { - if serde_json::from_str::(&closed).is_ok() { - return Some(closed); - } - } - cut = idx; - } - None -} - -/// **一条坏记录不该毁掉一整块。** -/// -/// 从前这里是 `serde_json::from_str::`——全有或全无。一个缺 `predicate` -/// 的对象、或者一次输出截断，整块的实体和事实一起作废，而一块里常有二十条好事实。 -/// 实测 246 次调用里 5 次这样丢掉（2%），并且会让整个 `extract_document` 任务失败、 -/// 走重试，三次之后文档标记失败。 -/// -/// 现在：先解成 `Value`（截断就先补齐括号），再逐项 `from_value`，好的收下、 -/// 坏的计数。**计数必须往外传**——静默跳过就是另一种"报告成完成"。 -pub fn parse_response(raw: &str) -> anyhow::Result { - let json_str = json_block(raw)?; - let (value, truncated) = match serde_json::from_str::(&json_str) { - Ok(v) => (v, false), - Err(e) => match repair_truncated(&json_str) { - Some(fixed) => ( - serde_json::from_str::(&fixed) - .map_err(|e| anyhow::anyhow!("Failed to parse extraction JSON: {e}"))?, - true, - ), - // 补不回来才是真解析失败：连一个完整对象都没有 - None => anyhow::bail!("Failed to parse extraction JSON: {e}"), - }, - }; - - fn take( - value: &serde_json::Value, - key: &str, - ) -> (Vec, usize) { - let Some(arr) = value.get(key).and_then(|v| v.as_array()) else { - return (Vec::new(), 0); - }; - let mut out = Vec::with_capacity(arr.len()); - let mut skipped = 0; - for item in arr { - match serde_json::from_value::(item.clone()) { - Ok(v) => out.push(v), - Err(_) => skipped += 1, - } - } - (out, skipped) - } - - let (mut entities, mut skipped_entities) = take::(&value, "entities"); - let (facts, skipped_facts) = take::(&value, "facts"); - // 名字条目坏了不算实体或事实被跳过：丢一个名字只是少一座桥，不丢断言 - let (names, _) = take::(&value, "names"); - - // A handle identifies exactly one entity definition within one response. Reject every - // definition participating in a duplicate (including identical duplicates): keeping the - // first or last would make fact attribution depend on array order. Empty handles are - // malformed too; legacy output is represented by an absent field, not an empty id. - let mut handle_counts = std::collections::HashMap::::new(); - for entity in &entities { - if let Some(handle) = entity.local_id.as_deref() { - *handle_counts.entry(handle.trim().to_string()).or_default() += 1; - } - } - entities.retain(|entity| match entity.local_id.as_deref() { - None => true, - Some(handle) => { - let handle = handle.trim(); - let valid = !handle.is_empty() && handle_counts.get(handle) == Some(&1); - if !valid { - skipped_entities += 1; - } - valid - } - }); - Ok(Extraction { - entities, - facts, - names, - skipped_entities, - skipped_facts, - truncated, - }) -} - // --------------------------------------------------------------------------- // 实体消解裁决（攒批：一次调用裁多对，LLM 只处理 embedding 分不出的灰区） // --------------------------------------------------------------------------- @@ -934,7 +350,7 @@ pub fn parse_leading_quantity(s: &str) -> Option<(f64, Option)> { } /// 货币：符号、ISO 码、中英文单词，统一成符号。**只认这张表**，认不出的不猜。 -pub fn currency_unit(tok: &str) -> Option<&'static str> { +pub(crate) fn currency_unit(tok: &str) -> Option<&'static str> { Some( match tok.trim_matches(|c: char| c == ',' || c == '.' || c == ';') { "$" | "USD" | "usd" | "US$" | "dollar" | "dollars" | "美元" => "$", @@ -1082,25 +498,6 @@ fn next_token(s: &str) -> (&str, &str) { (&s[..end], &s[end..]) } -/// 一个属性值落库时的样子：按 datatype 归一成 `{"value": …}`；失败返回 None，调用方记 -/// `attr_datatype`。 -/// -/// 日期属性上一个**相对**的值（#681 §4）：解不成日期、模型又标了 `relative`、写的是一段非空 -/// 文字时，照原文收下，值里带 `"relative": true`。它不是日期，从不当日期比较或排序。解得成 -/// 日期的照日期存，标错了也不当相对；没标的非日期值仍然不收 -pub fn attr_object_value( - datatype: &str, - raw: &serde_json::Value, - relative: bool, -) -> Option { - if let Some(value) = normalize_attr_value(datatype, raw) { - return Some(serde_json::json!({ "value": value })); - } - let written = raw.as_str().map(str::trim).filter(|s| !s.is_empty())?; - (datatype == "date" && relative) - .then(|| serde_json::json!({ "value": written, "relative": true })) -} - /// 属性值按 datatype 归一。失败返回 None——宁缺勿脏，调用方跳过并记日志。 /// number 容忍千分位/空格；date 收规则 3 的格式（YYYY[-MM[-DD]]、带时区的时刻，原样保留）， /// 也收写法说得清是哪天的日期（[`written_date`]），换成规则 3 的样子；bool 宽容 yes/no。 @@ -1321,271 +718,32 @@ fn split_zone(clock: &str) -> Option<(&str, chrono::Duration)> { } #[cfg(test)] -mod prompt_shape_tests { +mod tests { use super::*; - /// 第十份补充协议把截止日改成「触发日后 45 天」：模型标 relative，服务端照原文收； - /// 没标的、不是日期属性的、空的都不走这条 + /// #690：思考过程里的大括号不能把 JSON 的起止带偏。不切掉标记，"第一个 `{`" 的起点 + /// 会提前到思考过程里；代码围栏与前后的废话同样不算数 #[test] - fn a_relative_date_is_kept_as_written_only_when_marked() { - let raw = serde_json::json!("45 days after the Trigger Date"); - assert_eq!( - attr_object_value("date", &raw, true), - Some( - serde_json::json!({ "value": "45 days after the Trigger Date", "relative": true }) - ) - ); - assert_eq!(attr_object_value("date", &raw, false), None, "没标就不收"); - assert_eq!( - attr_object_value("date", &serde_json::json!(" "), true), - None - ); - // 解得成日期的照日期存，标了 relative 也不当相对 + fn json_block_ignores_a_think_block_and_a_fence_before_the_json() { + let raw = "先想想 {\"a\": 1，再回答。\n{\"entities\":[{\"name\":\"张三\"}]}"; assert_eq!( - attr_object_value("date", &serde_json::json!("2020-06-23"), true), - Some(serde_json::json!({ "value": "2020-06-23" })) + json_block(raw).unwrap(), + "{\"entities\":[{\"name\":\"张三\"}]}" ); - // 不是日期属性：只按它自己的 datatype 归一，relative 不起作用 - assert_eq!( - attr_object_value("bool", &serde_json::json!("45 days after"), true), - None - ); - assert_eq!( - attr_object_value("number", &serde_json::json!("1,250"), true), - Some(serde_json::json!({ "value": 1250.0 })) - ); - let fact: ExtractedFact = serde_json::from_value(serde_json::json!({ - "subject": "Lease", "predicate": "expansion_option_deadline", - "value": "45 days after the Trigger Date", "relative": true - })) - .unwrap(); - assert!(fact.relative); - let plain: ExtractedFact = serde_json::from_value(serde_json::json!({ - "subject": "Lease", "predicate": "expansion_option_deadline", "value": "2020-06-23" - })) - .unwrap(); - assert!(!plain.relative, "没写就不是"); - let msgs = build_messages( - &[], - &[], - &["lease.deadline (date)".into()], - None, - "a.txt", - &[], - "text", - ); - assert!(msgs[0].content.contains("add \"relative\": true")); - } - - fn rel(key: &str, description: &str, signature: &str) -> PromptRelation { - PromptRelation { - key: key.into(), - label: key.replace('_', " "), - description: description.into(), - signature: signature.into(), - temporal: "state".into(), - qualifiers: vec![], - } - } - - fn timed(key: &str, description: &str, temporal: &str) -> PromptRelation { - PromptRelation { - temporal: temporal.into(), - ..rel(key, description, "") - } - } - - /// 事件与恒常在清单里带标记，说明只出现一次（0031） - #[test] - fn an_event_and_an_eternal_relation_are_marked() { - let rels = vec![ - rel("works_at", "受雇于某个组织。", "person → organization"), - timed("acquired", "One company buys another.", "event"), - timed("capital_of", "", "eternal"), - ]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - let s = &msgs[0].content; - assert!(s.contains("- works_at (person → organization): 受雇于某个组织。")); - assert!(s.contains("- acquired [event]: One company buys another.")); - // 没有描述时括号里是 label，标记跟在括号后面 - assert!(s.contains("- capital_of (capital of) [eternal]")); - assert!(s.contains("A relation marked [event] happens at one moment")); - assert!(s.contains("leave valid_to null")); + let fenced = "好的，结果如下：\n```json\n{\"entities\":[]}\n```"; + assert_eq!(json_block(fenced).unwrap(), "{\"entities\":[]}"); + assert!(json_block("no json here").is_err()); } - /// **全是状态的库，提示词一字不变**：不标、不解释 + /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号；断在字符串中间的那一截不可用 #[test] - fn a_base_of_states_pays_nothing_for_the_marks() { - let rels = vec![rel("works_at", "d", "")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - let s = &msgs[0].content; - assert!(!s.contains("[event]")); - assert!(!s.contains("[eternal]")); - assert!(!s.contains("happens at one moment")); - } - - /// 签名进括号，而且**一律是 key**：中文库的 label 是"人物"， - /// 写进提示词等于教模型输出一个不存在的类型。 - #[test] - fn a_signature_takes_the_parenthesis_and_uses_keys() { - let rels = vec![rel("works_at", "受雇于某个组织。", "person → organization")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - assert!(msgs[0] - .content - .contains("- works_at (person → organization): 受雇于某个组织。")); - } - - /// 多值用 `|`，空的一侧用 `*` —— 都是 key 层面的记号，不是类型名 - #[test] - fn several_classes_join_with_a_pipe_and_an_empty_side_is_a_star() { - let rels = vec![rel("buys_from", "", "employee|team → *")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - assert!(msgs[0].content.contains("- buys_from (employee|team → *)")); - } - - /// **没有签名的库，提示词一字不变**：记号说明也不出现。 - /// 大多数库不会声明 domain/range，不该为此付每块的 token - #[test] - fn a_base_without_signatures_pays_nothing() { - let rels = vec![rel("works_at", "受雇于某个组织。", "")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - assert!(msgs[0].content.contains("- works_at: 受雇于某个组织。")); - assert!(!msgs[0].content.contains("type signature")); - assert!(!msgs[0].content.contains('→')); - } - - /// 签名是提示不是闸门。这句话必须在提示词里 —— 少了它， - /// 模型会把签名当硬规则，本体写错时就系统性丢数据（part_of 那种方式） - #[test] - fn the_prompt_says_the_signature_is_a_hint() { - let rels = vec![rel("works_at", "d", "person → organization")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - assert!(msgs[0].content.contains("hint, not a rule")); - } - - /// **但顺序不是提示。** - /// - /// 两句话必须同时在场，少哪一句都退回一种老毛病：少了「提示不是闸门」， - /// 本体写错时系统性丢数据（part_of 那种方式）；少了「顺序由签名定」， - /// 模型按英语直觉写 `Musk --employee--> Microsoft`，而 schema.org 声明的是 - /// `employee (organization → person)`——实测一次跑里 130 条可校验的事实 - /// 有 102 条这样反着落库。 - #[test] - fn the_prompt_says_the_order_is_not_a_hint() { - let rels = vec![rel("employee", "d", "organization → person")]; - let msgs = build_messages(&[], &rels, &[], None, "a.txt", &[], "text"); - let c = &msgs[0].content; - assert!(c.contains("hint, not a rule"), "类型那句丢了"); - assert!(c.contains("The order is not a hint"), "顺序那句丢了"); - assert!( - c.contains("swap subject and object"), - "只说了顺序重要，没说反着写时该怎么办" - ); - assert!( - c.contains("do not reverse the relation"), - "少了这句，模型可能去找一个反向关系而不是交换主宾" - ); - } - - #[test] - fn an_obligation_belongs_to_the_agreement_that_imposes_it() { - // 主租约里写着「签二期租约的截止日」，模型时而把截止日挂到二期租约上： - // 主租约的时间线上就少了这次改期（#681 §3） - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - assert!(msgs[0] - .content - .contains("belongs to the agreement, law or decision that imposes it")); - } - - #[test] - fn a_literal_keeps_its_units_but_a_date_takes_the_contract_format() { - // 8a 从前说「字面值按原文写」并把日期列在字面值里，而规则 3 与属性规则要求 - // YYYY-MM-DD：两条互相打架，模型写出「June 23, 2020」，服务端按格式不合整条丢掉。 - // Blackbaud 总部租约链上各轮累计丢了二十多次 - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - let system = &msgs[0].content; - assert!(system.contains("except a date, which is always written in the format of rule 3")); - } - - /// 补充协议把旧条款与新日期排成一张对照表，模型把表格里读到的新日期标 0.7（当成 - /// 推断），低于 0.75 的值不许接替前一个——截止日就一直停在旧值上。规则 6 说清楚： - /// 原文写着的值不论排成什么样都是明写 - #[test] - fn a_value_written_in_a_table_is_stated() { - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - let system = &msgs[0].content; - assert!(system.contains("stated whatever the layout")); - assert!(system.contains("a table cell")); - } - - /// 规则编号各不相同，「按规则 N」指得到唯一的一条。从前有两条 8c、两条 10， - /// 「as rule 10 says」说的是哪条要靠猜（#689 评审） - #[test] - fn every_rule_has_its_own_number_and_every_reference_lands() { - let rels = vec![PromptRelation { - key: "acquired".into(), - label: "acquired".into(), - description: String::new(), - signature: String::new(), - temporal: "event".into(), - qualifiers: vec![], - }]; - let attrs = vec!["lease.option_deadline (date)".to_string()]; - let msgs = build_messages(&[], &rels, &attrs, None, "a.txt", &[], "text"); - let system = &msgs[0].content; - let mut labels = Vec::new(); - for line in system.lines() { - let Some((label, _)) = line.trim_start().split_once(". ") else { - continue; - }; - let digits = label.trim_end_matches(|c: char| c.is_ascii_lowercase()); - if !digits.is_empty() - && digits.chars().all(|c| c.is_ascii_digit()) - && label.len() - digits.len() <= 1 - { - labels.push(label.to_string()); - } - } - let unique: std::collections::BTreeSet<_> = labels.iter().collect(); - assert_eq!(unique.len(), labels.len(), "规则编号重复：{labels:?}"); - for (i, _) in system.match_indices("rule ") { - let n: String = system[i + 5..] - .chars() - .take_while(|c| c.is_ascii_alphanumeric()) - .collect(); - assert!( - labels.contains(&n), - "「rule {n}」指不到任何一条：{labels:?}" - ); - } - } - - #[test] - fn a_later_chunk_reads_the_opening_of_its_document() { - let opening = "FIFTH AMENDMENT TO LEASE AGREEMENT entered into as of February 18, 2020"; - let msgs = build_messages_with_opening( - &[], - &[], - &[], - None, - "a.html", - &[], - Some(opening), - "The Existing Dates are extended to March 17, 2020.", - ); - let user = &msgs[1].content; - let at_opening = user.find(opening).expect("opening is in the user message"); - let at_text = user - .find("The Existing Dates") - .expect("text is in the user message"); - assert!( - at_opening < at_text, - "the opening comes before the text it frames" + fn brackets_inside_strings_are_not_structure() { + assert_eq!( + close_brackets(r#"{"s": [{"a": "a[b{c"}"#).as_deref(), + Some(r#"{"s": [{"a": "a[b{c"}]}"#) ); - // 没有开头时，提示词与从前一字不差 - let plain = build_messages(&[], &[], &[], None, "a.html", &[], "t"); - let framed = build_messages_with_opening(&[], &[], &[], None, "a.html", &[], None, "t"); - assert_eq!(plain[1].content, framed[1].content); + assert!(close_brackets(r#"{"s": "cut in the mid"#).is_none()); + assert!(close_brackets("]").is_none()); } #[test] @@ -1597,136 +755,6 @@ mod prompt_shape_tests { assert_eq!(opening_block(Some(" ")), ""); } - /// 已知实体必须落在 **user** 消息里、紧挨着正文。 - /// - /// 理由是服从性不是缓存：抽象规则打不过挨着它的具体块。清单放进 system 的 - /// 规则区，就会隔着输出格式、十条规则、文件名，离它要管的正文最远。 - #[test] - fn known_entities_stay_out_of_the_system_message() { - // 用一个规则 1 的例子里没有的名字：规则 1 也提"星云科技上海研究院"， - // 拿它断言等于测不出清单到底在哪条消息里 - let known = vec![KnownEntity { - handle: "k1".into(), - type_key: "organization".into(), - name: "华瑞集团智能制造研究院".into(), - }]; - let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text"); - assert_eq!(msgs[0].role, "system"); - assert!(!msgs[0].content.contains("Already recorded")); - assert!(!msgs[0].content.contains("华瑞集团智能制造研究院")); - assert!(msgs[1] - .content - .contains("k1 [organization]: 华瑞集团智能制造研究院")); - } - - /// 第一块没有"前面"，那一段应当完全不出现——成本为零，而不是一段空标题 - #[test] - fn the_first_chunk_carries_no_block() { - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - assert!(!msgs[1].content.contains("Already recorded")); - } - - /// 反向护栏必须在：给了参照物就会有人硬套（`concept` 那次的教训） - #[test] - fn the_block_tells_the_model_not_to_force_a_match() { - let known = vec![KnownEntity { - handle: "k1".into(), - type_key: "person".into(), - name: "陈立".into(), - }]; - let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text"); - assert!(msgs[1].content.contains("do not force it onto this list")); - } - - /// 有描述就不送 label——中文库的 label 是中文，混进提示词只会让 - /// 标识符与语料语言来回跳，而它相对 key 近乎零信息量。 - #[test] - fn described_types_drop_the_label() { - let types = vec![ - ( - "person".into(), - "人物".into(), - "有名有姓的具体的人。".into(), - ), - ("event".into(), "事件".into(), String::new()), - ]; - let msgs = build_messages(&types, &[], &[], None, "a.txt", &[], "text"); - let prompt = format!("{:?}", msgs); - assert!(prompt.contains("- person: 有名有姓的具体的人。")); - assert!(!prompt.contains("person (人物)")); - // 描述为空时 label 仍是唯一的额外线索，留着 - assert!(prompt.contains("- event (事件)")); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// 边上的属性（0037）：清单里跟在关系后面，回复里挂在事实上。 - #[test] - fn a_relation_lists_its_qualifiers_and_a_fact_carries_them() { - use serde_json::json; - let mut r = PromptRelation { - key: "invested_in".into(), - label: "invested in".into(), - description: "money into a company".into(), - signature: "organization → organization".into(), - temporal: "event".into(), - qualifiers: vec!["amount: number $".into(), "stake: number %".into()], - }; - let msgs = build_messages( - &[], - std::slice::from_ref(&r), - &[], - None, - "a.txt", - &[], - "text", - ); - let prompt = format!("{:?}", msgs); - // 签名、标记、属性清单三段顺序固定：`(签名) [event] {属性}` - assert!(prompt.contains( - "- invested_in (organization → organization) [event] {amount: number $, stake: number %}: money into a company" - ), "{prompt}"); - // 不带属性的关系不多一个花括号 - r.qualifiers.clear(); - let prompt = format!( - "{:?}", - build_messages( - &[], - std::slice::from_ref(&r), - &[], - None, - "a.txt", - &[], - "text" - ) - ); - assert!( - prompt.contains("- invested_in (organization → organization) [event]: money"), - "{prompt}" - ); - assert!(!prompt.contains("[event] {")); - - // 回复：qualifiers 挂在关系事实上；没写的是 None，旧回复不受影响 - let reply = r#"{"entities":[],"facts":[ - {"subject":"Vega","predicate":"invested_in","object":"Northwind", - "qualifiers":{"amount":"$5 billion"},"confidence":0.9}, - {"subject":"Vega","predicate":"invested_in","object":"Kestrel","confidence":0.9} - ]}"#; - let parsed = parse_response(reply).unwrap(); - assert_eq!(parsed.facts.len(), 2); - assert_eq!( - parsed.facts[0] - .qualifiers - .as_ref() - .and_then(|q| q.get("amount")), - Some(&json!("$5 billion")) - ); - assert!(parsed.facts[1].qualifiers.is_none()); - } - #[test] fn a_quantity_is_the_whole_string_or_nothing() { // 整体就是一个量：符号、量级词、千分位都读得动 @@ -1851,22 +879,6 @@ mod tests { assert_eq!(normalize_attr_value("number", &json!("about ten")), None); } - #[test] - fn a_description_is_cut_at_its_first_sentence() { - assert_eq!( - first_sentence( - " The date on which the CreativeWork was created. See also dateModified.\n\nExample: 2020-01-01." - ), - "The date on which the CreativeWork was created." - ); - assert_eq!( - first_sentence("一个有名有姓的人。可以是虚构的。"), - "一个有名有姓的人。" - ); - assert_eq!(first_sentence("A person"), "A person"); - assert_eq!(first_sentence(" "), ""); - } - #[test] fn parse_time_precisions() { assert_eq!(parse_time("2024").unwrap().1, "year"); @@ -1997,211 +1009,6 @@ mod tests { ); assert_eq!(normalize_attr_value("text", &json!([1])), None); } - - /// **一条坏记录不该毁掉一整块。** - /// - /// 形态取自真实日志：`missing field \`predicate\``。模型偶尔会漏写这个字段 - /// （`related_to` 退场后它没有万能选项可挑），从前 serde 会让整块作废， - /// 而这一块里另外两条事实是好的。 - #[test] - fn one_malformed_fact_does_not_take_the_whole_chunk() { - let raw = r#"{ - "entities": [{"name": "OpenAI", "type": "organization"}], - "facts": [ - {"subject": "OpenAI", "predicate": "produces", "object": "GPT-4"}, - {"subject": "OpenAI", "object": "ChatGPT"}, - {"subject": "Sam Altman", "predicate": "leads", "object": "OpenAI"} - ] - }"#; - let x = parse_response(raw).unwrap(); - assert_eq!(x.facts.len(), 2, "好的两条该留下"); - assert_eq!(x.skipped_facts, 1, "跳过的那条要报出来，不能静默"); - assert_eq!(x.entities.len(), 1); - assert!(!x.truncated); - } - - /// **输出被截断时，已经完整的那些要救回来。** - /// - /// 撞上 max_tokens 时模型写到一半就没了（真实日志：`EOF while parsing a list`）。 - /// 前面的对象是完整且正确的，整块作废等于把抽对的十几条一起扔掉。 - #[test] - fn a_cut_off_reply_keeps_what_was_complete() { - let raw = r#"{ - "entities": [{"name": "Anthropic", "type": "organization"}], - "facts": [ - {"subject": "Anthropic", "predicate": "produces", "object": "Claude"}, - {"subject": "Dario Amodei", "predicate": "leads", "object": "Anthropic"}, - {"subject": "Anthropic", "predicate": "loca"#; - let x = parse_response(raw).unwrap(); - assert!(x.truncated, "截断要标出来"); - assert_eq!(x.facts.len(), 2, "断点之前的两条是完整的"); - assert_eq!(x.entities.len(), 1); - } - - /// 括号出现在字符串里不算结构——`"a[b"` 不是一个开括号。 - #[test] - fn brackets_inside_strings_are_not_structure() { - let raw = - r#"{"entities": [], "facts": [{"subject": "a[b{c", "predicate": "p", "object": "o"}]}"#; - let x = parse_response(raw).unwrap(); - assert_eq!(x.facts.len(), 1); - assert!(!x.truncated, "结构完整，不该判成截断"); - } - - /// 连一个完整对象都没有时，仍然要报失败——**容错不是把空结果说成成功**。 - #[test] - fn a_reply_with_nothing_complete_still_fails() { - assert!(parse_response(r#"{"facts": [{"subject": "a"#).is_err()); - } - - /// 片段字段可有可无：老模型输出没有它们，照常解析 - #[test] - fn spans_parse_and_default_to_none() { - let with = parse_response( - r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic","subject_span":"Former OpenAI personnel","object_span":"Anthropic"}]}"#, - ) - .unwrap(); - assert_eq!( - with.facts[0].subject_span.as_deref(), - Some("Former OpenAI personnel") - ); - assert_eq!(with.facts[0].object_span.as_deref(), Some("Anthropic")); - let without = parse_response( - r#"{"entities":[],"facts":[{"subject":"OpenAI","predicate":"founded","object":"Anthropic"}]}"#, - ) - .unwrap(); - assert!(without.facts[0].subject_span.is_none()); - assert!(without.facts[0].object_span.is_none()); - } - - #[test] - fn names_are_parsed_and_a_malformed_one_is_skipped() { - let raw = r#"{"entities":[{"local_id":"e1","name":"海洋探测器1号","type":"equipment"}], - "facts":[], - "names":[{"ref":"e1","name":"海探1","quote":"海洋探测器1号（简称“海探1”）"}, - {"name":"no ref"}]}"#; - let x = parse_response(raw).unwrap(); - assert_eq!(x.names.len(), 1); - assert_eq!(x.names[0].entity_ref, "e1"); - assert_eq!(x.names[0].name, "海探1"); - assert_eq!(x.skipped_entities, 0, "a bad name is not a skipped entity"); - } - - #[test] - fn the_contract_asks_for_other_names_and_forbids_descriptions() { - let msgs = build_messages(&[], &[], &[], None, "f.txt", &[], "text"); - let system = &msgs[0].content; - assert!(system.contains("\"names\":[{\"ref\"")); - assert!(system.contains("1b. Every other name the text gives an entity")); - } - - #[test] - fn parse_response_with_fence() { - let raw = "好的，结果如下：\n```json\n{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}\n```"; - let e = parse_response(raw).unwrap(); - assert_eq!(e.entities.len(), 1); - assert_eq!(e.entities[0].type_key, "person"); - } - - /// #690：思考过程里的大括号不能把 JSON 的起止带偏。 - /// - /// 思考过程里这个没闭合的 `{` 会把"第一个 `{`"的起点提前，而修补截断的逻辑 - /// 认不出夹在中间的废话——不切掉标记，整块直接报解析失败作废。 - #[test] - fn parse_response_ignores_a_think_block_before_the_json() { - let raw = "先想想 {\"a\": 1，再回答。\n{\"entities\":[{\"name\":\"张三\",\"type\":\"person\"}],\"facts\":[]}"; - let e = parse_response(raw).unwrap(); - assert_eq!(e.entities.len(), 1); - assert_eq!(e.entities[0].type_key, "person"); - } - - #[test] - fn handles_and_fact_refs_are_optional_and_legacy_compatible() { - let handled = parse_response( - r#"{"entities":[{"local_id":"e1","name":"Zhang Wei","type":"person"}], - "facts":[{"subject":"Zhang Wei","subject_ref":"e1","predicate":"leads", - "object":"Finance","object_ref":"e2"}]}"#, - ) - .unwrap(); - assert_eq!(handled.entities[0].local_id.as_deref(), Some("e1")); - assert_eq!(handled.facts[0].subject_ref.as_deref(), Some("e1")); - assert_eq!(handled.facts[0].object_ref.as_deref(), Some("e2")); - - let legacy = parse_response( - r#"{"entities":[{"name":"Zhang Wei","type":"person"}], - "facts":[{"subject":"Zhang Wei","predicate":"leads","object":"Finance"}]}"#, - ) - .unwrap(); - assert_eq!(legacy.entities[0].local_id, None); - assert_eq!(legacy.facts[0].subject_ref, None); - assert_eq!(legacy.facts[0].object_ref, None); - } - - #[test] - fn duplicate_handle_definitions_are_all_malformed() { - let x = parse_response( - r#"{"entities":[ - {"local_id":"e1","name":"Zhang Wei","type":"person"}, - {"local_id":"e1","name":"John Smith","type":"person"}, - {"local_id":"e2","name":"Finance","type":"organization"}], - "facts":[]}"#, - ) - .unwrap(); - assert_eq!(x.skipped_entities, 2); - assert_eq!(x.entities.len(), 1); - assert_eq!(x.entities[0].local_id.as_deref(), Some("e2")); - } - - /// specific_type 在骨架里、也在规则里，且两处都说"永远要填"。 - /// - /// 只写进骨架是不够的：**规则与骨架冲突时骨架赢**（语言那条就栽过一次）。 - /// 这里两边一致，所以要一起钉住。 - #[test] - fn every_entity_is_asked_for_its_own_words() { - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - let sys = &msgs[0].content; - assert!(sys.contains("\"specific_type\":\"what you would call it\"")); - assert!(sys.contains("required on every entity")); - // 关键的一句：不校验。校验它就等于又造了一个词表 - assert!(sys.contains("never checked against the list")); - // 与 type 的关系必须说清楚，否则模型会把粗类抄一遍 - assert!(sys.contains("narrower than")); - } - - #[test] - fn extraction_contract_explains_handle_identity_without_forcing_splits() { - let msgs = build_messages(&[], &[], &[], None, "a.txt", &[], "text"); - let system = &msgs[0].content; - assert!(system.contains("\"local_id\":\"e1\"")); - assert!(system.contains("\"subject_ref\":\"e1\"")); - // #578：跟 X 有关的一群人不是 X - assert!(system.contains("subject_span and object_span are the exact words")); - assert!(system.contains("unique within this response")); - assert!(system.contains("Reuse the same local_id")); - assert!(system.contains("permanent identity is proven")); - assert!(system.contains("A shared name alone neither proves sameness nor requires a split")); - } - - #[test] - fn same_name_known_entities_keep_distinct_response_handles() { - let known = vec![ - KnownEntity { - handle: "k1".into(), - type_key: "person".into(), - name: "Zhang Wei".into(), - }, - KnownEntity { - handle: "k2".into(), - type_key: "person".into(), - name: "Zhang Wei".into(), - }, - ]; - let msgs = build_messages(&[], &[], &[], None, "a.txt", &known, "text"); - let user = &msgs[1].content; - assert!(user.contains("k1 [person]: Zhang Wei")); - assert!(user.contains("k2 [person]: Zhang Wei")); - assert!(user.contains("subject_ref/object_ref")); - } } #[cfg(test)] diff --git a/crates/utopia-extract/src/normalize.rs b/crates/utopia-extract/src/normalize.rs deleted file mode 100644 index e2abbae47..000000000 --- a/crates/utopia-extract/src/normalize.rs +++ /dev/null @@ -1,1067 +0,0 @@ -//! 模型回复落库前的**形状检查**：只看结构，不看词。 -//! -//! **分工。**读懂原文里的时间、判断一段话是不是一个东西——这是语言问题，归模型， -//! 契约（提示词 3c）说清楚它该怎么写。这里只核对输出有没有照契约的形状写，判据一律 -//! 是结构性的：引文里有没有这段字、值是不是只有标点、一侧是不是契约的日期格式、 -//! 同一句里有没有另一条边。**不认任何一种语言的词**——第一版按英文词表认「季度」 -//!「N months ended」，中文财报一条都认不出，还把四份报告的标题当成期间删了。 -//! -//! 每条规则做了什么都返回给服务端记进丢弃表：违约多常见、出在哪个模型，量得出来， -//! 契约该怎么改看数说话。 - -use crate::{read_time, ExtractedEntity, ExtractedFact, Extraction}; -use std::collections::HashSet; - -/// 形状检查做了什么；服务端按条记信号 -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Normalization { - /// 值只有破折号（`—`）或是空的：表里的「无」，不是一个值，不落。**只认破折号**： - /// `☒`、`✓` 这类符号没有字母数字，却是一格写着的内容（勾选了），照值落 - NoValue { predicate: String, written: String }, - /// 值后面有一截引文里没有的字：只留引文里有的那段。模型读对了表头、却把期间 - /// 写进了值（`(6,176) for three months ended July 27, 2025`），引文只有 `(6,176)` - ValueTrimmed { - predicate: String, - kept: String, - dropped: String, - }, - /// 没有宾语、没有值、只带边属性：每个属性落成主语上的一条值事实。 - /// 从前整条以 object_missing 丢掉，写对了的数跟着没了 - QualifiersWithoutObject { predicate: String, values: usize }, - /// 宾语是契约格式的日期（`2026-06-30`）：时间不是实体。边上的数落成值、日期进有效期； - /// 没带数的，把写出来的那段落成值——`2028`（「2028 年起上线」）、`4000`（人数）都 - /// 解析得成年份，丢掉就把一条信息整个丢了 - TimeAsObject { - predicate: String, - written: String, - values: usize, - }, - /// 主语是契约格式的日期：数是某个东西在那一刻的数，那个东西是谁回复里没说。不落 - TimeAsSubject { predicate: String, written: String }, - /// 宾语的名字包住了另一个声明实体，而同一句、同主语、同谓词已有一条指向那个实体的边： - /// 它**可能**是那个实体的描述。**只记，不删**：「non-GAAP net income, or earnings, per - /// diluted share」包住了「non-GAAP net income」，却是另一个指标（每股收益）。结构上 - /// 分不出描述与另一个东西，删错了就是实体连事实一起没了 - ObjectDescribesDeclared { - predicate: String, - name: String, - head: String, - }, - /// 上面几条去掉事实之后，没有任何事实再引用的声明：不建，否则就是一个孤点 - OrphanDeclaration { name: String }, - /// 引文抄自提示词里附的文件开头，而不是这一块：开头只作背景，它自己那一块会抽到。 - /// 照落的话，证据挂在这一块上，引的却是第一块的话——引错了出处 - QuoteFromOpening { predicate: String, quote: String }, -} - -/// 比对用的形态：空白折叠、小写 -fn norm(s: &str) -> String { - s.split_whitespace() - .collect::>() - .join(" ") - .to_lowercase() -} - -/// 比对引文用的词元：一段连续的数字（中间的 `,` `.` 算在数里，`13,237`、`89.0`），或者 -/// 一段连续的非数字字母。其余字符都是分隔。 -/// -/// **按词元比，不按子串比**：从前 `10 to 15 GW` 对着「10–15 GW」，`10` 作为子串在引文 -/// 里，尾巴 `to 15 GW` 作为整段不在，于是被剪成 `10`。数字与汉字之间也切开——中文里数 -/// 贴着字写（「营收为12,345元」），不切的话一个数永远对不上 -fn tokens(t: &str) -> Vec { - let chars: Vec = t.chars().collect(); - let mut out = Vec::new(); - let mut cur = String::new(); - let mut cur_digit = false; - for (i, &c) in chars.iter().enumerate() { - let digit = c.is_numeric(); - let joins_number = - (c == ',' || c == '.') && cur_digit && chars.get(i + 1).is_some_and(|n| n.is_numeric()); - if joins_number { - cur.push(c); - continue; - } - if !c.is_alphanumeric() { - if !cur.is_empty() { - out.push(std::mem::take(&mut cur)); - } - continue; - } - if !cur.is_empty() && digit != cur_digit { - out.push(std::mem::take(&mut cur)); - } - cur_digit = digit; - cur.extend(c.to_lowercase()); - } - if !cur.is_empty() { - out.push(cur); - } - out -} - -/// `needle` 的词元是否作为连续的一段出现在 `hay` 的词元里 -fn contains_tokens(hay: &[String], needle: &[String]) -> bool { - !needle.is_empty() && hay.windows(needle.len()).any(|w| w == needle) -} - -fn has_digit(t: &str) -> bool { - t.chars().any(char::is_numeric) -} - -/// 值后面是否挂着一截不属于它的字。返回 (保留的前缀, 去掉的尾巴)。 -/// -/// 两个条件同时成立才剪，都是结构，不认词，比的都是整词元（见 [`tokens`]）： -/// - **保留的那段在引文里，而且是一格的写法**——含数字、词元连续地出现在引文里，是原文 -/// 写的那个数；或者只有破折号（`—`），是原文那一格写的「没有」； -/// - **尾巴自己含数字，而且那些数一个都不在引文里**——它是另一条信息（一个期间、一个 -/// 日期、另一个百分比），不是这个数的单位。尾巴里有一个数在引文里，就说明它是原文的 -/// 一部分换了写法（`10 to 15 GW` 对着「10–15 GW」），不剪。 -/// -/// 第二条要数字，是因为挂在数后面、引文里又没有的，还有一类是对的：表头上的量级与 -/// 单位（`53,954 million USD`，这一行引文只有 `53,954`，`million` 在表头「in millions」）、 -/// 模型换了写法的单位（`10 gigawatts`）、缩写的头衔（`founder and CEO`）。它们不带数字， -/// 不剪。整个值在引文里的，一个字不动。 -fn ungrounded_tail<'a>(value: &'a str, quote: &str) -> Option<(&'a str, &'a str)> { - let q = norm(quote); - if q.is_empty() || q.contains(&norm(value)) { - return None; - } - let quote_tokens = tokens(quote); - let foreign_figures = |tail: &str| { - let numbers: Vec = tokens(tail).into_iter().filter(|w| has_digit(w)).collect(); - !numbers.is_empty() && numbers.iter().all(|w| !quote_tokens.contains(w)) - }; - let bounds: Vec = value - .char_indices() - .filter(|(i, c)| c.is_whitespace() && *i > 0) - .map(|(i, _)| i) - .collect(); - bounds - .iter() - .rev() - .map(|&i| { - ( - value[..i] - .trim() - .trim_end_matches([',', ';', ':', '\u{3001}', '\u{FF0C}']), - value[i..].trim(), - ) - }) - .find(|(p, r)| { - if p.is_empty() || r.is_empty() || !foreign_figures(r) { - return false; - } - if is_no_value(p) { - return q.contains(&norm(p)); - } - has_digit(p) && contains_tokens("e_tokens, &tokens(p)) - }) -} - -/// 表格里表示「没有」的那一格：空的，或者只有破折号。 -/// -/// **只认破折号（Unicode 的 Pd 类）**，不是「没有字母数字就算」：`☒`、`✓`、`☐` 也没有 -/// 字母数字，却是那一格真写着的内容，从前被当成空格子丢了 -fn is_no_value(written: &str) -> bool { - written.chars().filter(|c| !c.is_whitespace()).all(|c| { - matches!( - c, - '-' | '\u{058A}' | '\u{05BE}' | '\u{1400}' | '\u{1806}' | '\u{2010}' - ..='\u{2015}' - | '\u{2E17}' - | '\u{2E1A}' - | '\u{2E3A}' - | '\u{2E3B}' - | '\u{2E40}' - | '\u{301C}' - | '\u{3030}' - | '\u{30A0}' - | '\u{FE31}' - | '\u{FE32}' - | '\u{FE58}' - | '\u{FE63}' - | '\u{FF0D}' - ) - }) -} - -/// 边属性里不是值、是单位的那几个键（与服务端同一张表） -fn is_unit_key(k: &str) -> bool { - matches!( - k.trim().to_lowercase().as_str(), - "currency" | "币种" | "货币" | "unit" | "单位" - ) -} - -fn qualifier_values(f: &ExtractedFact) -> Vec<(String, serde_json::Value)> { - f.qualifiers - .as_ref() - .map(|q| { - q.iter() - .filter(|(k, v)| !v.is_null() && !is_unit_key(k)) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() - }) - .unwrap_or_default() -} - -fn value_fact( - f: &ExtractedFact, - predicate: String, - value: serde_json::Value, - valid_from: Option, -) -> ExtractedFact { - ExtractedFact { - subject: f.subject.clone(), - subject_ref: f.subject_ref.clone(), - predicate, - object: None, - object_ref: None, - value: Some(value), - qualifiers: None, - valid_from: valid_from.or_else(|| f.valid_from.clone()), - valid_to: f.valid_to.clone(), - confidence: f.confidence, - quote: f.quote.clone(), - subject_span: f.subject_span.clone(), - object_span: None, - relative: false, - } -} - -/// 一侧写的是一个时间：规则 3 的格式（`YYYY` / `YYYY-MM` / `YYYY-MM-DD`，带时区的时刻）， -/// 或写法说得清是哪天的日期（`written_date`，#688） -fn names_a_time(s: &str) -> bool { - read_time(s.trim()).is_some() -} - -/// 引文不在这一块、却在附上的文件开头里的事实：丢掉，连同因此没人引用的声明。 -/// -/// 只比「在不在」（空白、大小写不论），不比像不像：两边都找不到的引文不归这里管—— -/// 那是模型改写了原文，与开头无关 -pub fn drop_quotes_from_opening( - x: &mut Extraction, - chunk_text: &str, - opening: &str, -) -> Vec { - let (chunk, opening) = (norm(chunk_text), norm(opening)); - if opening.is_empty() { - return Vec::new(); - } - let before = referenced_names(&x.entities, &x.facts); - let mut out = Vec::new(); - x.facts.retain(|f| { - let q = norm(f.quote.as_deref().unwrap_or("")); - let from_opening = !q.is_empty() && !chunk.contains(&q) && opening.contains(&q); - if from_opening { - out.push(Normalization::QuoteFromOpening { - predicate: f.predicate.clone(), - quote: f.quote.clone().unwrap_or_default(), - }); - } - !from_opening - }); - if out.is_empty() { - return out; - } - let mut after = referenced_names(&x.entities, &x.facts); - // 与 normalize_facts 同一条：模型给它报了别的名字的声明不算孤点（0041） - after.extend(x.names.iter().filter_map(|n| { - x.entities - .iter() - .find(|e| e.local_id.as_deref().map(str::trim) == Some(n.entity_ref.trim())) - .map(|e| e.name.trim().to_lowercase()) - })); - let mut orphans = Vec::new(); - x.entities.retain(|e| { - let n = e.name.trim().to_lowercase(); - let orphan = before.contains(&n) && !after.contains(&n); - if orphan { - orphans.push(e.name.trim().to_string()); - } - !orphan - }); - out.extend( - orphans - .into_iter() - .map(|name| Normalization::OrphanDeclaration { name }), - ); - out -} - -/// 事实两侧绑到的声明名（小写）：有句柄按句柄，没有按写出来的名字 -fn referenced_names(entities: &[ExtractedEntity], facts: &[ExtractedFact]) -> HashSet { - let handle_name = |h: Option<&String>| { - h.and_then(|h| { - entities - .iter() - .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim())) - .map(|e| e.name.trim().to_string()) - }) - }; - facts - .iter() - .flat_map(|f| { - [ - handle_name(f.subject_ref.as_ref()).or_else(|| Some(f.subject.trim().to_string())), - handle_name(f.object_ref.as_ref()) - .or_else(|| f.object.as_deref().map(|o| o.trim().to_string())), - ] - }) - .flatten() - .map(|n| n.to_lowercase()) - .collect() -} - -pub fn normalize_facts(x: &mut Extraction) -> Vec { - let mut out = Vec::new(); - let entities = std::mem::take(&mut x.entities); - - // 一侧绑到的声明名：有句柄按句柄，没有按写出来的名字 - let handle_name = |h: Option<&String>| { - h.and_then(|h| { - entities - .iter() - .find(|e| e.local_id.as_deref().map(str::trim) == Some(h.trim())) - .map(|e| e.name.trim().to_string()) - }) - }; - let before = referenced_names(&entities, &x.facts); - - let mut facts: Vec = Vec::with_capacity(x.facts.len()); - for mut f in std::mem::take(&mut x.facts) { - let quote = f.quote.clone().unwrap_or_default(); - - // ---- 值 ---- - if let Some(written) = f.value.as_ref().and_then(|v| v.as_str()).map(str::to_owned) { - // 先剪再看空：`— for three months ended July 27, 2025` 剪掉尾巴才露出那一格是空的 - let figure = ungrounded_tail(&written, "e).map_or(written.as_str(), |(k, _)| k); - if is_no_value(figure) { - out.push(Normalization::NoValue { - predicate: f.predicate.clone(), - written, - }); - continue; - } - if let Some((kept, dropped)) = ungrounded_tail(&written, "e) { - out.push(Normalization::ValueTrimmed { - predicate: f.predicate.clone(), - kept: kept.to_string(), - dropped: dropped.to_string(), - }); - f.value = Some(serde_json::Value::String(kept.to_string())); - } - } - - // ---- 主语是时间 ---- - let subject = - handle_name(f.subject_ref.as_ref()).unwrap_or_else(|| f.subject.trim().to_string()); - if names_a_time(&subject) { - out.push(Normalization::TimeAsSubject { - predicate: f.predicate.clone(), - written: subject, - }); - continue; - } - - let values = qualifier_values(&f); - let object = handle_name(f.object_ref.as_ref()) - .or_else(|| f.object.as_deref().map(|o| o.trim().to_string())) - .filter(|o| !o.is_empty()); - let has_value = f.value.as_ref().is_some_and(|v| !v.is_null()); - - // ---- 只有边属性 ---- - if object.is_none() && !has_value && !values.is_empty() { - for (key, value) in &values { - let predicate = format!("{}.{}", f.predicate.trim(), key.trim()); - facts.push(value_fact(&f, predicate, value.clone(), None)); - } - out.push(Normalization::QualifiersWithoutObject { - predicate: f.predicate.clone(), - values: values.len(), - }); - continue; - } - - // ---- 宾语是时间 ---- - if let Some(o) = object.as_deref().filter(|o| names_a_time(o)) { - if values.is_empty() { - // 没带数：写出来的那段就是值。`2028`（「2028 年起上线」）、`4000`（人数）都解析 - // 得成年份，从前整条丢掉，一条信息就没了。宾语那个声明没人引用，下面按孤点去掉 - facts.push(value_fact( - &f, - f.predicate.trim().to_string(), - serde_json::Value::String(o.to_string()), - None, - )); - out.push(Normalization::TimeAsObject { - predicate: f.predicate.clone(), - written: o.to_string(), - values: 0, - }); - continue; - } - let several = values.len() > 1; - for (key, value) in &values { - let predicate = if several { - format!("{}.{}", f.predicate.trim(), key.trim()) - } else { - f.predicate.trim().to_string() - }; - facts.push(value_fact( - &f, - predicate, - value.clone(), - Some(o.to_string()), - )); - } - out.push(Normalization::TimeAsObject { - predicate: f.predicate.clone(), - written: o.to_string(), - values: values.len(), - }); - continue; - } - - facts.push(f); - } - - // ---- 描述：名字包住另一个声明实体，本尊那条边同句已在。只记，不删 ---- - let declared: Vec = entities.iter().map(|e| e.name.trim().to_string()).collect(); - let side = |r: Option<&String>, w: Option<&str>| { - handle_name(r) - .or_else(|| w.map(|s| s.trim().to_string())) - .filter(|s| !s.is_empty()) - }; - let objects: Vec> = facts - .iter() - .map(|f| side(f.object_ref.as_ref(), f.object.as_deref())) - .collect(); - let subjects: Vec> = facts - .iter() - .map(|f| side(f.subject_ref.as_ref(), Some(f.subject.as_str()))) - .collect(); - // 名字边界：拉丁字母数字前后不能粘着字母数字；汉字之间本来就没有空格，不设边界 - let contains_name = |outer: &str, inner: &str| { - let (o, i) = (norm(outer), norm(inner)); - !i.is_empty() - && o.len() > i.len() - && o.match_indices(&i).any(|(at, _)| { - let glued = |c: Option| c.is_some_and(|c| c.is_ascii_alphanumeric()); - let edge_in = i.chars().next().is_some_and(|c| c.is_ascii_alphanumeric()); - let edge_out = i - .chars() - .next_back() - .is_some_and(|c| c.is_ascii_alphanumeric()); - !(edge_in && glued(o[..at].chars().next_back())) - && !(edge_out && glued(o[at + i.len()..].chars().next())) - }) - }; - for i in 0..facts.len() { - let Some(name) = objects[i].as_deref() else { - continue; - }; - let quote_i = norm(facts[i].quote.as_deref().unwrap_or("")); - let head = declared.iter().find(|h| { - contains_name(name, h) - && (0..facts.len()).any(|j| { - j != i - && facts[j].predicate.eq_ignore_ascii_case(&facts[i].predicate) - && subjects[j] == subjects[i] - && objects[j] - .as_deref() - .is_some_and(|o| o.eq_ignore_ascii_case(h)) - && { - let quote_j = norm(facts[j].quote.as_deref().unwrap_or("")); - quote_i.contains("e_j) || quote_j.contains("e_i) - } - }) - }); - if let Some(head) = head { - // 不删（见枚举上的说明）：结构分不出「SB Energy 的发展」与「每股收益」这类 - // 包住了另一个名字的真指标。记下来，量得出这种形状多常见、有多少是描述 - out.push(Normalization::ObjectDescribesDeclared { - predicate: facts[i].predicate.clone(), - name: name.to_string(), - head: head.clone(), - }); - } - } - x.facts = facts; - - // ---- 被上面几条弄成孤点的声明 ---- - // 只去掉「原来有事实引用、现在没有了」的：模型一开始就只声明不连边的，不归这里管 - let mut after = referenced_names(&entities, &x.facts); - // 模型给它报了别的名字的声明也不算孤点：那些名字要绑在它身上（0041） - after.extend( - x.names - .iter() - .filter_map(|n| handle_name(Some(&n.entity_ref))) - .map(|n| n.to_lowercase()), - ); - let mut orphans = Vec::new(); - let mut kept_entities = entities; - kept_entities.retain(|e| { - let n = e.name.trim().to_lowercase(); - let orphan = before.contains(&n) && !after.contains(&n); - if orphan { - orphans.push(e.name.trim().to_string()); - } - !orphan - }); - x.entities = kept_entities; - out.extend( - orphans - .into_iter() - .map(|name| Normalization::OrphanDeclaration { name }), - ); - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ExtractedEntity; - - fn fact(subject: &str, predicate: &str, object: Option<&str>, quote: &str) -> ExtractedFact { - ExtractedFact { - subject: subject.into(), - subject_ref: None, - predicate: predicate.into(), - object: object.map(str::to_string), - object_ref: None, - value: None, - qualifiers: None, - valid_from: None, - valid_to: None, - confidence: Some(0.9), - quote: Some(quote.into()), - subject_span: Some(subject.into()), - object_span: object.map(str::to_string), - relative: false, - } - } - fn valued(subject: &str, predicate: &str, value: &str, quote: &str) -> ExtractedFact { - let mut f = fact(subject, predicate, None, quote); - f.value = Some(serde_json::Value::String(value.into())); - f - } - fn entity(id: &str, name: &str) -> ExtractedEntity { - ExtractedEntity { - local_id: Some(id.into()), - name: name.into(), - type_key: "organization".into(), - specific_type: None, - } - } - fn quals(pairs: &[(&str, &str)]) -> serde_json::Map { - pairs - .iter() - .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string()))) - .collect() - } - /// 第五份补充协议的第三块：模型从附上的开头里抄了「as of February 18, 2020」那句当引文。 - /// 那条事实丢掉，只为它而声明的实体跟着不建；引文在这一块里的照落 - #[test] - fn a_fact_quoting_the_opening_is_dropped_with_its_orphan() { - let opening = "THIS FIFTH AMENDMENT TO LEASE AGREEMENT is entered into as of February 18, 2020 by and between HPBB1, LLC and BLACKBAUD, INC."; - let chunk = "The Phase 2 Exercise Deadline is hereby extended to March 17, 2020."; - let mut x = Extraction { - entities: vec![entity("e1", "Lease"), entity("e2", "HPBB1, LLC")], - facts: vec![ - fact( - "Lease", - "landlord", - Some("HPBB1, LLC"), - "entered into as of February 18, 2020 by and between HPBB1, LLC", - ), - fact( - "Lease", - "expansion_option_deadline", - None, - "the phase 2 exercise deadline is hereby extended to March 17, 2020", - ), - ], - names: Vec::new(), - skipped_entities: 0, - skipped_facts: 0, - truncated: false, - }; - let n = drop_quotes_from_opening(&mut x, chunk, opening); - let kept: Vec<&str> = x.facts.iter().map(|f| f.predicate.as_str()).collect(); - assert_eq!( - kept, - ["expansion_option_deadline"], - "大小写与空白不论，引文在这一块里的留下" - ); - let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect(); - assert_eq!(names, ["Lease"], "只为被丢那条声明的 HPBB1 不建"); - assert!(n - .iter() - .any(|v| matches!(v, Normalization::QuoteFromOpening { .. }))); - assert!(n - .iter() - .any(|v| matches!(v, Normalization::OrphanDeclaration { .. }))); - - // 两边都找不到的引文不归这里管；没有开头时什么都不做 - let mut y = Extraction { - entities: vec![entity("e1", "Lease")], - facts: vec![fact("Lease", "note", None, "a paraphrase of neither")], - names: Vec::new(), - skipped_entities: 0, - skipped_facts: 0, - truncated: false, - }; - assert!(drop_quotes_from_opening(&mut y, chunk, opening).is_empty()); - assert!(drop_quotes_from_opening(&mut y, chunk, " ").is_empty()); - assert_eq!(y.facts.len(), 1); - } - - fn run( - entities: Vec, - facts: Vec, - ) -> (Extraction, Vec) { - let mut x = Extraction { - entities, - facts, - skipped_entities: 0, - skipped_facts: 0, - truncated: false, - names: Vec::new(), - }; - let n = normalize_facts(&mut x); - (x, n) - } - fn value_of(f: &ExtractedFact) -> &str { - f.value.as_ref().unwrap().as_str().unwrap() - } - - /// 截图里那 21 条：模型读对了表头，却把期间写进了值。引文里只有数 - #[test] - fn a_tail_the_quote_does_not_contain_is_not_the_value() { - let (x, n) = run( - vec![entity("e1", "NVIDIA")], - vec![ - valued( - "NVIDIA", - "net_cash_used", - "(6,176) for three months ended July 27, 2025", - "Net cash used in financing activities (6,176)", - ), - // 同样的形状，中文：不认词，照样剪 - valued( - "NVIDIA", - "经营现金流", - "12,345 截至2025年7月27日止三个月", - "经营活动产生的现金流量净额 | 12,345", - ), - valued( - "NVIDIA", - "revenue", - "$89.0 billion, up 18% from the previous quarter", - "Second-quarter revenue was $89.0 billion", - ), - ], - ); - assert_eq!(value_of(&x.facts[0]), "(6,176)"); - assert_eq!(value_of(&x.facts[1]), "12,345"); - // 数后面接着另一个数：「up 18%」是另一条事实，不是这个数的一部分；逗号跟着前缀走 - assert_eq!(value_of(&x.facts[2]), "$89.0 billion"); - assert!( - matches!(&n[0], Normalization::ValueTrimmed { dropped, .. } if dropped == "for three months ended July 27, 2025") - ); - } - - #[test] - fn a_value_grounded_in_the_quote_is_left_alone() { - let (x, n) = run( - vec![entity("e1", "Tench Coxe")], - vec![ - // 整个值在引文里 - valued( - "NVIDIA", - "operating_expenses", - "$9.2 billion", - "expected to be approximately $9.2 billion and $9.0 billion", - ), - // 尾巴 shares 在引文别处出现过：原文的单位，不剪 - valued( - "Tench Coxe", - "votes_for", - "15,411,252,412 shares", - "Number of shares For | 15,411,252,412", - ), - // 规范化过的值，前缀一个词都对不上：不是这条规则管的 - valued("Vega", "amount", "$5 billion", "invested 5 billion dollars"), - // 表头上的量级与币种：引文那一行只有数，尾巴不带数字，是它的单位，不剪 - valued( - "NVIDIA", - "net_income", - "53,954 million USD", - "Net income | $ | 53,954", - ), - // 模型换了写法的单位、缩写的头衔：不带数字，不剪 - valued( - "SB Energy", - "capacity", - "10 gigawatts", - "at least 10 GW of new generation", - ), - valued( - "Jensen Huang", - "job_title", - "founder and CEO", - "Jensen Huang, founder and chief executive officer", - ), - ], - ); - let got: Vec<&str> = x.facts.iter().map(value_of).collect(); - assert_eq!( - got, - [ - "$9.2 billion", - "15,411,252,412 shares", - "$5 billion", - "53,954 million USD", - "10 gigawatts", - "founder and CEO" - ] - ); - assert!(n.is_empty(), "{n:?}"); - } - - #[test] - fn a_dash_is_no_value() { - let (x, n) = run( - vec![entity("e1", "NVIDIA")], - vec![ - valued("NVIDIA", "dividend", "—", "Dividends | —"), - // 勾选框是那一格写着的内容，不是空格子：没有字母数字，照值落 - valued( - "NVIDIA", - "large_accelerated_filer", - "☒", - "Large accelerated filer | ☒", - ), - valued( - "NVIDIA", - "emerging_growth_company", - "☐", - "Emerging growth company | ☐", - ), - valued("NVIDIA", "dividend", " – ", "Dividends | –"), - valued("NVIDIA", "dividend", "$0.01", "Dividends | $0.01"), - // 空的那一格后面挂着列头上的期间：剪掉尾巴，剩下的仍是空 - valued( - "NVIDIA", - "amount", - "— for three months ended July 27, 2025", - "Purchases of marketable securities | — | (6,176)", - ), - // 反例：前缀只有标点、在引文里，可尾巴上的数也在引文里——那个数才是值 - valued( - "NVIDIA", - "net_income", - "$ 53,954 million", - "Net income | $ | 53,954", - ), - ], - ); - let kept: Vec<&str> = x.facts.iter().map(value_of).collect(); - assert_eq!(kept, ["☒", "☐", "$0.01", "$ 53,954 million"]); - assert_eq!( - n.iter() - .filter(|v| matches!(v, Normalization::NoValue { .. })) - .count(), - 3 - ); - } - - /// Coxe 的形状：`vote_result` 带着数，没有宾语也没有值 - #[test] - fn figures_on_an_edge_with_no_other_end_land_on_the_subject() { - let mut f = fact( - "Tench Coxe", - "vote_result", - None, - "Number of shares For | 15,411,252,412", - ); - f.qualifiers = Some(quals(&[ - ("for", "15,411,252,412"), - ("against", "1,399,727,580"), - ("unit", "shares"), - ])); - let (x, n) = run(vec![entity("e1", "Tench Coxe")], vec![f]); - let mut got: Vec<(String, String)> = x - .facts - .iter() - .map(|f| (f.predicate.clone(), value_of(f).to_string())) - .collect(); - got.sort(); - assert_eq!( - got, - [ - ( - "vote_result.against".to_string(), - "1,399,727,580".to_string() - ), - ("vote_result.for".to_string(), "15,411,252,412".to_string()), - ] - ); - assert_eq!( - n, - vec![Normalization::QualifiersWithoutObject { - predicate: "vote_result".into(), - values: 2 - }] - ); - } - - /// 时间做了宾语：边上的数落成值，时间进有效期，那个时间节点不建 - #[test] - fn a_time_object_becomes_the_validity_and_leaves_no_node() { - let mut margin = fact( - "NVIDIA", - "gross_margin", - Some("2026-06"), - "Gross margin | 75.0%", - ); - margin.object_ref = Some("e7".into()); - margin.qualifiers = Some(quals(&[("percentage", "75.0%")])); - let mut bare = fact("NVIDIA", "reported_in", Some("2026"), "reported in 2026"); - bare.object_ref = Some("e8".into()); - let (x, n) = run( - vec![ - entity("e1", "NVIDIA"), - entity("e7", "2026-06"), - entity("e8", "2026"), - ], - vec![margin, bare], - ); - // 带数的：数落成值、日期进有效期；没带数的：写出来的那段落成值，不丢 - assert_eq!(x.facts.len(), 2); - assert_eq!(x.facts[0].predicate, "gross_margin"); - assert_eq!(value_of(&x.facts[0]), "75.0%"); - assert_eq!(x.facts[0].valid_from.as_deref(), Some("2026-06")); - assert_eq!( - (x.facts[1].predicate.as_str(), value_of(&x.facts[1])), - ("reported_in", "2026") - ); - assert!(x.facts[1].object.is_none() && x.facts[1].object_ref.is_none()); - let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect(); - assert_eq!(names, ["NVIDIA"]); - assert_eq!( - n.iter() - .filter(|v| matches!(v, Normalization::OrphanDeclaration { .. })) - .count(), - 2 - ); - } - - #[test] - fn a_time_subject_is_not_placed() { - let f = valued( - "2026-07-26", - "revenue", - "$89.0 billion", - "revenue was $89.0 billion", - ); - let (x, n) = run(vec![entity("e1", "NVIDIA")], vec![f]); - assert!(x.facts.is_empty()); - assert!(matches!(&n[0], Normalization::TimeAsSubject { .. })); - } - - /// 不是契约格式的期间名（`Q2 FY27`、`第二季度`）不归这里判：那是模型的事，契约 3c 管 - #[test] - fn a_period_name_that_is_not_a_date_is_not_second_guessed() { - let mut f = fact( - "NVIDIA", - "gross_margin_for_period", - Some("Q2 FY27"), - "Gross margin | 75.0 | %", - ); - f.qualifiers = Some(quals(&[("percentage", "75.0%")])); - let (x, n) = run( - vec![entity("e1", "NVIDIA"), entity("e7", "Q2 FY27")], - vec![f], - ); - assert_eq!(x.facts.len(), 1); - assert_eq!(x.entities.len(), 2); - assert!(n.is_empty()); - } - - /// SB Energy 那句：名字包住了另一个声明实体，同句同主语同谓词已有一条指向它的边。 - /// 记一条信号，事实与声明都留着——结构分不出它与下面「每股收益」那种真指标 - #[test] - fn a_description_beside_its_head_is_flagged_not_removed() { - let quote = "NVIDIA to invest $1.5B in SB Energy now to support SB Energy\u{2019}s growth and commitments to the Ohio community"; - let mut head = fact("NVIDIA", "invested_in", Some("SB Energy"), quote); - head.object_ref = Some("e2".into()); - let mut desc = fact( - "NVIDIA", - "invested_in", - Some("SB Energy's growth and commitments to the Ohio community"), - quote, - ); - desc.object_ref = Some("e6".into()); - let (x, n) = run( - vec![ - entity("e1", "NVIDIA"), - entity("e2", "SB Energy"), - entity( - "e6", - "SB Energy's growth and commitments to the Ohio community", - ), - ], - vec![head, desc], - ); - // 只记不删：两条都在，那个声明也在（它仍被引用） - assert_eq!(x.facts.len(), 2); - assert_eq!(x.entities.len(), 3); - assert!(n.iter().any( - |v| matches!(v, Normalization::ObjectDescribesDeclared { head, .. } if head == "SB Energy") - )); - } - - /// 同样的结构，中文：不靠 's，照样认得出这个形状（只记） - #[test] - fn a_description_is_recognised_without_a_possessive() { - let quote = "英伟达向星辰能源投资15亿美元，支持星辰能源在俄亥俄的发展"; - let (x, n) = run( - vec![ - entity("e1", "英伟达"), - entity("e2", "星辰能源"), - entity("e3", "星辰能源在俄亥俄的发展"), - ], - vec![ - fact("英伟达", "投资", Some("星辰能源"), quote), - fact("英伟达", "投资", Some("星辰能源在俄亥俄的发展"), quote), - ], - ); - assert_eq!(x.facts.len(), 2); - assert_eq!(x.entities.len(), 3); - assert!(n - .iter() - .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. }))); - } - - /// **四处误伤，逐个钉住**（#637 实测）：每一处从前都把一条真信息丢了或剪坏了 - #[test] - fn what_the_shape_checks_used_to_break_is_kept() { - // 一、短数字被当子串匹配：`10 to 15 GW` 对着「10–15 GW」，从前剪成 `10` - let (x, n) = run( - vec![entity("e1", "SB Energy")], - vec![ - valued( - "SB Energy", - "planned_capacity", - "10 to 15 GW", - "SB Energy plans 10–15 GW of new generation", - ), - // 词元不是子串：`5` 不在「25 GW」里 - valued( - "SB Energy", - "planned_capacity", - "5 GW by 2030", - "SB Energy plans 25 GW by 2030", - ), - // 中文里数贴着字写，照样按词元对得上，尾巴那个日期不在引文里，剪 - valued( - "SB Energy", - "营收", - "12,345 截至2025年7月27日止三个月", - "营收为12,345元", - ), - ], - ); - let got: Vec<&str> = x.facts.iter().map(value_of).collect(); - assert_eq!(got, ["10 to 15 GW", "5 GW by 2030", "12,345"]); - assert_eq!(n.len(), 1, "{n:?}"); - - // 二、勾选框：见 a_dash_is_no_value - - // 三、四位数的宾语：`2028`、`4000` 解析得成年份，落成值，不丢 - let mut launch = fact( - "Aurora", - "launch_year", - Some("2028"), - "Aurora goes live from 2028", - ); - launch.object_ref = Some("e2".into()); - let mut staff = fact( - "Acme", - "employees", - Some("4000"), - "Acme employs 4000 people", - ); - staff.object_ref = Some("e3".into()); - let (x, _) = run( - vec![ - entity("e1", "Aurora"), - entity("e2", "2028"), - entity("e3", "4000"), - entity("e4", "Acme"), - ], - vec![launch, staff], - ); - let got: Vec<(&str, &str)> = x - .facts - .iter() - .map(|f| (f.predicate.as_str(), value_of(f))) - .collect(); - assert_eq!(got, [("launch_year", "2028"), ("employees", "4000")]); - let names: Vec<&str> = x.entities.iter().map(|e| e.name.as_str()).collect(); - assert_eq!(names, ["Aurora", "Acme"], "年份那两个声明不建成节点"); - - // 四、包住了另一个指标名的真指标：每股收益不是净利润的描述，留着 - let quote = "non-GAAP net income was $26.4 billion and non-GAAP net income, or earnings, per diluted share was $1.05"; - let (x, n) = run( - vec![ - entity("e1", "NVIDIA"), - entity("e2", "non-GAAP net income"), - entity("e3", "non-GAAP net income, or earnings, per diluted share"), - ], - vec![ - fact( - "NVIDIA", - "reported_metric", - Some("non-GAAP net income"), - quote, - ), - fact( - "NVIDIA", - "reported_metric", - Some("non-GAAP net income, or earnings, per diluted share"), - quote, - ), - ], - ); - assert_eq!(x.facts.len(), 2); - assert_eq!(x.entities.len(), 3); - assert!(n - .iter() - .any(|v| matches!(v, Normalization::ObjectDescribesDeclared { .. }))); - } - - /// 包住了别的名字、但旁边没有指向本尊的同一条边：可能真是一个东西，不动 - #[test] - fn a_name_that_contains_another_without_a_sibling_edge_is_left_alone() { - let quote = "Sam Altman was removed by OpenAI's board of directors"; - let (x, n) = run( - vec![ - entity("e1", "Sam Altman"), - entity("e2", "OpenAI"), - entity("e3", "OpenAI's board of directors"), - ], - vec![fact( - "Sam Altman", - "removed_by", - Some("OpenAI's board of directors"), - quote, - )], - ); - assert_eq!(x.facts.len(), 1); - assert_eq!(x.entities.len(), 3); - assert!(n.is_empty()); - } -} diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rs index 57f7c69c5..89a4a71d5 100644 --- a/crates/utopia-server/src/api/kbs.rs +++ b/crates/utopia-server/src/api/kbs.rs @@ -55,10 +55,6 @@ pub struct UpdateKbReq { /// 提示词都读它。探索生成的描述在另一个字段，PATCH 不了。见 #570 #[serde(default)] pub data_conventions: Option, - /// 开放抽取（0044 第一刀，#729）：开着，抽取只写开放图谱——陈述照文档的字落库， - /// 不读本体；关着走今天的类型化那条路。缺省关 - #[serde(default)] - pub open_extraction: Option, } /// 用户可见的 KB 列表（restricted 库仅矩阵成员与系统管理员可见）。 @@ -119,7 +115,6 @@ pub async fn create( None, None, None, - None, ) .await?; } @@ -202,7 +197,6 @@ pub async fn update( req.auto_type_resolution, req.governance, req.data_conventions.as_deref().map(str::trim), - req.open_extraction, ) .await?; // 打开开关就自动开始处理：排一轮，同库已排着的不重复 diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs index 647251c5c..887aa3e9e 100644 --- a/crates/utopia-server/src/extraction.rs +++ b/crates/utopia-server/src/extraction.rs @@ -1,13 +1,16 @@ -//! 图谱抽取任务：逐分块调用 LLM → 实体消解 v2 → 事实 + 证据写入账本。 +//! 图谱抽取任务的入口，以及开放写入路径（`extraction_open`）共用的那几块。 +//! +//! 抽取只写开放图谱（0044 决定 2）：每篇文档——记忆日志也在内——都从这里进 +//! `run_open`，提示词里没有本体，也没有带本体的第二条路。这里留下的是那条路借用的东西： +//! 限流退避的 chat 调用、来源给置信度设的上限、片段核对、丢弃信号、未抽完的判据、 +//! 名字到实体的消解（句柄、同名审核对）。 //! 与摄入管道分离（两段式可用）：索引完成即可搜可问，抽取慢慢跑。 //! 消解灰区只入审核队列并触发独立的攒批裁决任务——LLM 裁决永不阻塞本任务。 use crate::llm_util; -use crate::ontology_index; -use crate::predicate_match::PredicateIndex; use crate::state::AppState; use sqlx::PgPool; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::time::Duration; use utopia_core::models::Proposer; use uuid::Uuid; @@ -31,7 +34,7 @@ fn jitter(base: Duration) -> Duration { base / 2 + Duration::from_millis(if half == 0 { 0 } else { nanos % half }) } -/// 抽取的 chat 调用，**限流会退避重试**。 +/// 抽取的 chat 调用，**限流会退避重试**。温度由调用方定（开放抽取要 0，见 `extraction_open`）。 /// /// 限流与其他失败的区别是它会自己好，所以从前那句「跳过该分块」用在它身上 /// 就是把一分钟的等待换成永久的数据缺口——实测一次 1884 块的灌入里 @@ -43,16 +46,6 @@ fn jitter(base: Duration) -> Duration { /// 否则一个在等的分块会挡住本来可以通过的另一个。 /// - **`Retry-After` 多数厂商不发**，所以它只是「有则更准」，判据是错误类型 /// 本身；没有它就走指数退避。 -pub(crate) async fn chat_retrying_rate_limits( - state: &AppState, - settings: &utopia_core::models::LlmSettings, - client: &utopia_llm::LlmClient, - messages: &[utopia_llm::ChatMessage], -) -> anyhow::Result { - chat_retrying_rate_limits_at(state, settings, client, messages, None).await -} - -/// 同上，指定采样温度（开放抽取要 0，见 `extraction_open`）。 pub(crate) async fn chat_retrying_rate_limits_at( state: &AppState, settings: &utopia_core::models::LlmSettings, @@ -90,8 +83,6 @@ pub(crate) async fn chat_retrying_rate_limits_at( unreachable!("循环内必定 return") } -const MIN_CONFIDENCE: f32 = 0.6; - /// 模型看图描述出来的事实，置信度压到这里（0040 决定 4）：低于时态引擎自动关闭的门槛， /// 它能进图、能被搜到和引用，却不能单独把一段正确的旧值关掉——看柱状图读错一个数字是常事， /// 按普通事实入库的话，错的数会关掉它反驳的那个对的数，而库里没有一行说这次关闭靠的是一张图。 @@ -107,90 +98,6 @@ pub(crate) fn origin_ceiling(origin: &str, confidence: f32) -> f32 { } } -/// 这串字**是不是一个东西的名字**。 -/// -/// 判据是**词数**不是字符数。字符数分不开真假： -/// `US District Court for the Northern District of California`（57 字符）是真实体， -/// 而 `removal was driven by growing discontent and distrust with Altman`（65 字符） -/// 是一整个从句——两者字符数相近，词数也相近（9 vs 10），但后者带着**限定动词**。 -/// -/// 所以两条一起看：词数封顶挡住长句，而**句中的限定动词**挡住那些不长的从句。 -/// 机构名会长（"US District Court for the Northern District of California"）， -/// 但不会出现 "was driven by"、"showed"、"giving off" 这种谓语。 -/// -/// 上限取 12 个词：实测真实体里最长的机构名是 9 个词，留三个词的余量。 -/// 而被挡下的那些平均 14 个词。 -const MAX_NAME_WORDS: usize = 12; - -/// 句子里的谓语标志。**只列限定形式**——`used`、`flying` 这类分词在名词短语里 -/// 完全正常（"equipment used by X"），列进去会误伤真实体。 -const CLAUSE_MARKERS: &[&str] = &[ - "was", "were", "is", "are", "has", "have", "had", "will", "would", "showed", "said", "says", - "became", "went", "came", "did", "does", "gave", "took", "made", -]; - -/// 情态动词：英语里**封闭类**的限定形式——不像 `fell`、`hits` 那样开放无边，也不会 -/// 出现在名词短语里。`may`（月份、人名）与 `can`（容器）除外，它们兼作名词（#193） -const MODALS: &[&str] = &["could", "should", "might", "must", "shall"]; - -/// 结构信号：不靠词表也看得出的「这是一句话」（#193）。 -/// -/// 动词表挡不住新语料——英语有上千个限定形式，`could`、`fell`、`hits` 都不在那 19 个 -/// 词里，而且第一个例句恰好 12 个词，卡在上限上。结构信号迁移得动： -/// 1. **句号结尾**：末词是小写词并以句号收尾（`… as a whole.`）。专名缩写 `Inc.` / -/// `Co.` 是大写开头，不误伤。 -/// 2. **情态动词**：封闭类，见 [`MODALS`]。 -/// -/// 这两条**当场拒绝**——它们和词数上限一样是结构判据，不是又一份词表。 -/// `words` 是小写化、去标点的词，`raw_last` 是保留原样的末词 -fn reads_like_a_sentence(words: &[String], raw_last: Option<&str>) -> bool { - if words.len() > 2 && words.iter().any(|w| MODALS.contains(&w.as_str())) { - return true; - } - if let Some(stem) = raw_last.and_then(|w| w.strip_suffix('.')) { - if stem.chars().count() >= 3 && stem.chars().all(|c| c.is_lowercase()) { - return true; - } - } - false -} - -/// 弱信号：像从句，但不敢当场拒绝（#193）。 -/// -/// 守卫的样本全部来自一份语料，换一份就漏——换规则之前先要一份**跨语料的标注集**。 -/// 命中只记 `clause_suspect`（例句进 `extraction_drops`），实体照常落库；攒够两份语料的 -/// 样本再决定哪条升成硬规则。返回的是信号名，作为记录的 detail -/// 槽位片段核对的结论（#582） -#[derive(Debug, PartialEq)] -enum SpanVerdict { - /// 没给片段，或片段就是所绑的那个名字（同名、同词干、名字的一部分、名字后面接着 - /// 大写的续词——"Anthropic PBC"） - Ok, - /// 片段不在引文里：模型没照抄。当没给处理，记一笔 - NotInQuote, - /// 片段点的是另一个声明过的实体：改绑到它 - Rebind(String), - /// 片段是围着某个名字的短语，那个名字在里面只是修饰语（后面还有词、或带所有格）： - /// 描述，不是实体 - Described(String), - /// 片段是所绑名字前面带了别的词：头衔（"entrepreneur Tasha McCauley"）还是另一件 - /// 东西（"companies using OpenAI"），机器分不开。绑定照旧，只记 - Prefixed(String), - /// 片段里没有任何声明过的名字：模型消解了指代（"him"、"the company"）。绑定照旧，只记 - Coreference(String), - /// 片段抄的是事实另一侧的名字（宾语片段写成了主语）：抄错了位置。绑定照旧，只记 - Misplaced(String), -} - -/// 模型报给 `bound` 的别名，是不是已经声明成了另一个实体的名字（0041）。 -/// -/// 结构判据，不认词：同一个名字不会同时是两样东西的名字。模型把「海探1项目」列成 -/// 一个机构，又把它报成探测器的别名——两个答案打架，别名那个不要 -fn name_claimed_elsewhere(name: &str, bound: Uuid, declared: &HashMap) -> bool { - let key = utopia_store::resolution::normalize_name(name).to_lowercase(); - declared.get(&key).is_some_and(|id| *id != bound) -} - /// 片段在不在引文里：大小写、空白都不论 pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool { let norm = |s: &str| { @@ -203,291 +110,6 @@ pub(crate) fn span_in_quote(span: &str, quote: &str) -> bool { !s.is_empty() && q.contains(&s) } -/// 一个词：去掉两头标点和所有格后的小写形态，连同原样（看大小写用） -#[derive(Debug, Clone)] -struct Word<'a> { - clean: String, - raw: &'a str, -} - -/// 片段拆成词：去两头标点、去所有格（"OpenAI's" → openai）、去开头的冠词 -fn span_words(s: &str) -> Vec> { - let mut words: Vec> = s - .split_whitespace() - .filter_map(|raw| { - let w = raw.trim_matches(|c: char| !c.is_alphanumeric()); - let w = w - .strip_suffix("'s") - .or_else(|| w.strip_suffix("\u{2019}s")) - .unwrap_or(w); - (!w.is_empty()).then(|| Word { - clean: w.to_lowercase(), - raw, - }) - }) - .collect(); - while words - .first() - .is_some_and(|w| matches!(w.clean.as_str(), "the" | "a" | "an")) - { - words.remove(0); - } - words -} - -/// 一个词是不是专名的样子：首字符大写或数字（"PBC"、"LLC"、"LX"、"Global"） -fn looks_proper(raw: &str) -> bool { - raw.chars() - .find(|c| c.is_alphanumeric()) - .is_some_and(|c| c.is_uppercase() || c.is_numeric()) -} - -/// 词序列 needle 在 hay 里连续出现的位置 -fn find_words(hay: &[Word<'_>], needle: &[Word<'_>]) -> Option { - if needle.is_empty() || needle.len() > hay.len() { - return None; - } - (0..=hay.len() - needle.len()).find(|&i| { - needle - .iter() - .zip(&hay[i..]) - .all(|(n, h)| n.clean == h.clean) - }) -} - -/// 片段说的是不是这个名字：同名、同词干（Acme / Acme Corp.）、泛用后缀互推—— -/// 与消解召回同一套判据（`recall_keys`）；或者片段的词全在名字里（"Altman" 之于 -/// "Sam Altman"，"Anthropic" 之于 "Anthropic, PBC"） -fn slot_matches(span: &str, name: &str) -> bool { - let keys = |s: &str| { - let clean = span_words(s) - .into_iter() - .map(|w| w.clean) - .collect::>() - .join(" "); - utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean)) - }; - let (a, b) = (keys(span), keys(name)); - if !a.is_empty() && a.iter().any(|k| b.contains(k)) { - return true; - } - let (s, n) = (span_words(span), span_words(name)); - !s.is_empty() && s.iter().all(|w| n.iter().any(|x| x.clean == w.clean)) -} - -/// **模型抄，机器判**（#582）。 -/// -/// #559、#578、#581 是同一件事的三张脸：模型把事实挂到了错的参与者身上—— -/// "former OpenAI personnel" 写成 OpenAI，"lawsuit against OpenAI" 造成节点。给每一种 -/// 形状写一条规则、配一张词表，量出来规则的服从率一半上下，词表只认见过的形状。 -/// 这里换一个问法：不问模型「这算不算实体」，让它把引文里点名每一侧的那几个字 -/// 抄出来（`subject_span` / `object_span`）。抄是模型稳定会做的事；判断交给机器： -/// -/// 1. 片段是所绑的名字（同名、同词干、名字的一部分）→ 放行； -/// 2. 片段是另一个声明过的实体的名字（或那名字的一部分）→ 改绑； -/// 3. 片段里含所绑的名字、名字**后面**还有词（"former OpenAI personnel"、"The Verge -/// reporter"）或名字带所有格（"Anthropic's safeguards"）→ 名字在里面只是修饰语， -/// 这是描述。后面的词全是专名的样子（"Anthropic PBC"、"OpenAI Global LLC"）不算； -/// 4. 名字只在片段**末尾**、前面带了词 → 头衔还是另一件东西分不开，绑定照旧、只记； -/// 5. 片段里是别的声明过的名字带着修饰 → 描述；一个名字都没有 → 指代，绑定照旧、只记。 -/// -/// 按语法位置来的补充：名字后面紧跟逗号是同位语（"Helen Toner, strategy director -/// for …"），还是它；片段以这条事实**另一侧**的名字结尾（宾语片段抄成了主语），是抄错 -/// 位置，只记。 -/// -/// 同一套判据还用在模型**写的名字**和它 **ref 指的实体**之间（`written_verdict`）：写 -/// "OpenAI employees" 却 ref 到 OpenAI，是它自己的两个答案打架，v5 那轮的最后一条假边 -/// （"Eleven employees left OpenAI … to establish Anthropic" → Anthropic founded_by OpenAI） -/// 就是这么来的——片段 "eleven employees" 里没有名字，只看片段拦不住。 -/// -/// 只有 3 和 5 前半改动事实（主语丢、宾语落字面值）；其余都是信号，让每一层的比率 -/// 按库、按模型读得出来 -fn verify_span( - span: Option<&str>, - name: &str, - other: &str, - quote: &str, - declared: &HashMap, -) -> SpanVerdict { - let Some(span) = span.map(str::trim).filter(|s| !s.is_empty()) else { - return SpanVerdict::Ok; - }; - if !span_in_quote(span, quote) { - return SpanVerdict::NotInQuote; - } - if slot_matches(span, name) { - return SpanVerdict::Ok; - } - let s = span_words(span); - let n = span_words(name); - if s.is_empty() { - return SpanVerdict::Coreference(span.to_string()); - } - - // 2. 另一个声明过的实体：精确/词干命中优先，其次名字包住片段的（取最短的那个） - let mut others: Vec<&String> = declared.keys().filter(|k| k.as_str() != name).collect(); - others.sort(); - if let Some(k) = others.iter().find(|k| { - let keys = |x: &str| { - let clean = span_words(x) - .into_iter() - .map(|w| w.clean) - .collect::>() - .join(" "); - utopia_store::resolution::recall_keys(&utopia_store::resolution::normalize_name(&clean)) - }; - let (a, b) = (keys(span), keys(k)); - a.iter().any(|x| b.contains(x)) - }) { - return SpanVerdict::Rebind((*k).clone()); - } - // 单个词包在别的名字里不算改绑："company" 是指代，句首的 "Stockholders" 大写也 - // 不是专名的证据（召回测量台第二轮：主语从 NVIDIA 改绑到了「年度股东大会」）。 - // 单个词只认精确/词干命中（上面 `slot_matches` 那一关） - let names_something = s.len() >= 2; - let mut supersets: Vec<(&String, usize)> = others - .iter() - .filter_map(|k| { - let kw = span_words(k); - (names_something && s.iter().all(|w| kw.iter().any(|x| x.clean == w.clean))) - .then_some((*k, kw.len())) - }) - .collect(); - supersets.sort_by_key(|(_, len)| *len); - if let Some((k, _)) = supersets.first() { - return SpanVerdict::Rebind((*k).clone()); - } - // 所绑的名字在片段里的位置；名字后面紧跟逗号是同位语（"TBPN, a media company in - // California"），还是它——先于下面所有判断 - let found = find_words(&s, &n); - if let Some(i) = found { - let end = i + n.len(); - if end < s.len() && s[end - 1].raw.trim_end().ends_with(',') { - return SpanVerdict::Ok; - } - } - - // 片段以事实另一侧的名字结尾：抄错了位置（宾语片段写成了主语），不是绑错了实体。 - // 「片段以别的声明名结尾就改绑到它」试过两轮（v4/v5）：介词的补语、名单的最后一项、 - // 并列的后一个（"Swisher and … Alex Heath"）都会被当成头，错的多过对的，不做 - let ends_with = |k: &str| { - let kw = span_words(k); - !kw.is_empty() && kw.len() < s.len() && find_words(&s[s.len() - kw.len()..], &kw) == Some(0) - }; - if !other.is_empty() && !slot_matches(name, other) && ends_with(other) { - return SpanVerdict::Misplaced(span.to_string()); - } - - // 3 / 4. 所绑的名字在片段里 - if let Some(i) = found { - let end = i + n.len(); - let last = s[end - 1].raw; - let possessive = last.ends_with("'s") || last.ends_with("\u{2019}s"); - let after = &s[end..]; - // 名字后面接着 and / & 再接专名，是并列（"SB Energy and SoftBank"）：名字是名单里 - // 的一项，不是修饰语。连词跟冠词一样是语法词，不是词表 - let after = match after.first() { - Some(w) if matches!(w.clean.as_str(), "and" | "&") => &after[1..], - _ => after, - }; - if possessive || after.iter().any(|w| !looks_proper(w.raw)) { - return SpanVerdict::Described(span.to_string()); - } - if after.is_empty() && i > 0 { - return SpanVerdict::Prefixed(span.to_string()); - } - return SpanVerdict::Ok; - } - - // 5. 别的名字带着修饰，或一个名字都没有 - if others.iter().any(|k| { - let kw = span_words(k); - find_words(&s, &kw).is_some() - }) { - return SpanVerdict::Described(span.to_string()); - } - SpanVerdict::Coreference(span.to_string()) -} - -fn clause_suspect(name: &str) -> Option<&'static str> { - let words: Vec = name - .split_whitespace() - .map(|w| { - w.trim_matches(|c: char| !c.is_alphanumeric()) - .to_lowercase() - }) - .collect(); - // 限定词起头的长串：「The removal … industry as a whole」；机构名也会长，但 - // 通常不以 the 起头（US District Court …），起头的（The New York Times）不长 - if words.len() >= 8 && matches!(words[0].as_str(), "a" | "an" | "the") { - return Some("determiner_opens_a_long_string"); - } - // 句中的关系词 / 从属连词：「the committee that reviewed …」 - if words.len() >= 5 - && words - .iter() - .skip(1) - .any(|w| matches!(w.as_str(), "that" | "which" | "who" | "because" | "while")) - { - return Some("relative_or_subordinate_clause"); - } - None -} - -fn is_entity_name(name: &str) -> bool { - let name = name.trim(); - if name.is_empty() || name.chars().count() > 100 { - return false; - } - let raw: Vec<&str> = name.split_whitespace().collect(); - let words: Vec = raw - .iter() - .map(|w| { - w.trim_matches(|c: char| !c.is_alphanumeric()) - .to_lowercase() - }) - .collect(); - if words.len() > MAX_NAME_WORDS { - return false; - } - // 一个词的名字不可能是从句，别让 "Is" 这种专名被误伤 - if words.len() > 2 && words.iter().any(|w| CLAUSE_MARKERS.contains(&w.as_str())) { - return false; - } - if reads_like_a_sentence(&words, raw.last().copied()) { - return false; - } - // 部分格：`745 of OpenAI's 770 employees` 是一个数量描述，不是一个东西。 - // 它没有限定动词，词数也不多，上面两条都接不住它。 - // - // 判据收得很窄——**首词是纯数字且第二词是 of**。真实体里以数字开头的 - //（`3M`、`7-Eleven`、`23andMe`）首词不是纯数字；`2023 Nobel Prize` 首词是纯数字， - // 但第二个词不是 `of`。宽一格就会误伤它们。 - if words.len() >= 3 - && words[1] == "of" - && !words[0].is_empty() - && words[0].chars().all(|c| c.is_ascii_digit()) - { - return false; - } - /* **整体就是一个量的，不是一个东西**：`$5 billion`、`52%`、`3.5 million`。 - 上面那条部分格只接住 `745 of …`，接不住这些。 - - 这道闸装在**这里**才管用。抽取那边也有一道（`looks_literal`），但它前面 - 挂着 `!known_predicate`：谓词一旦是本体里列出来的关系，整段判断直接跳过。 - 实测就是这么漏的——`hasAmount` 来自 FIBO 包、抽取前就在本体里，于是 - `Microsoft hasAmount $1 billion` 里那个数额照样被造成了节点， - 而同一个库里 `invested`（语料自己长的、当时还未知）走到了那道闸、被拦下。 - `is_entity_name` 不问谓词，主语宾语一视同仁，两条路都过它。 - - 判据仍旧从严（见 `parse_quantity`）：尾巴上有实词就不算， - `3M`、`7-Eleven`、`23andMe`、`2025 Atlantic hurricane season` 一个都不误伤。 */ - if utopia_extract::parse_quantity(name).is_some() { - return false; - } - true -} - /// 记一条丢弃信号。抽取器有七处 `continue`，每一处都是"事实抽出来了、被挡掉、 /// 什么都不说"。信号写失败绝不能带垮整篇文档的抽取，所以这里吞掉错误。 pub(crate) async fn drop_signal( @@ -541,30 +163,6 @@ pub(crate) fn incomplete_reason(unextracted: &[(i32, String)], attempted: usize) )) } -/// 自动扩本体的唯一入队点。**成功与失败两条路都要走到它。** -/// -/// 开关在这里重读，而不是沿用调用方手上那份：失败路径压根没加载过 kb， -/// 而成功路径那份是文档**开抽时**读的——一篇 73 块的文档要跑一个多小时， -/// 期间有人在设置里关掉了开关，沿用旧值就是拿一小时前的意图办事。 -async fn enqueue_bootstrap(state: &AppState, kb_id: Uuid) -> anyhow::Result<()> { - if !utopia_store::kbs::get(&state.pool, kb_id) - .await? - .auto_extend_ontology - { - return Ok(()); - } - if !utopia_store::documents::extraction_idle(&state.pool, kb_id).await? { - return Ok(()); - } - utopia_store::jobs::enqueue( - &state.pool, - "bootstrap_ontology", - serde_json::json!({ "kb_id": kb_id }), - ) - .await?; - Ok(()) -} - /// `proposer`：这篇文档若是记忆日志，抽出的事实等人点头，据此记下「谁说的」 /// ——人，以及经 MCP 时那个 agent（0014 的令牌）。批量摄入的文档传默认值： /// 那条路不经待确认队列，这两位都用不上 @@ -585,60 +183,12 @@ pub async fn extract_document( .await; if let Ok(doc) = utopia_store::documents::get(&state.pool, document_id).await { state.emit_document(doc.kb_id, document_id); - // **失败也要触发自动扩本体。** - // - // 入队从前只写在成功路径上，于是这一串会把知识库永久卡住： - // 前 14 篇成功（每篇都看到还有别的在飞，不触发），第 15 篇重试 - // 耗尽变 failed —— 这时 extraction_idle 恰好为真（failed 不算 - // queued/extracting），可**再没有任何一篇文档会完成来做这次检查**。 - // 结果是提案堆在池子里、本体永远停在种子那几个关系、半张图永远是 - // 兜底谓词，而界面上没有任何东西说这件事发生过。 - // - // 任务本身幂等且会重查开关与门槛，所以这里多入队一次是安全的。 - let _ = enqueue_bootstrap(state, doc.kb_id).await; } Err(e) } } } -/// mention → 实体 id。同一文档内同名同类型直接复用（单文档语境里罕有同名不同人， -/// 也把消解调用摊薄到每个名字一次）；跨文档歧义由 resolve_mention 的画像比对处理。 -#[allow(clippy::too_many_arguments)] -async fn resolve( - pool: &PgPool, - kb_id: Uuid, - // None = 模型给的类型不在本体里，或这个库根本还没有类（0009） - type_id: Option, - name: &str, - ctx: Option<&[f32]>, - // 本次提及所在分块的原文：画像分不开同名候选时的事实旁证（#331） - text: Option<&str>, - doc_cache: &mut HashMap<(Option, String), Uuid>, - needs_adjudication: &mut bool, -) -> anyhow::Result { - let key = ( - type_id, - utopia_store::resolution::normalize_name(name).to_lowercase(), - ); - if let Some(id) = doc_cache.get(&key) { - return Ok(*id); - } - let id = resolve_uncached( - pool, - kb_id, - type_id, - name, - ctx, - text, - &[], - needs_adjudication, - ) - .await?; - doc_cache.insert(key, id); - Ok(id) -} - /// Resolve without the document's name cache. Handles use this path so two identities claimed /// separately in one response cannot collapse before their fact refs are bound. #[allow(clippy::too_many_arguments)] @@ -699,78 +249,6 @@ async fn create_namesake_reviews( Ok(created) } -#[derive(Clone, Copy)] -struct BoundEntity { - id: Uuid, - type_id: Option, -} - -/// 模型写的名字和它 ref 指的实体对不对得上（#582）。没有 ref、或写的就是那个名字时 -/// 无事；否则把写的名字当片段核对，写的名字自己当引文（免掉在不在引文那一关）。 -/// 只有描述和改绑两种结论会用到，其余当无事 -fn written_verdict( - written: &str, - bound: &str, - other: &str, - referenced: bool, - declared: &HashMap, -) -> SpanVerdict { - if !referenced || slot_matches(written, bound) { - return SpanVerdict::Ok; - } - match verify_span(Some(written), bound, other, written, declared) { - v @ (SpanVerdict::Described(_) | SpanVerdict::Rebind(_)) => v, - _ => SpanVerdict::Ok, - } -} - -/// 一侧绑上的名字：有 ref 就是 ref 指的那个实体声明的名字，没有就是模型写的（#582） -fn bound_name<'a>( - ref_names: &'a HashMap, - reference: Option<&str>, - written: &'a str, -) -> &'a str { - reference - .map(str::trim) - .and_then(|h| ref_names.get(h)) - .map(String::as_str) - .unwrap_or(written) -} - -fn referenced_entity( - ref_entities: &HashMap, - reference: &str, -) -> Option { - ref_entities.get(reference.trim()).copied() -} - -#[derive(Debug, PartialEq, Eq)] -enum NoRefNameBinding { - Legacy(Uuid), - AmbiguousHandled, - Missing, -} - -fn no_ref_name_binding( - entity_ids: &HashMap, - handled_by_name: &HashMap>, - name: &str, -) -> NoRefNameBinding { - let normalized = utopia_store::resolution::normalize_name(name).to_lowercase(); - if handled_by_name - .get(&normalized) - .is_some_and(|ids| ids.len() > 1) - { - NoRefNameBinding::AmbiguousHandled - } else { - entity_ids - .get(name) - .copied() - .map(NoRefNameBinding::Legacy) - .unwrap_or(NoRefNameBinding::Missing) - } -} - #[allow(clippy::too_many_arguments)] pub(crate) async fn resolve_handle( pool: &PgPool, @@ -836,2614 +314,37 @@ pub(crate) async fn resolve_handle( .or_default() .push(id); let document_claims = handled_by_name.entry(normalized).or_default(); - if !document_claims.contains(&id) { - document_claims.push(id); - } - Ok(id) -} - -#[allow(clippy::too_many_arguments)] -async fn resolve_bare( - pool: &PgPool, - kb_id: Uuid, - type_id: Option, - name: &str, - ctx: Option<&[f32]>, - text: Option<&str>, - doc_cache: &mut HashMap<(Option, String), Uuid>, - handled_by_name: &HashMap>, - ambiguous_bare_cache: &mut HashMap, - needs_adjudication: &mut bool, - human_reviews_found: &mut bool, -) -> anyhow::Result { - let normalized = utopia_store::resolution::normalize_name(name).to_lowercase(); - let claims = handled_by_name - .get(&normalized) - .filter(|ids| ids.len() > 1) - .cloned(); - let Some(claims) = claims else { - return resolve( - pool, - kb_id, - type_id, - name, - ctx, - text, - doc_cache, - needs_adjudication, - ) - .await; - }; - if let Some(id) = ambiguous_bare_cache.get(&normalized) { - return Ok(*id); - } - - // The text supplies no evidence for choosing among the handled namesakes. Preserve the - // fact on one document-scoped provisional entity and expose every possible identity link - // to a person. Subsequent bare mentions in this document reuse this provisional entity. - let id = resolve_uncached( - pool, - kb_id, - type_id, - name, - ctx, - text, - &claims, - needs_adjudication, - ) - .await?; - if create_namesake_reviews(pool, kb_id, id, &claims).await? { - *human_reviews_found = true; - } - ambiguous_bare_cache.insert(normalized, id); - Ok(id) -} - -async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> { - let doc = utopia_store::documents::get(&state.pool, document_id).await?; - // 排队之后被删了（#268）：墓碑不抽——抽出来的事实会活在一个已删除的出处上 - if doc.deleted_at.is_some() { - tracing::info!(document = %document_id, "skipping a deleted document"); - return Ok(()); - } - // **本体向量门控（#526）——把等待从 worker 里搬回队列。** - // - // 在这之前 `extraction::run` 直接调 `ontology_index::refresh` 等到补齐才动。 - // 那次等待有三个坏处：它把 worker 槽占住，让同一批次的其余文档和别的库的 - // 任务全卡在锁上；它让文档在这段时间里挂着 `extracting`，用户看见 32 篇 - // 全在「抽取中」却没一个事实落库；它用一份可能没补齐的索引作依据，抽出来 - // 的图是基于半个本体写的，再也不会被重抽。 - // - // 换成队列内门控：本体超出提示词预算且需要嵌入时，先把 `embed_ontology` - // 排上、把这次抽取挂回 `queued` 等 30s，让 worker 槽立刻空出来——同一批 - // 其余文档和其它库的任务都能继续认领。下次轮到这个文档时本体可能已就绪， - // 也可能还没，那就再等一轮。**不是同一个文档在等，是同一个抽取器在等**， - // 而等候归队列管，attempts 不烧。 - // - // 没配嵌入模型的库不等，照旧送完整本体——那种部署本来就没有检索。 - // 等也有期限（`jobs::DEFER_WINDOW_SECS`），补齐任务一直失败时这篇按失败处理。 - let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?; - if !kb.open_extraction && ontology_index::gate_required(state, doc.kb_id).await? { - // gate_required 已经把 `embed_ontology` 入队过了（如果应该入队的话）。 - // 这里只挂等待时长，不重复 enqueue。attempts 会被 mark_failed 退回去—— - // 同一个等待条件两次排队不应该消耗两次预算。 - let err = anyhow::Error::msg("waiting for ontology index to be embedded") - .context(utopia_core::Deferred::new(Duration::from_secs(30))); - return Err(err); - } - let settings = utopia_store::settings::get(&state.pool, kb.workspace_id) - .await? - .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?; - let client = llm_util::chat_client(&settings) - .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?; - - // 所有权凭证：重抽会自增 epoch，本任务据此察觉自己已被接管（见分块循环） - let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?; - utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?; - state.emit_document(doc.kb_id, document_id); - // **开放图谱**（0044 第 1 刀，#729）：开关开着的库只写文档自己的话，本体不进提示词。 - // 记忆日志也走这条路，只是它的陈述先进待确认表等人点头（0015），点头时才落成开放陈述 - if kb.open_extraction { - let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?; - return crate::extraction_open::run_open( - state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod, - ) - .await; - } - let etypes = utopia_store::graph::entity_types(&state.pool, doc.kb_id).await?; - // 这一轮落过的事实（新建或重复观察）：结尾对它们跑一遍签名检查 - let mut touched_facts: Vec = Vec::new(); - let mut rtypes = utopia_store::graph::relation_types(&state.pool, doc.kb_id).await?; - // 名字属性不进给模型的清单（0041）：名字走回复里的 `names`，服务端核对它在原文里 - rtypes.retain(|r| !utopia_store::names::is_name_attribute(r)); - // 关系与属性分道：属性走字面值通道，不进关系清单。 - // - // **本体里没有对应关系时就没有谓词**（见 `facts.predicate_id`）。原词落进 - // fact_evidence.proposed_predicate，显示时由 fact_surface_predicate() 取回。 - // - // 从前这里是一个叫 related_to 的兜底关系，且刻意不列给模型——它摆进提示词就成了 - // 逃生舱，模型读到说不清的关系时不去写原文说法，直接挑这个万能选项。 - // 现在它连行都没有了，逃生舱和"记得别列它"这两件事一起消失。 - // - // **这两件事必须一起做，缺一件比都不做更糟。** 只删库里的行不够，还要删 - // `DEFAULT_RELATION_TYPES` 里的种子——而这里的排除过滤已经跟着删了。于是 - // `ensure_default_ontology` 七分钟后把行种回来，`related_to` 第一次被**列进 - // 提示词给模型看**。0001 量过：359 次使用里 321 次是模型从清单上挑的。 - // 谁要往种子表里加回一个兜底关系，先看 0010——不过那张表现在已经没有了 - // （`#128`），连播种函数一起退场。 - let type_key_by_id: HashMap = - etypes.iter().map(|t| (t.id, t.key.as_str())).collect(); - let attr_meta: HashMap<&str, &utopia_core::models::RelationType> = rtypes - .iter() - .filter(|r| r.kind == "attribute") - .map(|r| (r.key.as_str(), r)) - .collect(); - /* **边上的属性**（0037）：一条关系声明过的属性定义，按关系 id 取。 - 属性定义就是 kind='attribute' 的行，datatype / unit / 换算全复用； - 写入时按这里的 key 对模型给的 qualifiers，对不上的进丢弃表让人看见 */ - let rtype_by_id: HashMap = - rtypes.iter().map(|r| (r.id, r)).collect(); - let attr_by_key: HashMap = rtypes - .iter() - .filter(|r| r.kind == "attribute") - .map(|r| (r.key.to_lowercase(), r)) - .collect(); - let mut qualifier_defs: HashMap> = rtypes - .iter() - .filter(|r| r.kind != "attribute" && !r.qualifiers.is_empty()) - .map(|r| { - let defs = r - .qualifiers - .iter() - .filter_map(|q| rtype_by_id.get(q).copied()) - .filter(|q| q.kind == "attribute") - .collect(); - (r.id, defs) - }) - .collect(); - let type_ids: HashMap<&str, Uuid> = etypes.iter().map(|t| (t.key.as_str(), t.id)).collect(); - let rel_ids: HashMap<&str, Uuid> = rtypes.iter().map(|r| (r.key.as_str(), r.id)).collect(); - // 模型说出的谓词往本体已有关系上落：写法、时态、被动都对齐（见 predicate_match）。 - // 没有它的时候，`produces` 明明在词表里，模型写 `produced_by` 就被降级扔了 - let pred_index = PredicateIndex::build(&rtypes); - // 「本体认不认识这个说法」——字面值那一档与关系那一档必须用同一个判据。 - // 分开写的话，模糊匹配得上的谓词会先被字面值那一档当成属性分流走， - // 同一个词在两条路上得到相反的回答 - let known_predicate = |p: &str| rel_ids.contains_key(p) || pred_index.lookup(p).is_some(); - // 时态对账只对带唯一性约束的状态关系生效（本体元数据）：(functional, inverse_functional, temporal) - let rel_meta: HashMap = rtypes - .iter() - .map(|r| { - ( - r.id, - (r.functional, r.inverse_functional, r.temporal.clone()), - ) - }) - .collect(); - let type_parents: HashMap = etypes - .iter() - .map(|t| (t.id, t.parents.as_slice())) - .collect(); - - // **本体全铺还是按分块检索。** - // - // 全铺是今天的行为，小本体下它对且便宜：40 个类约 2k 字符，检索反而是多余的 - // 往返。大本体下它是灾难——schema.org 实测每个分块 108k tokens，而同一份语料 - // 只给种子类时抽到 25 个实体、给全量时只剩 18 个。**多给的那 959 个类 - // 吃掉了 7 个实体。** - // - // 所以按预算切换：装得下就全铺，装不下就每块检索。判据量的是**实际要排的 - // 那段字**（build_lists 自己数），不是另写一个估算公式——公式会跟排版分叉。 - let full = build_lists(&etypes, &rtypes, None, None); - let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?; - let retrieve_per_chunk = full.chars() > budget; - if retrieve_per_chunk { - tracing::info!( - %document_id, chars = full.chars(), budget, - classes = etypes.len(), - "本体超出提示词预算，改为按分块检索候选" - ); - // 走到这里说明本体超出预算且按预算逻辑需要按块检索——但本任务的等待 - // 早就在 `gate_required` 里完成（要么已经嵌好，要么通过 `Deferred` 挂回 - // 队列）。剩下的就是按块检索本身，不再有「顺便 refresh」这一步： - // 那一步是在 worker 里等 PER_KB 锁，正是 #526 想消除的副作用。 - // - // 留一个注释方便日后回看：若 budget 在 `gate_required` 与此处之间被改小， - // 本文档会按全量本体抽——比 #526 之前的「等几分钟」更接近「对的那一边」。 - } - // 内置类恒在：检索漏掉的分块仍要有地方落脚，否则模型无类可选 - let seed_classes: HashSet = etypes.iter().filter(|t| t.builtin).map(|t| t.id).collect(); - - // 属性 domain 允许子类：主语类型沿 parent 链上溯命中 domain 即可 - // 沿 subClassOf 上溯。**广度优先 + 访问集**，不是单链循环： - // 一个类可以有多个父（FOAF 的 Person 同时是 Agent 与 SpatialThing）， - // 而菱形继承会从两条路到达同一个祖先，没有访问集就会重复展开。 - // - // 深度上限换成了访问集：写入侧 set_parents 已经查环，这里再靠"最多走十层" - // 兜底既挡不住宽的图，也会悄悄放过深的层级。 - let type_matches_domain = |ty: Uuid, domain: Uuid| -> bool { - let mut seen: HashSet = HashSet::new(); - let mut queue = vec![ty]; - while let Some(cur) = queue.pop() { - if cur == domain { - return true; - } - if !seen.insert(cur) { - continue; - } - if let Some(ps) = type_parents.get(&cur) { - queue.extend(ps.iter().copied()); - } - } - false - }; - // 本轮要从头讲一遍这篇文档的故事，旧信号先清掉（重抽自动作数） - let _ = utopia_store::extraction_drops::clear_for_document(&state.pool, document_id).await; - - // **记忆抽出的事实先等人点头**（0015）。一句 remember 一次一句、人就在对话里， - // 确认成本最低的时刻就是说完那句话的时候；而批量摄入一次上万条，逐条确认 - // 不可能，那条路仍旧乐观写入 + 事后审阅。判据只有一个：这篇是不是记忆日志。 - // 实体照常消解并创建——`pending_facts.subject_id` 是外键，这是 0018 定下的取舍 - let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?; - let mut pending_count = 0usize; - - let doc_time = doc.doc_time.map(|t| t.format("%Y-%m-%d").to_string()); - let chunks = utopia_store::documents::chunks_for_extraction(&state.pool, document_id).await?; - // 文件开头：序号最小的现存分块（不是还没抽的第一块）。备忘文件一个片段一块， - // 前一段不是后一段的开头，不附 - let opening_chunk = if await_nod { - None - } else { - utopia_store::documents::opening_chunk(&state.pool, document_id).await? - }; - - let mut doc_cache: HashMap<(Option, String), Uuid> = HashMap::new(); - // Identities introduced through handles, grouped only for detecting document-local - // namesake ambiguity. A later bare mention gets its own provisional entity instead of - // guessing among this group. - let mut handled_by_name: HashMap> = HashMap::new(); - let mut ambiguous_bare_cache: HashMap = HashMap::new(); - let mut touched_names: HashSet = HashSet::new(); - // 本文档已经认下的实体，按首次出现排序，送进后续分块的提示词。 - // - // **按 entity_id 去重，不按名字**：第 3 块写"上海研究院"若消解到了第 1 块的 - // "星云科技上海研究院"，那它不该以第二个名字进清单——清单里每个实体只有 - // 一个展示形态，就是这篇文档第一次用的那个。中文里全称先出现，所以这也是较全的那个。 - let mut doc_entities: Vec<(Uuid, String, String)> = Vec::new(); - // 整块没抽成的：(seq, 原因)。收尾时据此拒绝把这篇文档标成 done - let mut unextracted: Vec<(i32, String)> = Vec::new(); - let mut needs_adjudication = false; - let mut human_reviews_found = false; - let mut conflicts_found = false; - let mut fact_count = 0usize; - // 不设分块上限：静默截断等于丢知识，长文档的成本由部署者自己权衡 - // （成本优化走 prompt 前缀缓存与更新时 chunk 级跳过，而非丢数据） - for chunk in chunks.iter() { - // 被接管则安静退场：不写 failed、不碰状态，舞台留给新任务。 - // 检查放在调用 LLM 之前——取消粒度即一个分块，不必等整篇跑完 - if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch { - tracing::info!(%document_id, "抽取任务已被新一轮接管，退出"); - return Ok(()); - } - let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice()); - // 本体装得下就用全量那份；装不下就拿**这一块自己的向量**检索候选。 - // 向量是现成的——实体消解本来就在用它（上面那个 ctx），检索一次 - // 嵌入都不用加。检索不出来（没配嵌入模型、或这块没向量）就退回全量： - // 提示词大是慢，没有类可选是抽不出东西 - let lists = if retrieve_per_chunk { - match ctx { - Some(v) => { - chunk_lists(state, doc.kb_id, v, &etypes, &rtypes, &seed_classes, budget) - .await - .unwrap_or(None) - } - None => None, - } - } else { - None - }; - let lists = lists.as_ref().unwrap_or(&full); - let known: Vec = doc_entities - .iter() - .enumerate() - .map(|(index, (_, type_key, name))| utopia_extract::KnownEntity { - handle: format!("k{}", index + 1), - type_key: type_key.clone(), - name: name.clone(), - }) - .collect(); - // 这一块就是开头本身时不再重复一遍 - let opening = opening_chunk - .as_ref() - .filter(|(id, _)| *id != chunk.id) - .map(|(_, text)| text.as_str()); - let messages = utopia_extract::build_messages_with_opening( - &lists.types, - &lists.relations, - &lists.attributes, - doc_time.as_deref(), - &doc.filename, - &known, - opening, - &chunk.text, - ); - // 这两处 continue 跳过的是**整个分块**——它一条事实都没产出。 - // 记下来，收尾时据此决定这篇文档算不算抽完（见循环之后） - let reply = match chat_retrying_rate_limits(state, &settings, &client, &messages).await { - Ok(r) => r, - Err(e) => { - tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取调用失败，跳过该分块"); - unextracted.push((chunk.seq, format!("调用失败：{e}"))); - continue; - } - }; - // 模型的原话只在 debug 级别看得到：查它对哪几个字段怎么填（#582 的片段）时开 - tracing::debug!(%document_id, seq = chunk.seq, reply = %reply, "抽取原始回复"); - let mut extraction = match utopia_extract::parse_response(&reply) { - Ok(x) => x, - Err(e) => { - tracing::warn!(%document_id, seq = chunk.seq, error = %e, "抽取结果解析失败，跳过该分块"); - unextracted.push((chunk.seq, format!("结果解析失败：{e}"))); - continue; - } - }; - // **跳过了什么必须说出来。** 逐项解析救回了整块，但被跳过的那几条 - // 如果不落信号，就成了另一种「部分抽取报告成完成」（#108 修过一次） - if extraction.truncated { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::TRUNCATED_REPLY, - &format!("分块 #{} 的输出被截断", chunk.seq), - None, - ) - .await; - } - let skipped = extraction.skipped_entities + extraction.skipped_facts; - if skipped > 0 { - tracing::warn!( - %document_id, - seq = chunk.seq, - entities = extraction.skipped_entities, - facts = extraction.skipped_facts, - "跳过了结构不合的条目" - ); - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - &format!( - "分块 #{} 跳过 {} 个实体 / {} 条事实", - chunk.seq, extraction.skipped_entities, extraction.skipped_facts - ), - None, - ) - .await; - } - - // 实体消解：名称 → 实体 id（本分块的事实按原文名字连线） - // **落库前先查形状**（utopia_extract::normalize）：只看结构、不看词——引文里有没有 - // 这段字、值是不是只有标点、一侧是不是契约的日期、同句有没有另一条边。读懂时间 - // 归模型（提示词 3c），这里只核对它照没照契约写，做了什么都记进丢弃表 - let from_opening = match opening { - Some(text) => { - utopia_extract::drop_quotes_from_opening(&mut extraction, &chunk.text, text) - } - None => Vec::new(), - }; - for n in from_opening - .into_iter() - .chain(utopia_extract::normalize_facts(&mut extraction)) - { - use utopia_extract::Normalization as N; - use utopia_store::extraction_drops::reason; - let (r, detail, example) = match n { - N::NoValue { predicate, written } => (reason::NO_VALUE, predicate, written), - N::ValueTrimmed { - predicate, - kept, - dropped, - } => ( - reason::VALUE_TRIMMED, - predicate, - format!("{kept} ✂ {dropped}"), - ), - N::QualifiersWithoutObject { predicate, values } => ( - reason::QUALIFIERS_WITHOUT_OBJECT, - predicate, - format!("{values} value(s) moved onto the subject"), - ), - N::TimeAsObject { - predicate, - written, - values, - } => ( - reason::TIME_AS_OBJECT, - predicate, - if values == 0 { - format!("{written} kept as a value") - } else { - format!("{written} → {values} value(s)") - }, - ), - N::TimeAsSubject { predicate, written } => { - (reason::TIME_AS_SUBJECT, predicate, written) - } - N::ObjectDescribesDeclared { - predicate, - name, - head, - } => ( - reason::OBJECT_DESCRIBES_DECLARED, - predicate, - format!("{name} ← {head}"), - ), - N::OrphanDeclaration { name } => { - (reason::ORPHAN_DECLARATION, "entity".to_string(), name) - } - N::QuoteFromOpening { predicate, quote } => { - (reason::QUOTE_FROM_OPENING, predicate, quote) - } - }; - drop_signal(state, doc.kb_id, document_id, r, &detail, Some(&example)).await; - } - let mut entity_ids: HashMap = HashMap::new(); - // 名称 → 声明类型（属性 domain 校验用：salary 不能挂在 Organization 上） - let mut entity_type_of: HashMap> = HashMap::new(); - let mut ref_entities: HashMap = doc_entities - .iter() - .enumerate() - .map(|(index, (id, type_key, _))| { - ( - format!("k{}", index + 1), - BoundEntity { - id: *id, - type_id: type_ids.get(type_key.as_str()).copied(), - }, - ) - }) - .collect(); - // 句柄 → 声明的名字：片段核对要对着**绑上的**那个名字看（#582）。模型写 - // "OpenAI personnel" 却 ref 到 e1=OpenAI 时，错在 ref 上，片段对着 "OpenAI" - // 才看得出来 - let mut ref_names: HashMap = doc_entities - .iter() - .enumerate() - .map(|(index, (_, _, name))| (format!("k{}", index + 1), name.clone())) - .collect(); - let mut response_claims: HashMap> = HashMap::new(); - - // Resolve handled definitions first. This matters for a mixed response: a later - // handle-less same-name item must see the ambiguity rather than winning by array order. - for e in extraction - .entities - .iter() - .filter(|e| e.local_id.is_some()) - .chain(extraction.entities.iter().filter(|e| e.local_id.is_none())) - { - let name = e.name.trim(); - if !is_entity_name(name) { - // 从前这里是静默 `continue`——正是 `drop_signal` 当初为之而建的 - // 那种"抽出来了、被挡掉、什么都不说" - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME, - &e.type_key, - Some(name), - ) - .await; - continue; - } - // 守卫放行、结构却像从句：只记不挡（#193 先攒标注集） - if let Some(signal) = clause_suspect(name) { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::CLAUSE_SUSPECT, - signal, - Some(name), - ) - .await; - } - if let Some(handle) = e.local_id.as_deref().map(str::trim) { - if ref_entities.contains_key(handle) { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - "local_id collides with a known handle", - Some(handle), - ) - .await; - continue; - } - } - // 降级时记住模型提议的那个词：本体装不下不等于它说错了。 - // 只留计数的话，日后想加 model 类就找不出那 43 个实体——它们混在 - // concept 里面，唯一的出路是整库重抽 - let mut proposed: Option<&str> = None; - let type_id = match type_ids.get(e.type_key.as_str()) { - Some(id) => Some(*id), - None => { - // 白名单外类型：**留空**，并记入未匹配统计（本体扩展的信号）。 - // - // 从前这里降级到 concept 那行哨兵。现在「还没判出来」就是 - // `type_id IS NULL`（0009）——实体照常建、事实照常落、证据照常有， - // 只是暂时没有类型。之后装一个包再跑类型消解，它会被重新分配 - let _ = utopia_store::ontology::record_miss( - &state.pool, - doc.kb_id, - "entity_type", - &e.type_key, - Some(name), - ) - .await; - proposed = Some(e.type_key.as_str()); - None - } - }; - let normalized = utopia_store::resolution::normalize_name(name).to_lowercase(); - touched_names.insert(normalized.clone()); - let id = if let Some(handle) = e.local_id.as_deref() { - let handle = handle.trim(); - let id = resolve_handle( - &state.pool, - doc.kb_id, - type_id, - name, - ctx, - Some(&chunk.text), - &mut response_claims, - &mut handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await?; - ref_entities.insert(handle.to_string(), BoundEntity { id, type_id }); - ref_names.insert(handle.to_string(), name.to_string()); - id - } else { - resolve_bare( - &state.pool, - doc.kb_id, - type_id, - name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await? - }; - if let Some(p) = proposed { - let _ = utopia_store::resolution::set_proposed_type(&state.pool, id, p).await; - } - // 模型写下的这个名字就在这一块原文里时，给这条名字事实补出处（0041）。 - // 给已知句柄时它照提示词写的是清单上的全称，这一块里未必有——那就不补， - // 这一块用的别的写法走下面的 `names`。 - // 记忆日志里的不补：那一句算不算出处，要等人点头（0018） - if !await_nod && span_in_quote(name, &chunk.text) { - let _ = utopia_store::names::record( - &state.pool, - doc.kb_id, - id, - name, - Some(utopia_store::names::NameSource { - chunk_id: chunk.id, - quote: name, - }), - doc.doc_time, - ) - .await; - } - // 模型自己的说法。**跟 proposed_type 分开存**：那一列的含义是 - // "本体里没有"，增长回路靠它的稀有性设门槛；这一列每个实体都有。 - // 与粗类同名的不记——那不是更具体的说法，只是把清单抄了一遍 - if let Some(st) = e - .specific_type - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - .filter(|s| !s.eq_ignore_ascii_case(&e.type_key)) - { - let _ = utopia_store::resolution::set_specific_type(&state.pool, id, st).await; - } - // 只记模型自己声明过类型的：主宾兜底那条路没有类型可依， - // 把一个猜出来的类型放进清单等于让后续分块照着猜的抄。 - // - // 本体装不下那个类时用模型自己的说法（proposed）：这份清单是给后文 - // 认人用的，"同一个名字别写成两个实体"才是它的活。从前这里只能写死 - // concept，反倒把几个不同的词抹平成同一个标签 - if !doc_entities.iter().any(|(eid, _, _)| *eid == id) { - let tk = type_id - .and_then(|t| type_key_by_id.get(&t).copied()) - .or(proposed) - .unwrap_or("?"); - doc_entities.push((id, tk.to_string(), name.to_string())); - } - // Legacy name maps retain their old last-write-wins behavior. Handled facts bind - // through ref_entities; inserting handled definitions here only supports mixed - // model output when the surface name is unambiguous. - entity_ids.insert(name.to_string(), id); - entity_type_of.insert(name.to_string(), type_id); - } - - // 实体的别的名字（0041 决定 2）。**名字与引文都要在这一块原文里**：名字是召回的桥， - // 一座凭空造的桥会把两个不相干的实体接到一起。认不认「简称」「又名」是模型的事， - // 服务端不认词，只核对它抄的字是不是真在原文里 - // 这次回复声明的名字，加上本文档前面几块认下的：别名撞上它们之一就不收 - let declared_names: HashMap = entity_ids - .iter() - .map(|(name, id)| (name.as_str(), *id)) - .chain( - doc_entities - .iter() - .map(|(id, _, name)| (name.as_str(), *id)), - ) - .map(|(name, id)| { - ( - utopia_store::resolution::normalize_name(name).to_lowercase(), - id, - ) - }) - .collect(); - for n in &extraction.names { - let name = n.name.trim(); - let Some(bound) = ref_entities.get(n.entity_ref.trim()) else { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - "name ref is not a declared handle", - Some(name), - ) - .await; - continue; - }; - let quote = n - .quote - .as_deref() - .map(str::trim) - .filter(|q| !q.is_empty()) - .unwrap_or(name); - if name_claimed_elsewhere(name, bound.id, &declared_names) { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::NAME_CLAIMED_BY_ANOTHER, - &n.entity_ref, - Some(name), - ) - .await; - continue; - } - if !is_entity_name(name) - || !span_in_quote(name, quote) - || !span_in_quote(quote, &chunk.text) - { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::NAME_NOT_IN_TEXT, - &n.entity_ref, - Some(name), - ) - .await; - continue; - } - // 记忆日志读到的名字与它的其它事实一样等人点头（0018）：名字是召回的桥， - // 一句没确认过的话不该先把桥搭上。点头之后是一条普通的名字事实； - // 这一步不配对，同名的配对等下一次在文档里读到它 - if await_nod { - let known_as = utopia_store::names::ensure_known_as(&state.pool, doc.kb_id).await?; - let value = serde_json::json!({ - "value": utopia_store::resolution::normalize_name(name) - }); - if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose( - &state.pool, - utopia_store::pending::Proposal { - kb_id: doc.kb_id, - subject_id: bound.id, - predicate_id: Some(known_as), - object_id: None, - object_value: Some(&value), - proposed_predicate: Some(utopia_store::names::KNOWN_AS), - validity: utopia_store::graph::Validity { - attested_at: doc.doc_time, - ..Default::default() - }, - confidence: 1.0, - chunk_id: chunk.id, - proposed_by: proposer.user_id, - proposed_token: proposer.token_id, - phrase: None, - qualifiers: None, - time_words: None, - quote_span: None, - }, - ) - .await? - { - pending_count += 1; - } - continue; - } - utopia_store::names::record( - &state.pool, - doc.kb_id, - bound.id, - name, - Some(utopia_store::names::NameSource { - chunk_id: chunk.id, - quote, - }), - doc.doc_time, - ) - .await?; - // 别的实体已经叫这个名字：送去裁决，不合并 - if utopia_store::names::pair_shared_name(&state.pool, doc.kb_id, bound.id, name).await? - > 0 - { - needs_adjudication = true; - } - } - - // 改绑的候选：这次回复声明的实体，加上提示词里给过的库内实体（#582） - let span_declared: HashMap = { - let mut m = entity_ids.clone(); - for (id, _, name) in &doc_entities { - m.entry(name.clone()).or_insert(*id); - } - m - }; - for f in &extraction.facts { - let confidence = f.confidence.unwrap_or(0.7).clamp(0.0, 1.0); - if confidence < MIN_CONFIDENCE { - // 设计上的阈值，但用户同样无从知道"抽到了，只是不够自信" - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::LOW_CONFIDENCE, - &f.predicate, - Some(&format!("{} ({:.0}%)", f.subject, confidence * 100.0)), - ) - .await; - continue; - } - let confidence = origin_ceiling(&chunk.origin, confidence); - let validity = - validity_of(f.valid_from.as_deref(), f.valid_to.as_deref(), doc.doc_time); - - // 属性事实：谓词命中属性 → 字面值通道。datatype 校验失败宁缺勿脏； - // domain 校验（含子类上溯）挡住"把 salary 挂到 Organization"这类张冠李戴。 - // 模型偶尔照抄清单里的 "person.salary" 全限定名——剥掉类前缀再查一次 - let attr_hit = attr_meta.get(f.predicate.as_str()).or_else(|| { - f.predicate - .rsplit_once('.') - .and_then(|(_, k)| attr_meta.get(k)) - }); - if let Some(attr) = attr_hit { - let subject_name = f.subject.trim(); - // 主语没在 entities 里声明：类型不明，domain 无从校验，属性不落。 - // 关系路径遇到同样的缺失会兜底按 concept 消解——这里学不来， - // 按 concept 解出来 domain 照样不匹配，只是从这里掉进下面那一档 - let bound = match f.subject_ref.as_deref().map(str::trim) { - Some(handle) => referenced_entity(&ref_entities, handle), - None => { - match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) { - NoRefNameBinding::Legacy(id) => entity_type_of - .get(subject_name) - .copied() - .map(|type_id| BoundEntity { id, type_id }), - NoRefNameBinding::Missing => None, - NoRefNameBinding::AmbiguousHandled => { - let type_id = entity_type_of.get(subject_name).copied().flatten(); - touched_names.insert( - utopia_store::resolution::normalize_name(subject_name) - .to_lowercase(), - ); - let id = resolve_bare( - &state.pool, - doc.kb_id, - type_id, - subject_name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await?; - Some(BoundEntity { id, type_id }) - } - } - } - }; - let Some(BoundEntity { - id: subject_id, - type_id: subject_type, - }) = bound - else { - drop_signal( - state, - doc.kb_id, - document_id, - if f.subject_ref.is_some() { - utopia_store::extraction_drops::reason::MALFORMED_ITEM - } else { - utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED - }, - &attr.key, - Some(f.subject_ref.as_deref().unwrap_or(subject_name)), - ) - .await; - continue; - }; - // 不可达：store 层强制 attribute 必有 domain，且 domain 不可改， - // 无 domain 的属性连提示词都进不去。留着是防御，不需要信号 - if attr.domains.is_empty() { - continue; - } - // **任一 domain 命中即可**：属性挂在多个类下时，主语属于其中之一就算数 - if !attr - .domains - .iter() - .any(|d| subject_type.is_some_and(|t| type_matches_domain(t, *d))) - { - let subj_key = subject_type - .and_then(|t| type_key_by_id.get(&t).copied()) - .unwrap_or("?"); - let dom_key = attr - .domains - .iter() - .filter_map(|d| type_key_by_id.get(d).copied()) - .collect::>() - .join("|"); - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::ATTR_DOMAIN_MISMATCH, - &format!("{}@{subj_key} (wants {dom_key})", attr.key), - Some(subject_name), - ) - .await; - continue; - } - let raw = match (&f.value, &f.object) { - (Some(v), _) => v.clone(), - // 模型偶尔把值放进 object：宽容接住 - (None, Some(o)) if !o.trim().is_empty() => { - serde_json::Value::String(o.trim().to_string()) - } - _ => { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::ATTR_NO_VALUE, - &attr.key, - Some(subject_name), - ) - .await; - continue; - } - }; - let datatype = attr.datatype.as_deref().unwrap_or("text"); - // 只相对一件事给出的日期（「触发日后 45 天」，#681 §4）照原文收下、带着标记： - // 它是新的状态值，时态引擎照常用它接替前一个截止日 - let Some(mut object_value) = - utopia_extract::attr_object_value(datatype, &raw, f.relative) - else { - tracing::debug!(%document_id, attr = attr.key, ?raw, "属性值不合 datatype，跳过"); - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::ATTR_DATATYPE, - &format!("{} ({datatype})", attr.key), - Some(&format!("{subject_name} → {raw}")), - ) - .await; - continue; - }; - // 单位随事实落笔：类型上的单位以后改了，旧值仍按记录时的单位读。 - // 记哪个单位照 `unit_for`——从前这里无条件盖上声明的单位，实测 - //「提供 500 兆瓦的风电」被模型记成金额，再盖上 ¥ 就成了 500 块钱 - if let Some(u) = unit_for(&raw, datatype, None, attr.unit.as_deref()) { - object_value["unit"] = serde_json::json!(u); - } - if await_nod { - if let utopia_store::pending::Outcome::Proposed(_) = - utopia_store::pending::propose( - &state.pool, - utopia_store::pending::Proposal { - kb_id: doc.kb_id, - subject_id, - predicate_id: Some(attr.id), - object_id: None, - object_value: Some(&object_value), - proposed_predicate: Some(f.predicate.as_str()), - validity, - confidence, - chunk_id: chunk.id, - proposed_by: proposer.user_id, - proposed_token: proposer.token_id, - phrase: None, - qualifiers: None, - time_words: None, - quote_span: None, - }, - ) - .await? - { - pending_count += 1; - } - continue; - } - let (fact_id, created) = utopia_store::graph::insert_value_fact( - &state.pool, - doc.kb_id, - subject_id, - Some(attr.id), - &object_value, - validity, - confidence, - ) - .await?; - touched_facts.push(fact_id); - // 属性谓词也留原词：模型偶尔照抄 "person.salary" 全限定名， - // 命中的是剥掉前缀后的 key，原样是什么值得留着 - utopia_store::graph::add_evidence( - &state.pool, - fact_id, - chunk.id, - f.quote.as_deref(), - Some(f.predicate.as_str()), - ) - .await?; - if created { - fact_count += 1; - } - // 单值属性 = functional：新值闭合旧值（属性历史由此而来）。并进已有断言的也对： - // 这份证据的日期可能更早，时间线的形状跟着变（#679） - if attr.functional && attr.temporal == "state" { - let report = utopia_store::temporal::reconcile_new_fact( - &state.pool, - doc.kb_id, - fact_id, - subject_id, - attr.id, - None, - Some(&object_value), - utopia_store::temporal::Uniqueness::SubjectSide, - validity, - confidence, - ) - .await?; - if report.conflicts > 0 { - conflicts_found = true; - } - } - continue; - } - - // **词表外的字面值：既不丢，也不给它编一个实体。** - // - // 走到这里说明谓词既不是已知属性也还没查关系表。它带着字面值时 - // 有两种走法，从前两种都不好： - // value 有而 object 空 → 掉进下面的"宾语必填"，整条静默消失； - // object 里塞着字面值 → 按 concept 消解，凭空造出一个叫「2015」 - // 的实体，图里多一个假节点，事后再修还得改事实的形状。 - // 现在都存成 object_value 且没有谓词：值在图里、有证据、有时态， - // 原词进 proposed_predicate，消解那一遍只需换谓词，形状已经是对的。 - // - // object 里的东西算不算字面值，判据从严：**模型没把它声明成实体**， - // 且**它整体就是一个量或一个日期**。"杭州"两条都不满足，"2015"、 - // "$5 billion"、"52%" 都满足；"900 million weekly active users" - // 不满足——尾巴上还有实词，它说的就不再只是那个数了。 - // 文本值的属性（schema.org 里 323 个）在这一档仍会变成实体—— - // 那里没有可靠判据，猜错会吃掉真实体，不猜 - // 第二格是表层谓词：通常就是模型写的谓词；值旁边挂着一个没声明的宾语短语时， - // 短语并进来（见下面那一档） - let literal: Option<(serde_json::Value, String)> = - match (&f.value, f.object.as_deref().map(str::trim)) { - // **给了值、没给宾语——不管这个谓词本体认不认识。** - // - // 从前这里卡着 `!known_predicate`：`job_title` 在 schema.org 里是关系 - //（它的 range 是 `Text|DefinedTerm`，含一个类就走关系通道），于是模型 - // 写 `job_title` + "founder and CEO" 时既进不了属性档、又在关系档因为 - // 缺宾语被丢掉——`object_missing` 实测 69 次，四篇文档里每个人的职务 - // 就是这么没的。谓词认不认识与「这条事实带的是值还是实体」无关： - // 值在手上就收下，原词进 proposed_predicate，等本体采纳时再换谓词， - // 形状已经是对的（0010） - (Some(v), None | Some("")) => Some((v.clone(), f.predicate.clone())), - /* **宾语整体是一个量：一律当值收下**，不问谓词认不认识、 - 也不问模型有没有把它声明成实体。 - - 下面那一档卡着 `!known_predicate`，理由是本体说得上话的时候 - 别去二猜模型。可量值这里没有可猜的余地：一个数额不会因为 - 谓词恰好在本体里就变成一个东西。实测漏的正是这一格—— - `hasAmount` 来自 FIBO 包、抽取前就在本体里， - `Microsoft hasAmount $1 billion` 于是绕过下面那一档， - 把数额造成了节点；同一个库里 `invested` 当时还未知， - 走到下面那一档、被拦住了。同一个数额，两种下场。 - - 收下而不是丢掉：`is_entity_name` 那道闸现在也拦纯量值， - 不在这里接住的话，这条事实会连同那个数一起进丢弃表。 - 原词照旧进 `proposed_predicate`，采纳时再换谓词 */ - (_, Some(o)) if utopia_extract::parse_quantity(o).is_some() => Some(( - serde_json::Value::String(o.to_string()), - f.predicate.clone(), - )), - /* **给了值、宾语却是一个没声明的短语：值收下，短语并进表层谓词**（#685）。 - - `build` + 宾语「new energy generation」+ 值「at least 10 GW」：宾语不是 - 回复里声明的实体、没有句柄、也不是本文档认下的名字，于是上面几档都接 - 不住，关系那条路又因为它不是实体把整条丢掉——那个数跟着没了。模型把 - 同一句话写成纯值（「at least 10 GW of new energy generation」）时就落得 - 下，落不落全看它挑了两种同样说得通的写法里的哪一种。 - - 只看结构：值在、宾语没有句柄、宾语不在已声明的名字里。宾语是个认得的 - 实体时照旧走边。短语不丢，进表层谓词（`build new energy generation`）， - 采纳时再换谓词；原句在证据里 */ - (Some(v), Some(o)) - if undeclared_beside_value(f.object_ref.as_deref(), o, &span_declared) => - { - Some((v.clone(), format!("{} {o}", f.predicate.trim()))) - } - (_, Some(o)) - if !o.is_empty() - && !known_predicate(f.predicate.as_str()) - && !entity_ids.contains_key(o) - && looks_literal(o) => - { - Some(( - serde_json::Value::String(o.to_string()), - f.predicate.clone(), - )) - } - _ => None, - }; - if let Some((value, surface)) = literal { - let subject_name = f.subject.trim(); - // **主语按关系那条路解，不要求它在本次回复里重新声明过。** - // - // 从前这里只查 `entity_ids`，模型用「已知实体」句柄带进来的、 - // 或者只写了名字没重列的主语一律落空——实测一轮 51 块里 - // `subject_not_declared` 丢掉 113 条，丢的是 NVIDIA 的营收、 - // 净利、每股收益，是十位董事的赞成票与反对票，全是有名有姓的 - // 主语。属性那一档要求主语有类型（domain 要校验），这一档没有 - // domain 可校验，也就没有理由比关系那条路更严 - let subject_id = match f.subject_ref.as_deref().map(str::trim) { - Some(handle) => match referenced_entity(&ref_entities, handle) { - Some(bound) => bound.id, - None => { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - &f.predicate, - Some(handle), - ) - .await; - continue; - } - }, - None => { - match no_ref_name_binding(&entity_ids, &handled_by_name, subject_name) { - NoRefNameBinding::Legacy(id) => id, - NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { - touched_names.insert( - utopia_store::resolution::normalize_name(subject_name) - .to_lowercase(), - ); - resolve_bare( - &state.pool, - doc.kb_id, - entity_type_of.get(subject_name).copied().flatten(), - subject_name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await? - } - } - } - }; - let _ = utopia_store::ontology::record_miss( - &state.pool, - doc.kb_id, - "attribute_type", - &surface, - Some(&format!("{subject_name} → {value}")), - ) - .await; - /* 值照原文落笔（提示词 8a 要的就是「units and all」），**单位另记一格**。 - 采纳成属性时按 datatype 把 `$5 billion` 换算成 5e9，那一步只看得懂 - 数；符号丢在原文里就再也取不出来了，而「5000000000」少了那个 `$` - 就不知道是钱还是别的什么 */ - let unit = value - .as_str() - .and_then(utopia_extract::parse_quantity) - .and_then(|(_, u)| u); - let mut literal = serde_json::json!({ "value": value }); - if let Some(unit) = unit { - literal["unit"] = serde_json::Value::String(unit); - } - let literal = literal; - if await_nod { - if let utopia_store::pending::Outcome::Proposed(_) = - utopia_store::pending::propose( - &state.pool, - utopia_store::pending::Proposal { - kb_id: doc.kb_id, - subject_id, - predicate_id: None, - object_id: None, - object_value: Some(&literal), - proposed_predicate: Some(surface.as_str()), - validity, - confidence, - chunk_id: chunk.id, - proposed_by: proposer.user_id, - proposed_token: proposer.token_id, - phrase: None, - qualifiers: None, - time_words: None, - quote_span: None, - }, - ) - .await? - { - pending_count += 1; - } - continue; - } - let (fact_id, created) = utopia_store::graph::insert_value_fact( - &state.pool, - doc.kb_id, - subject_id, - None, - &literal, - validity, - confidence, - ) - .await?; - touched_facts.push(fact_id); - utopia_store::graph::add_evidence( - &state.pool, - fact_id, - chunk.id, - f.quote.as_deref(), - Some(surface.as_str()), - ) - .await?; - if created { - fact_count += 1; - } - continue; - } - - // 关系事实：宾语必填 - let Some(object_name) = f.object.as_deref().map(str::trim).filter(|s| !s.is_empty()) - else { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::OBJECT_MISSING, - &f.predicate, - Some(f.subject.trim()), - ) - .await; - continue; - }; - // **未声明的主宾也要过同一道判据。** - // - // 从前守卫只装在上面那条声明实体的路上，而这里绕过了它：模型把一整句话 - // 写进 `object`、那句话没出现在 entities 里，这里就转头把它造成了实体。 - // 实测（ai-timeline-ends × schema.org）421 个实体里 76 个无类型， - // 最长的那个 111 字符——"thermal-imaging equipment used by volunteers - // flying over the site showed at least 33 generators giving off heat"， - // 那是一整个从句，不是一个东西。**守卫拦住了前门，后门是开的。** - // - // 这类实体的害处不止于此：它们永远匹配不到别处的任何提及， - // 在图上是孤点（实测 59 个），还会拖累消解——每一个都要跟已有实体比一遍。 - if !is_entity_name(f.subject.trim()) || !is_entity_name(object_name) { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::NOT_AN_ENTITY_NAME, - &f.predicate, - Some(if is_entity_name(f.subject.trim()) { - object_name - } else { - f.subject.trim() - }), - ) - .await; - continue; - } - for side in [f.subject.trim(), object_name] { - if let Some(signal) = clause_suspect(side) { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::CLAUSE_SUSPECT, - signal, - Some(side), - ) - .await; - } - } - - // **槽位片段核对**（#582，取代 #578 的词表）：模型交出引文里点名每一侧的那几个 - // 字，机器核对（`verify_span`）。描述做主语不落、做宾语落成字面值；点了别的 - // 实体就改绑；其余都只记不拦 - let quote_text = f.quote.as_deref().unwrap_or(""); - // 对着绑上的名字看：有 ref 就是 ref 指的那个实体的名字，没有就是模型写的名字 - let bound_subject = bound_name(&ref_names, f.subject_ref.as_deref(), f.subject.trim()); - let bound_object = bound_name(&ref_names, f.object_ref.as_deref(), object_name); - // 片段说了算；片段没说清（放行、不在引文、前缀、指代）时，再看模型写的名字 - // 跟它 ref 指的实体对不对得上 - let decisive = |v: &SpanVerdict| { - matches!( - v, - SpanVerdict::Described(_) | SpanVerdict::Rebind(_) | SpanVerdict::Misplaced(_) - ) - }; - let mut subject_verdict = verify_span( - f.subject_span.as_deref(), - bound_subject, - bound_object, - quote_text, - &span_declared, - ); - if !decisive(&subject_verdict) { - let w = written_verdict( - f.subject.trim(), - bound_subject, - bound_object, - f.subject_ref.is_some(), - &span_declared, - ); - if decisive(&w) { - subject_verdict = w; - } - } - let mut object_verdict = verify_span( - f.object_span.as_deref(), - bound_object, - bound_subject, - quote_text, - &span_declared, - ); - if !decisive(&object_verdict) { - let w = written_verdict( - object_name, - bound_object, - bound_subject, - f.object_ref.is_some(), - &span_declared, - ); - if decisive(&w) { - object_verdict = w; - } - } - let mut rebound_subject: Option = None; - let mut rebound_object: Option = None; - let mut object_described: Option = None; - let mut subject_described = false; - for (side, verdict, span, bound) in [ - ( - "subject", - &subject_verdict, - f.subject_span.as_deref(), - bound_subject, - ), - ( - "object", - &object_verdict, - f.object_span.as_deref(), - bound_object, - ), - ] { - let (reason, example) = match verdict { - SpanVerdict::Ok => continue, - SpanVerdict::NotInQuote => ( - utopia_store::extraction_drops::reason::SPAN_NOT_IN_QUOTE, - format!("{} ({bound})", span.unwrap_or("")), - ), - SpanVerdict::Rebind(name) => { - if side == "subject" { - rebound_subject = Some(name.clone()); - } else { - rebound_object = Some(name.clone()); - } - ( - utopia_store::extraction_drops::reason::SPAN_REBOUND, - format!("{} → {name} (was {bound})", span.unwrap_or("")), - ) - } - SpanVerdict::Described(text) => { - if side == "subject" { - subject_described = true; - ( - utopia_store::extraction_drops::reason::SUBJECT_DESCRIBED, - format!("{text} ({bound})"), - ) - } else { - object_described = Some(text.clone()); - ( - utopia_store::extraction_drops::reason::OBJECT_DESCRIBED, - format!("{text} ({bound})"), - ) - } - } - SpanVerdict::Prefixed(text) => ( - utopia_store::extraction_drops::reason::SPAN_PREFIXED, - format!("{text} ({bound})"), - ), - SpanVerdict::Coreference(text) => ( - utopia_store::extraction_drops::reason::SPAN_COREFERENCE, - format!("{text} ({bound})"), - ), - SpanVerdict::Misplaced(text) => ( - utopia_store::extraction_drops::reason::SPAN_MISPLACED, - format!("{text} ({bound})"), - ), - }; - drop_signal( - state, - doc.kb_id, - document_id, - reason, - &f.predicate, - Some(&example), - ) - .await; - } - if subject_described { - continue; - } - let subject_name: &str = rebound_subject.as_deref().unwrap_or(f.subject.trim()); - let subject_ref_eff: Option<&str> = if rebound_subject.is_some() { - None - } else { - f.subject_ref.as_deref().map(str::trim) - }; - let object_name: &str = rebound_object.as_deref().unwrap_or(object_name); - let object_ref_eff: Option<&str> = - if rebound_object.is_some() || object_described.is_some() { - None - } else { - f.object_ref.as_deref().map(str::trim) - }; - // 主宾没在 entities 里声明时（模型偶尔漏报）：库里已经有这个名字的就用它， - // 库里也没有的**不建**（#559）。从前这里一律先建出来、类型留空，结果 - // 一个库里 20% 的实体是 "lawsuit against OpenAI"、"$5 billion"、"March 2024" - // 这样的描述——几乎全部只当过宾语，从没当过主语。漏报的实体多半在 - // 前面的分块或别的文档里已经声明过，按名字找得到；找不到的就是描述。 - // 描述做主语的事实丢掉并记账，做宾语的事实照落，宾语落成字面值 - let subject_id = match subject_ref_eff { - Some(handle) => match referenced_entity(&ref_entities, handle) { - Some(bound) => bound.id, - None => { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - &f.predicate, - Some(handle), - ) - .await; - continue; - } - }, - None => { - let binding = no_ref_name_binding(&entity_ids, &handled_by_name, subject_name); - if matches!(binding, NoRefNameBinding::Missing) - && utopia_store::resolution::existing_by_name( - &state.pool, - doc.kb_id, - subject_name, - ) - .await? - .is_none() - { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED, - &f.predicate, - Some(subject_name), - ) - .await; - continue; - } - match binding { - NoRefNameBinding::Legacy(id) => id, - NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { - touched_names.insert( - utopia_store::resolution::normalize_name(subject_name) - .to_lowercase(), - ); - resolve_bare( - &state.pool, - doc.kb_id, - None, - subject_name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await? - } - } - } - }; - let object_id = match object_ref_eff { - Some(handle) => match referenced_entity(&ref_entities, handle) { - Some(bound) => bound.id, - None => { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::MALFORMED_ITEM, - &f.predicate, - Some(handle), - ) - .await; - continue; - } - }, - None => { - let binding = no_ref_name_binding(&entity_ids, &handled_by_name, object_name); - if object_described.is_some() - || (matches!(binding, NoRefNameBinding::Missing) - && utopia_store::resolution::existing_by_name( - &state.pool, - doc.kb_id, - object_name, - ) - .await? - .is_none()) - { - // 没声明、库里也没有，或者片段说它是个描述：宾语落成字面值。谓词照旧对本体； - // 被动形（`_by`）本该主宾对调，而字面值当不了主语，那条就不给谓词， - // 原词留在证据上 - let predicate_id = match pred_index.lookup(f.predicate.as_str()) { - Some((id, false)) => Some(id), - Some((_, true)) => None, - None => { - let _ = utopia_store::ontology::record_miss( - &state.pool, - doc.kb_id, - "relation_type", - &f.predicate, - Some(&format!("{} → {}", f.subject, object_name)), - ) - .await; - None - } - }; - let literal_text: &str = object_described.as_deref().unwrap_or(object_name); - // 描述那一路在上面核对时已经记过账；这里只记「没声明」的 - if object_described.is_none() { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::OBJECT_UNDECLARED, - &f.predicate, - Some(object_name), - ) - .await; - } - let literal = serde_json::json!({ "value": literal_text }); - if await_nod { - if let utopia_store::pending::Outcome::Proposed(_) = - utopia_store::pending::propose( - &state.pool, - utopia_store::pending::Proposal { - kb_id: doc.kb_id, - subject_id, - predicate_id, - object_id: None, - object_value: Some(&literal), - proposed_predicate: Some(f.predicate.as_str()), - validity, - confidence, - chunk_id: chunk.id, - proposed_by: proposer.user_id, - proposed_token: proposer.token_id, - phrase: None, - qualifiers: None, - time_words: None, - quote_span: None, - }, - ) - .await? - { - pending_count += 1; - } - continue; - } - let (fact_id, created) = utopia_store::graph::insert_value_fact( - &state.pool, - doc.kb_id, - subject_id, - predicate_id, - &literal, - validity, - confidence, - ) - .await?; - touched_facts.push(fact_id); - utopia_store::graph::add_evidence( - &state.pool, - fact_id, - chunk.id, - f.quote.as_deref(), - Some(f.predicate.as_str()), - ) - .await?; - if created { - fact_count += 1; - } - continue; - } - match binding { - NoRefNameBinding::Legacy(id) => id, - NoRefNameBinding::AmbiguousHandled | NoRefNameBinding::Missing => { - touched_names.insert( - utopia_store::resolution::normalize_name(object_name) - .to_lowercase(), - ); - resolve_bare( - &state.pool, - doc.kb_id, - None, - object_name, - ctx, - Some(&chunk.text), - &mut doc_cache, - &handled_by_name, - &mut ambiguous_bare_cache, - &mut needs_adjudication, - &mut human_reviews_found, - ) - .await? - } - } - } - }; - if subject_id == object_id { - continue; - } - // 先尽量落到本体已有的关系上（写法/时态/被动），**落不上才降级**为 related_to - // 并记入未匹配统计。降级会把原意抹平成"有关联"——原词写进证据行的 - // proposed_predicate，是这条事实身上唯一还留着原意的地方（谓词消解据此映射回本体） - let (predicate_id, swap) = match pred_index.lookup(f.predicate.as_str()) { - Some((id, swap)) => (Some(id), swap), - None => { - let _ = utopia_store::ontology::record_miss( - &state.pool, - doc.kb_id, - "relation_type", - &f.predicate, - Some(&format!("{} → {}", f.subject, object_name)), - ) - .await; - // 本体里没有对应的关系 → **就是没有谓词**（见 `facts.predicate_id`）。 - // 原意留在证据的 proposed_predicate 里，显示时取回。 - // 从前这里落到 related_to 上，还要额外担心"兜底关系被删了"—— - // 那条失败模式连同它的 continue 一起消失了 - (None, false) - } - }; - // 被动说法命中的是同一条边的反向：`ChatGPT produced_by OpenAI` 与 - // `OpenAI produces ChatGPT` 是同一条边，存的时候要按本体的方向来， - // 否则它跟已有的那 130 条 produces 各存各的，图上是两条相反的箭头 - let (subject_id, object_id) = if swap { - (object_id, subject_id) - } else { - (subject_id, object_id) - }; - - // **主语违反 domain、而宾语符合时，按本体声明的方向把它掰正。** - // - // 先试过提示词，三轮都没赢：违反率从 57% 压到 35%，但压下去的全是 - // 类型判错那一半；**真·反向纹丝不动**（22.7% → 17.1% → 17.6%， - // 后两个在噪声里）。模型看得见 `employee (organization → person)`， - // 就是不照做——英语的 "X is an employee of Y" 太强。 - // - // 这不是新原则：`produced_by` 命中 `produces` 时（见上面那个 `swap`） - // 早就在自动翻转主宾了，区别只在触发条件是**措辞**还是**签名**。 - // - // 当初反对自动对调的理由是实体类型不可靠——实测 Elon Musk 被判成 - // `researcher`。那个前提已经不成立：祖先地板与签名类恒在修好之后， - // 同一批人判成了 `person`。而判据本身很窄——**主语违反且宾语符合**， - // 两侧都要对上才动。 - // - // **但绝不静默。** 掰正要留信号：0001 反对的是「用可能错的声明驱动 - // 自动动作」，而看得见、可复查、可反悔的动作不属于那一类。 - let (predicate_id, subject_id, object_id) = if let Some(pid) = predicate_id { - // 类型**从库里的实体读**，不用抽取器手上那份 `entity_type_of`： - // 那份只覆盖模型在这一块里声明过的实体，而宾语常是别处已存在的实体， - // 这一块没重新声明它，于是查不到、判不了、掰不动。实测差别不小—— - // 用声明那份时反向只降到 7.0%，剩下的正是宾语类型查不到的那些 - // - // 判断本身在 store（`ontology::judge_direction`），**与采纳共用**（#190）： - // 写谓词的路不止这一条，守卫只装在一条上就等于没装。查不出来（库错） - // 按没有判据处理，照原样落——宁可少掰一条，不能因为一次查询失败丢事实 - let fit = utopia_store::ontology::judge_direction( - &state.pool, - pid, - subject_id, - object_id, - ) - .await - .unwrap_or(utopia_store::ontology::Fit::Unchecked); - match fit { - utopia_store::ontology::Fit::Swap => { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::DIRECTION_CORRECTED, - &f.predicate, - Some(&format!( - "{} → {} 按签名掰正为 {} → {}", - f.subject, - f.object.as_deref().unwrap_or("?"), - f.object.as_deref().unwrap_or("?"), - f.subject - )), - ) - .await; - (Some(pid), object_id, subject_id) - } - utopia_store::ontology::Fit::Neither => { - // **对调也不合法 → 退回没有谓词。** - // - // 这不是方向问题，是这个关系压根不适用：schema.org 的 - // `affectedBy` 是医学检验用的，模型要表达「受……影响」时按名字 - // 撞了上来；`amount` 属于融资工具而不是公司，模型没造那个中间 - // 节点就把边挂到了公司上。 - // - // 从前照原样落库，等于**用本体的名义说一件本体不同意的事**—— - // 图上写着 "OpenAI affectedBy …"，读者会以为那是一条医学断言。 - // 这是自信的错误，比空谓词严重得多。 - // - // 退回空谓词不丢信息：原词落进 `fact_evidence.proposed_predicate`， - // 显示时由 `fact_surface_predicate()` 取回（0010）。主宾、时间、 - // 证据全都留着，只是不再冒认一个本体关系。**诚实的沉默。** - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::DOMAIN_MISMATCH, - &f.predicate, - Some(&format!( - "{} — {} → 主宾都对不上，退回原文说法", - f.subject, f.predicate - )), - ) - .await; - (None, subject_id, object_id) - } - utopia_store::ontology::Fit::Keep | utopia_store::ontology::Fit::Unchecked => { - (Some(pid), subject_id, object_id) - } - } - } else { - (predicate_id, subject_id, object_id) - }; - - if await_nod { - if let utopia_store::pending::Outcome::Proposed(_) = utopia_store::pending::propose( - &state.pool, - utopia_store::pending::Proposal { - kb_id: doc.kb_id, - subject_id, - predicate_id, - object_id: Some(object_id), - object_value: None, - proposed_predicate: Some(f.predicate.as_str()), - validity, - confidence, - chunk_id: chunk.id, - proposed_by: proposer.user_id, - proposed_token: proposer.token_id, - phrase: None, - qualifiers: None, - time_words: None, - quote_span: None, - }, - ) - .await? - { - pending_count += 1; - } - continue; - } - { - let (fact_id, created) = utopia_store::graph::insert_fact( - &state.pool, - doc.kb_id, - subject_id, - predicate_id, - object_id, - validity, - confidence, - ) - .await?; - touched_facts.push(fact_id); - /* **边上的属性落笔**（0037）。属性不进去重键：`insert_fact` 复用了旧行也照写—— - 同一条边再听到一次带了金额的，是同一条边补上金额。 - 值照 datatype 换算，单位另记一格（与 object_value 同形）； - key 不在声明里、换不动、与已记的不一致——三种都进丢弃表，不静默 */ - /* 谓词未知（0010，说法在证据上）时属性照写：值落在事实上，声明等 - 关系被采纳时在 `adopt` 里补。不写的话，`invested_in` 在 schema.org - 库里是未知说法，一句话里的 $1.5 billion 就没有地方放——召回台上 - `oh-invest` 那一条正是这么丢的 */ - if let Some(quals) = f.qualifiers.as_ref() { - let pid = predicate_id; - // 克隆出这一组引用：下面撞上已有属性时要往 qualifier_defs 里追加声明 - let defs: Vec<&utopia_core::models::RelationType> = pid - .and_then(|p| qualifier_defs.get(&p).cloned()) - .unwrap_or_default(); - /* 模型常把币种单独写成一个键（`"amount": "1500000000", "currency": "CNY"`）， - 而不是写进数额里。那不是一个属性，是数额的单位——先把它拿出来， - 数值属性解不出单位时用它，别让它作为未知 key 进丢弃表 */ - let sibling_currency: Option<&'static str> = quals - .iter() - .find(|(k, _)| { - matches!( - k.trim().to_lowercase().as_str(), - "currency" | "币种" | "货币" | "unit" | "单位" - ) - }) - .and_then(|(_, v)| v.as_str()) - .and_then(utopia_extract::currency_unit); - for (key, raw) in quals { - // 模型对没提到的属性会写 null：那是「原文没说」，不是坏值，不记 - if raw.is_null() { - continue; - } - if matches!( - key.trim().to_lowercase().as_str(), - "currency" | "币种" | "货币" | "unit" | "单位" - ) { - continue; - } - let declared = defs - .iter() - .find(|q| q.key.eq_ignore_ascii_case(key.trim())) - .copied(); - /* **未知 key 撞上本库已有的属性定义 → 补一条声明，不丢值。** - 实测不声明时模型照样写 `amount`、`stake`、`round`，八条全进 - 丢弃表——而这三个属性定义明明都在库里，缺的只是关系上的一条 - 声明。补声明不新建任何东西、可撤（本体页取消勾选即可）， - 所以跟自动扩本体走同一个开关 */ - let adopted = match declared { - Some(d) => Some(d), - /* 谓词还未知（0010，说法在证据上）：没有关系可声明，值绑到库里 - 已有的属性定义、先落在事实上，采纳时 `adopt` 再补声明。这一步 - 不动本体，所以不看自动扩本体的开关 */ - None if pid.is_none() => { - attr_by_key.get(&key.trim().to_lowercase()).copied() - } - None if kb.auto_extend_ontology => { - match attr_by_key.get(&key.trim().to_lowercase()).copied() { - Some(attr) => { - let pid = pid.expect("checked above"); - match utopia_store::ontology::add_relation_qualifier( - &state.pool, - doc.kb_id, - pid, - attr.id, - ) - .await - { - Ok(()) => { - tracing::info!(kb_id = %doc.kb_id, relation = %f.predicate, qualifier = %attr.key, "边上的属性按语料补了声明"); - qualifier_defs.entry(pid).or_default().push(attr); - Some(attr) - } - Err(e) => { - tracing::warn!(kb_id = %doc.kb_id, error = %e, "补声明失败"); - None - } - } - } - None => { - tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, attrs = attr_by_key.len(), "边上的属性：key 撞不上本库任何属性定义"); - None - } - } - } - None => { - tracing::debug!(kb_id = %doc.kb_id, relation = %f.predicate, key = %key, auto_extend = kb.auto_extend_ontology, "边上的属性：未声明且不自动扩本体"); - None - } - }; - let Some(def) = adopted else { - /* **绑不上属性定义的数也不丢。** 库里没有这个属性（schema.org 里 - `amount` 是关系不是属性）、或本体冻着不让扩——从前这里进丢弃表， - 数就只剩丢弃表里的一个样例。现在照 8a 的样子落成主语上的一条 - 字面值事实：值按原文、单位另记一格、原词 `关系.键` 进证据的 - proposed_predicate，缺的定义记进 ontology_misses（0010 的样子）， - 采纳时人来决定它归哪儿。图里有它、有证据、能查到 */ - let wording = format!("{}.{}", f.predicate.trim(), key.trim()); - let unit = raw - .as_str() - .and_then(utopia_extract::parse_leading_quantity) - .and_then(|(_, u)| u) - .or_else(|| sibling_currency.map(str::to_string)); - let mut literal = serde_json::json!({ "value": raw }); - if let Some(u) = unit { - literal["unit"] = serde_json::Value::String(u); - } - let _ = utopia_store::ontology::record_miss( - &state.pool, - doc.kb_id, - "attribute_type", - &wording, - Some(&format!("{} → {raw}", f.subject.trim())), - ) - .await; - let (literal_id, _) = utopia_store::graph::insert_value_fact( - &state.pool, - doc.kb_id, - subject_id, - None, - &literal, - validity, - confidence, - ) - .await?; - touched_facts.push(literal_id); - utopia_store::graph::add_evidence( - &state.pool, - literal_id, - chunk.id, - f.quote.as_deref(), - Some(wording.as_str()), - ) - .await?; - tracing::debug!(kb_id = %doc.kb_id, wording = %wording, "边上的属性绑不上定义，落成主语上的字面值"); - continue; - }; - let dt = def.datatype.as_deref().unwrap_or("text"); - let Some(normalized) = utopia_extract::normalize_attr_value(dt, raw) else { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::QUALIFIER_DATATYPE, - &format!("{}.{} ({dt})", f.predicate, def.key), - Some(&raw.to_string()), - ) - .await; - continue; - }; - let mut value = serde_json::json!({ "value": normalized }); - if let Some(u) = unit_for(raw, dt, sibling_currency, def.unit.as_deref()) { - value["unit"] = serde_json::Value::String(u); - } - let write = utopia_store::graph::upsert_fact_qualifier( - &state.pool, - fact_id, - def.id, - &value, - ) - .await?; - if write == utopia_store::graph::QualifierWrite::Conflict { - drop_signal( - state, - doc.kb_id, - document_id, - utopia_store::extraction_drops::reason::QUALIFIER_CONFLICT, - &format!("{}.{}", f.predicate, def.key), - Some(&raw.to_string()), - ) - .await; - } - } - } - // 重复观察也要挂证据：多来源相互印证，任一来源删除后事实不孤儿化。 - // 表层谓词随每次观察落笔——甲块说 "runs on"、乙块说 "optimized for" - // 会并进同一条事实，放事实上就是先写者胜，放证据上两个都留着 - utopia_store::graph::add_evidence( - &state.pool, - fact_id, - chunk.id, - f.quote.as_deref(), - Some(f.predicate.as_str()), - ) - .await?; - if created { - fact_count += 1; - } - // 时态对账：带唯一性约束的状态关系落新事实即检测矛盾（纯规则点查， - // 自动闭合走"作废+改写"，拿不准进 fact_conflicts 人裁）。并进已有断言的 - // 也对：多了一份证据，时间线的形状可能跟着变（#679） - // 没有谓词就没有关系元数据，也就不参与时态对账—— - // 一条说不出是什么关系的边，本来就不可能带唯一性约束 - if let Some((pid, (func, inv_func, temporal))) = - predicate_id.and_then(|id| rel_meta.get(&id).map(|m| (id, m))) - { - if temporal == "state" { - let mut directions = Vec::new(); - if *func { - directions.push(utopia_store::temporal::Uniqueness::SubjectSide); - } - if *inv_func { - directions.push(utopia_store::temporal::Uniqueness::ObjectSide); - } - for dir in directions { - let report = utopia_store::temporal::reconcile_new_fact( - &state.pool, - doc.kb_id, - fact_id, - subject_id, - pid, - Some(object_id), - None, - dir, - validity, - confidence, - ) - .await?; - if report.conflicts > 0 { - conflicts_found = true; - } - } - } - } - } - } - - // 本块抽取完成即打标：更新时被认领的块携带标记跳过；中断的抽取可续跑 - // （LLM 调用/解析失败的块在上方 continue 掉，不打标，下次重试） - utopia_store::documents::mark_chunk_extracted(&state.pool, chunk.id).await?; - } - // 队列里多了东西才叫醒人：Review 的计数与对话里那张确认卡都靠这一声 - if pending_count > 0 { - tracing::info!(%document_id, pending_count, "记忆抽出的事实进了待确认队列"); - state.emit_pending(doc.kb_id); - state.emit_review(doc.kb_id); - } - - // 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新 - touched_names.extend(doc_cache.keys().map(|(_, name)| name.clone())); - for name in &touched_names { - utopia_store::resolution::refresh_disambiguators(&state.pool, doc.kb_id, name).await?; - } - - // 出口再验一次：接管可能发生在最后一个分块之后，那时循环里的检查已经跑完。 - // 漏掉这里，被顶替的任务会把 done 写在一篇 extracted_at 刚被清空的文档上—— - // 界面显示"已完成"，实则一条都没抽，要等新任务开跑才纠正回来。 - if utopia_store::documents::extract_epoch(&state.pool, document_id).await? != my_epoch { - tracing::info!(%document_id, "抽取任务已被新一轮接管，收尾时退出"); - return Ok(()); - } - - // **有分块没抽成就不许标 done。** - // - // 从前这里无条件写 done：一次网络抖动让六篇文档 60 块里只抽成 12 块， - // 六篇全部显示"抽取完成"，八成的内容没进图，而界面上没有任何东西说出来。 - // 失败只进了日志，而只进日志的错误等于没有错误。 - // - // 返回 Err 之后这条链是完整的：`extract_document` 落 graph_failed + 原因， - // 界面上那篇文档变成可点开看错误的 failed；任务按 30s×attempts² 退避重试， - // 而已抽成的分块带着 extracted_at 会被跳过——所以重试很便宜，网络恢复就自愈。 - // 重试耗尽才留在 failed，那时它说的是实话。 - // - // 判据是"全抽完"而不是某个比例：任何比例都是拍的，而这里本来就有一个 - // 不需要拍的判据——**每一块都成了才叫抽完**。 - if let Some(msg) = incomplete_reason(&unextracted, chunks.len()) { - return Err(anyhow::anyhow!(msg)); - } - // 刚落的事实立刻过一遍签名。写入时只掰方向（judge_direction）；掰不动的 - // ——两个方向都对不上、或宾语没类型判不了——从前要等人按 Review 里的 - // Run check 才露面，Axioms 一直是 0，图里却躺着反向事实（#222）。 - // 检查失败不影响抽取本身：事实已经在库里，下一次 Run check 仍然查得到 - if !touched_facts.is_empty() { - match utopia_store::reasoning::signature_breaks( - &state.pool, - doc.kb_id, - Some(&touched_facts), - ) - .await - { - Ok(broken) if !broken.is_empty() => { - match utopia_store::reasoning::record_signature_breaks( - &state.pool, - doc.kb_id, - &broken, - ) - .await - { - Ok(_) => state.emit_review(doc.kb_id), - Err(e) => { - tracing::warn!(%document_id, error = %e, "抽取后的签名违规没记进队列") - } - } - } - Ok(_) => {} - Err(e) => tracing::warn!(%document_id, error = %e, "抽取后的签名检查失败"), - } - } - utopia_store::documents::set_graph_status(&state.pool, document_id, "done").await?; - state.emit_document(doc.kb_id, document_id); - - // 灰区对进了审核队列 → 触发攒批裁决任务（独立后台跑，抽取本身到此已完成）。 - // 治理开着（0025）排的是治理任务：灰区对与直接转人工的对都从那条先进先出的 - // 队列走，先读台账再裁；同库已排着的不重复 - if kb.governance { - if needs_adjudication || human_reviews_found { - utopia_store::jobs::enqueue_unless_queued( - &state.pool, - "govern", - serde_json::json!({ "kb_id": doc.kb_id }), - ) - .await?; - } - } else if needs_adjudication { - // 同库已排着的不重复——与下面的 resolve_types 一样。一批文档同时抽完 - // 会各排一个，而它们读到的是同一批待裁项 - utopia_store::jobs::enqueue_unless_queued( - &state.pool, - "adjudicate_entities", - serde_json::json!({ "kb_id": doc.kb_id }), - ) - .await?; - } - if needs_adjudication || human_reviews_found || conflicts_found { - state.emit_review(doc.kb_id); - } - // 类型消解排队自动跑（0016 C2）：开关开着就排一个库级任务，同库已排着的不重复。 - // 任务自己只看引擎没看过的实体、只自动落地子树内精化的那一档 - if kb.auto_type_resolution { - utopia_store::jobs::enqueue_unless_queued( - &state.pool, - "resolve_types", - serde_json::json!({ "kb_id": doc.kb_id }), - ) - .await?; - } - // 自动扩本体：开关开着、且这一批都抽完了，由最后一篇触发。 - // 判据是显式开关而不是"本体有没有被碰过"——后者是从行为推断意图， - // 推错的后果很荒唐（在提案上点一次 Add 就永久关掉建议），而且一旦为假 - // 就永不再真，本体会冻结在第一批文档碰巧包含的词汇上。 - // 并发下可能入队两次，任务自己会重查开关与状态 - enqueue_bootstrap(state, doc.kb_id).await?; - - tracing::info!(%document_id, facts = fact_count, "图谱抽取完成"); - Ok(()) -} - -/// 一条事实同时给了值和宾语时，宾语是不是一个**没声明的短语**（#685）：没有句柄， -/// 也不是本次回复或本文档前面认下的名字（大小写不计）。是的话这条事实按值落， -/// 宾语短语并进表层谓词；不是的话宾语是个实体，照旧走边 -fn undeclared_beside_value( - object_ref: Option<&str>, - object: &str, - declared: &HashMap, -) -> bool { - let object = object.trim(); - !object.is_empty() - && object_ref.map(str::trim).is_none_or(str::is_empty) - && !declared - .keys() - .any(|name| name.trim().eq_ignore_ascii_case(object)) -} - -/// 宾语位上的这串东西，是不是一个字面值而不是实体的名字。 -/// -/// **只认数字与日期。** 这是个会吃掉真实体的判断，所以宁可漏认： -/// 漏了不过是维持今天的行为（造一个 concept 实体），认错了却是把一个 -/// 真实体降成一段文本，图里少一个节点。 -/// -/// "2015"、"2023-03"、"6" 认；"杭州"、"首席技术官"、"3M"、"V3" 不认。 -/// 调用方还额外要求模型**没有**把它声明成实体——两道门一起过才算数。 -/// 模型给的区间两端 → 落库的有效区间。 -/// -/// **两端各记各的粒度**（见 `facts.valid_to_precision`）。从前一个精度列描述两个端点， -/// 于是「2020 年开始、2023-05-06 结束」这种只能共用一个值。两端都用 `read_time` 读： -/// 规则 3 的格式，或写法说得清是哪天的日期（#688），精度随写了几位。 -/// -/// 模型给的 valid_to = "unknown" 表示**原文说它结束了、但没说哪天**。 -/// read_time 解不出它（本来就不是日期），落在这里显式认掉—— -/// 不认的话它退化成 None，那条事实就又变回"仍在持续"了 -fn validity_of( - valid_from: Option<&str>, - valid_to: Option<&str>, - doc_time: Option>, -) -> utopia_store::graph::Validity<'static> { - let from = valid_from.and_then(utopia_extract::read_time); - let to = valid_to.and_then(utopia_extract::read_time); - let ended_unknown = valid_to - .map(str::trim) - .is_some_and(|v| v.eq_ignore_ascii_case(utopia_store::graph::ENDED_UNKNOWN)); - utopia_store::graph::Validity { - from: from.map(|(t, _)| t), - from_precision: from.map(|(_, p)| p), - to: to.map(|(t, _)| t), - to_precision: to - .map(|(_, p)| p) - .or(ended_unknown.then_some(utopia_store::graph::ENDED_UNKNOWN)), - // 这次观察出自哪一天的文档（0022）：没起点的事实从它起成立，结束了 - // 不知哪天的到它为止。没有文档日期就是记下的此刻——账本能给的最好的 - attested_at: doc_time, - } -} - -fn looks_literal(s: &str) -> bool { - let s = s.trim(); - if s.is_empty() { - return false; - } - /* **整体是一个量**：可选货币符号 + 数字 + 可选量级词 + 可选百分号， - 此外一个词都不许有（判据与例子见 `parse_quantity`）。 - - 从前这里只认裸数字（`s.parse::()`），于是 `$5 billion` 两头不着： - 它不是裸数字、也解不成日期，掉进关系那条路，凭空长出一个叫「$5 billion」 - 的节点。同名的又会并成一个点，于是 SSI Inc. 与 Nvidia 因为都出现过这个 - 数额而在图上相连——一条没有含义的路径。实测一个 1415 实体的库里，8 个 - 这样的点、15 条事实指着它们，而**没有任何一条拿它们当主语**： - 一个从不当主语、只当宾语、名字整体是个量的东西，是值，不是实体。 - - `parse_quantity` 已经把裸数字那一档包含在内（`"42"` → 42）， - 所以这里不必再单留一条。全角「２０１５」仍旧解不动，仍旧是想要的 */ - if utopia_extract::parse_quantity(s).is_some() { - return true; - } - // 日期：复用抽取侧那个解析器，它认 2015 / 2015-03 / 2015-03-01，也认写出来的日期（#688） - utopia_extract::read_time(s).is_some() -} - -/// 提示词里那三段清单：类、关系、属性。 -/// -/// **抽出来是为了让"全给"和"按分块检索"共用同一段排版逻辑。**两条路各排一份 -/// 的话迟早分叉，而分叉在这里的后果是提示词说的与代码认的不是一回事。 -struct PromptLists { - types: Vec<(String, String, String)>, - relations: Vec, - attributes: Vec, -} - -impl PromptLists { - /// 这三段铺进提示词有多长。budget 判据用它——**量的是实际要排的那段字**， - /// 不是另写一个估算公式（公式会跟排版分叉）。 - fn chars(&self) -> usize { - self.types - .iter() - .map(|(k, l, d)| k.len() + l.len() + d.len() + 6) - .sum::() - + self - .relations - .iter() - .map(|r| r.key.len() + r.label.len() + r.description.len() + r.signature.len() + 8) - .sum::() - + self.attributes.iter().map(|a| a.len() + 1).sum::() - } -} - -/// 把当前本体在提示词里的字符数算出来。**空铺**（不筛类/关系）——这就是 -/// `extract_document` 用的「全铺」档，也是判断「要不要按块检索」的标准。 -/// -/// 抽成独立函数是因为 `ontology_index::gate_required`（#526）要在加载抽取器 -/// 之前问一次预算——那时 `build_lists` 还没被调用。两个路径必须用同一个判据， -/// 否则 gate 的判定会和实际的「全铺」走分。 -pub(crate) fn full_ontology_chars( - etypes: &[utopia_core::models::EntityType], - rtypes: &[utopia_core::models::RelationType], -) -> usize { - build_lists(etypes, rtypes, None, None).chars() -} - -/// 从一个**选择集**排出三段清单。`None` = 全给（本体小于预算时的老路）。 -/// -/// 三处细节都是选择带来的，全给时它们不会触发： -/// -/// 1. **签名只能提到选中的类**。`works_at (person → organization)` 里那两个 key -/// 必须是模型看得见的——写一个没铺出去的类名，等于教它输出一个不存在的类型。 -/// 整侧都没选中就退回 `*`。 -/// 2. **属性跟着 domain 走**。属性行是 `class.attr`，它的类没铺出去这行就没意义。 -/// 这也顺带解决了属性段（占提示词 28%）的裁剪，不用单独处理。 -/// 3. **内置类恒在**。检索漏掉的分块仍然要有地方落脚，否则模型无类可选。 -fn build_lists( - etypes: &[utopia_core::models::EntityType], - rtypes: &[utopia_core::models::RelationType], - classes: Option<&HashSet>, - rels: Option<&HashSet>, -) -> PromptLists { - let picked_class = |id: &Uuid| classes.is_none_or(|s| s.contains(id)); - // 按块检索（有选择集）时描述只带第一句：检索那条路只有大的导入本体才走，一块铺上百行， - // 描述占清单的八成（#701）。全铺的是装得下预算的小本体，描述原样 - let describe = |d: &str| -> String { - if classes.is_some() { - utopia_extract::first_sentence(d).to_string() - } else { - d.to_string() - } - }; - // 边上能带的属性：关系.qualifiers → 属性行（0037）。这里只排版，写入侧另有一份同样的查法 - let rtype_by_id: HashMap = - rtypes.iter().map(|r| (r.id, r)).collect(); - let qualifier_line = |r: &utopia_core::models::RelationType| -> Vec { - r.qualifiers - .iter() - .filter_map(|q| rtype_by_id.get(q).copied()) - .filter(|q| q.kind == "attribute") - .map(|q| { - let dt = q.datatype.as_deref().unwrap_or("text"); - match q.unit.as_deref().filter(|u| !u.is_empty()) { - Some(u) => format!("{}: {dt} {u}", q.key), - None => format!("{}: {dt}", q.key), - } - }) - .collect() - }; - let picked_rel = |id: &Uuid| rels.is_none_or(|s| s.contains(id)); - let key_of: HashMap = etypes - .iter() - .filter(|t| picked_class(&t.id)) - .map(|t| (t.id, t.key.as_str())) - .collect(); - - let types = etypes - .iter() - .filter(|t| picked_class(&t.id)) - .map(|t| (t.key.clone(), t.label.clone(), describe(&t.description))) - .collect(); - - // 一侧的类一个都没铺出去就写 `*`：签名是导向，指向看不见的类只会误导 - let sig_of = |ids: &[Uuid]| -> String { - let mut keys: Vec<&str> = ids - .iter() - .filter_map(|id| key_of.get(id).copied()) - .collect(); - if keys.is_empty() { - return "*".into(); - } - keys.sort_unstable(); - keys.join("|") - }; - let relations = rtypes - .iter() - .filter(|r| r.kind != "attribute") - .filter(|r| picked_rel(&r.id)) - .map(|r| { - let signature = if r.domains.is_empty() && r.ranges.is_empty() { - String::new() - } else { - format!("{} → {}", sig_of(&r.domains), sig_of(&r.ranges)) - }; - utopia_extract::PromptRelation { - key: r.key.clone(), - label: r.label.clone(), - description: describe(&r.description), - signature, - temporal: r.temporal.clone(), - // `amount: number $`——模型要按这个 key 写，单位提醒它别换算 - qualifiers: qualifier_line(r), - } - }) - .collect(); - - let attributes = rtypes - .iter() - .filter(|r| r.kind == "attribute" && picked_rel(&r.id)) - .flat_map(|r| r.domains.iter().map(move |d| (r, d))) - .filter_map(|(r, domain_id)| { - let class_key = key_of.get(domain_id)?; - let dt = r.datatype.as_deref().unwrap_or("text"); - let spec = match &r.unit { - Some(u) if !u.is_empty() => format!("{dt}, {u}"), - _ => dt.to_string(), - }; - let d = describe(&r.description); - let d = d.trim(); - Some(if d.is_empty() { - format!("- {class_key}.{} ({spec})", r.key) - } else { - format!("- {class_key}.{} ({spec}): {d}", r.key) - }) - }) - .collect(); - - PromptLists { - types, - relations, - attributes, - } -} - -/// 每块检索多少个类 / 关系 / 属性。**待测**——跟预算一样，定它们要那条曲线。 -const PER_CHUNK_CLASSES: i64 = 40; -const PER_CHUNK_RELATIONS: i64 = 30; -const PER_CHUNK_ATTRIBUTES: i64 = 30; -/// 「这批类身上声明的关系／属性」这道地板给多少名额。 -/// -/// 比按相似度那 30 个宽得多，因为它的池子已经被 domain 收窄过一轮—— -/// schema.org 里 person + organization + corporation 三个类身上一共只有 86 个关系。 -/// 上限只是防病态情况（一块认出上百个类），不是筛选手段。 -const PER_CHUNK_DOMAIN_RELATIONS: i64 = 120; -const PER_CHUNK_DOMAIN_ATTRIBUTES: i64 = 40; - -/// 按这一块的向量检索候选，排出这一块专用的三段清单。 -/// -/// 检索失败返回 `Ok(None)` 而不是错误：调用方会退回全量。提示词大是慢， -/// 没有类可选是抽不出东西——两者之间选前者。 -async fn chunk_lists( - state: &AppState, - kb_id: Uuid, - embedding: &[f32], - etypes: &[utopia_core::models::EntityType], - rtypes: &[utopia_core::models::RelationType], - seed_classes: &HashSet, - budget: usize, -) -> anyhow::Result> { - let mut classes: HashSet = seed_classes.clone(); - classes.extend( - utopia_store::ontology::nearest_entity_type_ids( - &state.pool, - kb_id, - embedding, - PER_CHUNK_CLASSES, - ) - .await?, - ); - // **命中什么，就把它的祖先一起铺出去。** - // - // 向量检索天然偏爱字面出现在正文里的叶子类。实测一个讲 Sutskever 的分块， - // 976 个类按距离排：`researcher` 第 4、`corporation` 第 27， - // 而 `organization` 第 177、`person` 第 359——**前 40 名里一个泛化基类都没有**。 - // 正文写的是 "a researcher at"、"the corporation"，从不写 "person"。 - // - // 两个症状，同一个根因： - // - // - 实体判成 `researcher`（schema.org 里它是 `Audience` 的子类，不是人）， - // 于是 `works_for (domain=person)` 全成了违规 - // - `employee (organization → person)` 的签名**退化成 `(* → *)`**——`sig_of` - // 只认铺出去的类，一侧没铺就写 `*`。模型根本没见过那个方向约束 - // - // 从前这道地板由 `seed_classes`（`builtin` 的类）兜着，`build_lists` 的注释写着 - // 「内置类恒在：检索漏掉的分块仍然要有地方落脚」。种子退场后（#128）判据就悬空了—— - // 它当初碰巧等价，只因为种子类正好是那几个通用类。 - // - // 用祖先补这道地板，比维护一张"通用类"清单好：**继承链本来就是本体自己声明的 - // 泛化关系**，谁是谁的上位不需要我们再判断一次。代价是每块多铺几层祖先。 - if !classes.is_empty() { - let picked: Vec = classes.iter().copied().collect(); - classes.extend(utopia_store::ontology::ancestors_of(&state.pool, &picked).await?); - } - // 关系与属性分开检索：两段在提示词里是分开的，混在一起取会让其中一段 - // 被另一段挤空。检索回来是按距离排好的，排队时交替取，两段一起往下让 - let nearest = interleave( - utopia_store::ontology::nearest_relation_type_ids( - &state.pool, - kb_id, - embedding, - PER_CHUNK_RELATIONS, - Some("relation"), - ) - .await?, - utopia_store::ontology::nearest_relation_type_ids( - &state.pool, - kb_id, - embedding, - PER_CHUNK_ATTRIBUTES, - Some("attribute"), - ) - .await?, - ); - let rels: HashSet = nearest.iter().copied().collect(); - // **一个关系被铺出去，它签名点名的类就得跟着铺。** - // - // 类与关系是各自独立检索的，而签名依赖两者的交集——`sig_of` 只认铺出去的类， - // 一侧没铺就写 `*`。于是常出现这种局面：`employee` 跟正文语义相近被捞了进来， - // 而它的 `organization`（第 795 名）与 `person`（第 630 名）离正文字面很远， - // 一个都没捞到，签名退化成 `(* → *)`——**方向约束整个消失**，模型按英语直觉 - // 写 `Musk --employee--> Microsoft`，而 schema.org 声明的是 organization → person。 - // - // 上面那道祖先地板治不了这种块：它从"命中的叶子"往上长，而这里一个相关的 - // 叶子都没命中，没有叶子也就没有祖先。 - // - // `sig_of` 的注释说「签名指向看不见的类只会误导」——顾虑是对的，但抹掉签名 - // 是拿丢失方向来换。**把类拉进来**两头都保住：模型看得见那个类，签名也排得出。 - // 顺带还对：这些类正是模型马上要用来判类型的那些，`employee` 在场就说明 - // 这一块讲的是雇佣，`organization`/`person` 本来就该在候选里—— - // 按字面相似度捞不到它们，但**本体的结构知道**。 - let signature_classes = |kept: &HashSet| -> HashSet { - rtypes - .iter() - .filter(|r| kept.contains(&r.id)) - .flat_map(|r| r.domains.iter().chain(r.ranges.iter()).copied()) - .collect() - }; - let mut domain_pool = classes.clone(); - domain_pool.extend(signature_classes(&rels)); - - // **类进来了，就把本体声明在它们身上的关系也铺出去。** - // - // 上面那道祖先地板治的是「类捞不到」，这道治的是「关系捞不到」——同一个 - // 病的两侧。实测一块讲「Jensen Huang, founder and CEO of NVIDIA」的正文： - // `employee` 排第 10 进了窗口，模型就用了它；而 `founder` 排 267、 - // `job_title` 排 618、`has_occupation` 排 811，一个都没进——**四篇文档里 - // 每个人的职务因此全部没落进图，而且不留任何丢弃信号：模型没被问到， - // 也就什么都没说，drops 与 misses 两张表都看不见它**（2026-09-08 实测）。 - // - // 收窄的判据是本体自己声明的 domain，不是又一次相似度猜测：这一块认出了 - // person 与 organization，那么「本体说人和组织能有什么」就该摆在模型面前。 - // 同一块里 `founder` 升到 29、`job_title` 升到 55（池子 86）。 - // - // **放在 `sig_classes` 之后，是为了不让它反过来撑大类清单。** 放在前面时 - // 这批关系的 range 会顺着签名规则把一大票类拉进来，同一块的提示词从 11.7k - // 涨到 21.4k——为了一个谓词付两倍的钱。它们的 domain 侧本来就在清单里 - // （地板正是这么选出来的），range 侧退化成 `*` 可以接受：这道地板要办的事 - // 是「让模型看见这个说法存在」，不是把签名补全。 - let domain_ids: Vec = domain_pool.iter().copied().collect(); - let mut on_domains = Vec::new(); - for (limit, kind) in [ - (PER_CHUNK_DOMAIN_RELATIONS, "relation"), - (PER_CHUNK_DOMAIN_ATTRIBUTES, "attribute"), - ] { - on_domains.push( - utopia_store::ontology::nearest_relation_type_ids_in_domains( - &state.pool, - kb_id, - embedding, - limit, - Some(kind), - &domain_ids, - ) - .await?, - ); - } - let domain_attributes = on_domains.pop().unwrap_or_default(); - let domain_relations = on_domains.pop().unwrap_or_default(); - - // 排队：按相似度检索到的在前，地板补进来的在后 - let mut seen = rels.clone(); - let mut ranked = nearest; - ranked.extend( - interleave(domain_relations, domain_attributes) - .into_iter() - .filter(|id| seen.insert(*id)), - ); - - // 一个候选都没检索到 = 索引还没建好，退回全量而不是给一份空清单 - if classes.len() <= seed_classes.len() && ranked.is_empty() { - return Ok(None); - } - - // **这一块的清单也守那个预算**（#701）。预算原本只判「全铺装不装得下」，检索出来的 - // 清单没有上限：三道地板叠上去，schema.org 一块铺到 5.6 万字符，是预算的 2.3 倍， - // 提示词两万 token 里正文不到 2%。 - // - // 按排队顺序取前 k 个，每个带着它的结构（签名点名的类；祖先在类清单里已经有了），量 - // 实际排出来的那段字，取装得下的最大 k。多一个候选只会多几行，长度随 k 单调，二分就行。 - // 地板补进来的关系不拉签名类，理由见上 - let lists_for = |k: usize| { - let kept: HashSet = ranked[..k].iter().copied().collect(); - let near_kept: HashSet = kept.intersection(&rels).copied().collect(); - let mut picked = classes.clone(); - picked.extend(signature_classes(&near_kept)); - build_lists(etypes, rtypes, Some(&picked), Some(&kept)) - }; - let (mut lo, mut hi) = (0usize, ranked.len()); - while lo < hi { - let mid = (lo + hi).div_ceil(2); - if lists_for(mid).chars() <= budget { - lo = mid; - } else { - hi = mid - 1; - } - } - if lo < ranked.len() { - tracing::debug!( - kept = lo, - retrieved = ranked.len(), - budget, - "按块清单按预算截断" - ); + if !document_claims.contains(&id) { + document_claims.push(id); } - Ok(Some(lists_for(lo))) + Ok(id) } -/// 两串按距离排好的 id 交替并成一串：a0 b0 a1 b1 …，短的那串用完就接着排长的 -fn interleave(a: Vec, b: Vec) -> Vec { - let mut out = Vec::with_capacity(a.len() + b.len()); - let (mut a, mut b) = (a.into_iter(), b.into_iter()); - loop { - match (a.next(), b.next()) { - (None, None) => return out, - (x, y) => out.extend(x.into_iter().chain(y)), - } +/// 每篇文档都走开放图谱（0044 决定 2）：只写文档自己的话，本体不进提示词。 +/// 记忆日志同样从这里进，只是它的陈述先进待确认表等人点头（0015），点头时才落成开放陈述 +async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow::Result<()> { + let doc = utopia_store::documents::get(&state.pool, document_id).await?; + // 排队之后被删了（#268）：墓碑不抽——抽出来的事实会活在一个已删除的出处上 + if doc.deleted_at.is_some() { + tracing::info!(document = %document_id, "skipping a deleted document"); + return Ok(()); } -} + let kb = utopia_store::kbs::get(&state.pool, doc.kb_id).await?; + let settings = utopia_store::settings::get(&state.pool, kb.workspace_id) + .await? + .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?; + let client = llm_util::chat_client(&settings) + .ok_or_else(|| anyhow::anyhow!("Chat model not configured; cannot extract"))?; -/// 一条值该记什么单位（#600「单位是读出来的，不是猜的」，实体上的属性与边上的属性同一条规矩）。 -/// -/// 原文里认得出的用原文的（€、¥、%、"EUR 30 million" 的 €）；模型把币种单独写成 -/// 一个键时用那个；原文里写着声明的那个单位（"4300 人" 对 "人"）也算读出来的。 -/// 原文带着一个认不出的单位记号（"500 兆瓦"、"francs"）时**不能**拿声明的缺省顶上—— -/// 实测 `EUR 30 million` 被存成了 `$`，`500 兆瓦` 被存成了 500 块钱；单位写错比不写更糟。 -/// 只有原文完全没有单位记号（一个光秃秃的数）才落回声明的缺省。 -fn unit_for( - raw: &serde_json::Value, - datatype: &str, - sibling_currency: Option<&str>, - declared: Option<&str>, -) -> Option { - // 文本、日期、布尔值没有单位可言："3年"、"30日" 是文本，尾巴上的字不是单位 - // ——实测「期限=3年」被记成了 `3年 年` - if datatype != "number" { - return None; - } - let declared = declared.map(str::trim).filter(|u| !u.is_empty()); - let text = raw.as_str(); - if let Some(u) = text - .and_then(utopia_extract::parse_leading_quantity) - .and_then(|(_, u)| u) - { - return Some(u); - } - if let (Some(c), "number") = (sibling_currency, datatype) { - return Some(c.to_string()); - } - if let (Some(t), Some(d)) = (text, declared) { - if t.contains(d) { - return Some(d.to_string()); - } - } - let has_unit_token = text.is_some_and(|t| { - t.chars() - .any(|c| c.is_alphabetic() || matches!(c, '$' | '€' | '£' | '¥' | '₩' | '₹' | '%')) - }); - if has_unit_token { - None - } else { - declared.map(str::to_string) - } + // 所有权凭证：重抽会自增 epoch，任务据此察觉自己已被接管（见 `run_open` 的分块循环） + let my_epoch = utopia_store::documents::extract_epoch(&state.pool, document_id).await?; + utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?; + state.emit_document(doc.kb_id, document_id); + let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?; + crate::extraction_open::run_open( + state, &doc, &kb, &settings, &client, my_epoch, proposer, await_nod, + ) + .await } #[cfg(test)] @@ -3454,7 +355,7 @@ mod origin_ceiling_tests { fn a_described_fact_cannot_close_a_value_by_itself() { let ceiling = origin_ceiling("described", 0.95); assert!(ceiling < utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE); - assert!(ceiling >= MIN_CONFIDENCE, "it still enters the graph"); + assert_eq!(ceiling, DESCRIBED_CEILING, "it still enters the graph"); assert_eq!(origin_ceiling("described", 0.62), 0.62); for origin in ["stated", "ocr", "transcribed"] { assert_eq!(origin_ceiling(origin, 0.95), 0.95); @@ -3462,625 +363,11 @@ mod origin_ceiling_tests { } } -#[cfg(test)] -mod unit_for_tests { - use super::unit_for; - use serde_json::{json, Value}; - - fn s(t: &str) -> Value { - Value::String(t.to_string()) - } - - #[test] - fn a_unit_is_read_from_the_text_before_anything_else() { - assert_eq!( - unit_for(&s("EUR 30 million"), "number", None, Some("$")).as_deref(), - Some("€") - ); - assert_eq!( - unit_for(&s("8.6亿元"), "number", None, Some("$")).as_deref(), - Some("¥") - ); - assert_eq!( - unit_for(&s("12%"), "number", None, Some("¥")).as_deref(), - Some("%") - ); - assert_eq!( - unit_for(&s("$5 billion"), "number", None, None).as_deref(), - Some("$") - ); - } - - #[test] - fn a_sibling_currency_is_the_unit_of_a_bare_number() { - assert_eq!( - unit_for(&s("1500000000"), "number", Some("CNY"), Some("$")).as_deref(), - Some("CNY") - ); - // 文本型属性没有币种可言 - assert_eq!(unit_for(&s("B 轮"), "text", Some("CNY"), None), None); - // 文本值尾巴上的字也不是单位："3年" 是期限的写法，不是 3 个「年」 - assert_eq!(unit_for(&s("3年"), "text", None, Some("")), None); - assert_eq!(unit_for(&s("30日"), "text", None, None), None); - } - - #[test] - fn the_declared_unit_written_in_the_text_counts_as_read() { - assert_eq!( - unit_for(&s("4300 人"), "number", None, Some("人")).as_deref(), - Some("人") - ); - } - - #[test] - fn an_unknown_unit_token_is_never_overwritten_by_the_default() { - // 宽松扫描把尾巴上的记号当单位读出来：记的是原文的单位，不是声明的 ¥ - assert_eq!( - unit_for(&s("500 兆瓦"), "number", None, Some("¥")).as_deref(), - Some("兆瓦") - ); - assert_eq!( - unit_for(&s("30 million francs"), "number", None, Some("$")).as_deref(), - Some("francs") - ); - // 扫描读不出、原文却明明带着字：也不拿缺省顶上 - assert_eq!( - unit_for(&s("about five hundred"), "number", None, Some("$")), - None - ); - } - - #[test] - fn a_bare_figure_takes_the_declared_default() { - assert_eq!( - unit_for(&s("4300"), "number", None, Some("人")).as_deref(), - Some("人") - ); - assert_eq!( - unit_for(&json!(4300), "number", None, Some("人")).as_deref(), - Some("人") - ); - assert_eq!(unit_for(&s("4300"), "number", None, Some("")), None); - } -} - -#[cfg(test)] -mod name_tests { - use super::{clause_suspect, is_entity_name}; - - /// 样本全部取自实跑出来的库（ai-timeline-ends × schema.org），不是编的。 - #[test] - fn a_clause_is_not_a_thing() { - for s in [ - "thermal-imaging equipment used by volunteers flying over the site showed at least 33 generators giving off heat", - "about the same amount of power as the Tennessee Valley Authority's large gas-fired power plant nearby", - "removal was driven by growing discontent and distrust with Altman", - "a risk of developing cancer at four times the national average in 2013", - "745 of OpenAI's 770 employees", - ] { - assert!(!is_entity_name(s), "这是一句话，不该当成实体名：{s}"); - } - } - - /// 一个数额不是一个东西。 - /// - /// 样本取自实跑出来的库：`$5 billion`、`$1 billion`、`$30 billion` 各自成过节点， - /// 而且同名的会并成一个点——SSI Inc. 与 Nvidia 因为都出现过「$5 billion」 - /// 在图上相连，那条路径没有任何含义。判据的窄处在**尾巴**： - /// 后面还有实词的一律放行，因为那时它说的就不再只是那个数。 - #[test] - fn a_quantity_is_not_a_thing() { - for s in ["$5 billion", "€1.5 million", "52%", "3.5 million", "35,000"] { - assert!(!is_entity_name(s), "这是一个量，不该当成实体名：{s}"); - } - // **以数字开头的真实体一个都不能误伤。** 量级词只认全写，所以 `3M` 解不动； - // 后面挂着实词的，尾巴那一条接住 - for s in [ - "3M", - "7-Eleven", - "23andMe", - "2025 Atlantic hurricane season", - "900 million weekly active users", - "1000 Islands", - "$10 billion investment", - ] { - assert!(is_entity_name(s), "这是真实体，不该被挡：{s}"); - } - } - - /// **真实体会长，但不带谓语。** 判据是词数 + 限定动词，不是字符数—— - /// 下面第一个 57 字符，比上面那条 65 字符的从句还短不了多少 - #[test] - fn a_long_name_is_still_a_name() { - for s in [ - "US District Court for the Northern District of California", - "United States District Court for the District of Delaware", - "OpenAI's board of directors", - "Safe Superintelligence Inc.", - "École Polytechnique", - "GPT-4", - ] { - assert!(is_entity_name(s), "这是真实体，不该被挡：{s}"); - } - } - - /// #193：词表之外的句子，结构信号接住——句号结尾、情态动词。样本来自第二份语料 - #[test] - fn a_sentence_is_caught_without_its_verb_on_the_list() { - for s in [ - "The removal could slow down the artificial intelligence industry as a whole.", - "Shares in Microsoft fell nearly three percent following the announcement.", - "the board should reconsider its position", - ] { - assert!(!is_entity_name(s), "这是一句话，不该当成实体名：{s}"); - } - // 大写缩写的句号、兼作名词的情态词，都不误伤 - for s in [ - "Safe Superintelligence Inc.", - "Theresa May", - "Trash Can Museum", - ] { - assert!(is_entity_name(s), "这是真实体，不该被挡：{s}"); - } - } - - /// 弱信号只记不挡：像从句的照常落库，但留下样本 - #[test] - fn a_suspect_is_recorded_not_rejected() { - let s = "The committee that reviewed the merger of the two companies"; - assert!(is_entity_name(s)); - assert_eq!(clause_suspect(s), Some("determiner_opens_a_long_string")); - assert_eq!( - clause_suspect("committee members who reviewed the merger"), - Some("relative_or_subordinate_clause") - ); - for s in [ - "US District Court for the Northern District of California", - "OpenAI's board of directors", - "The New York Times", - "The Men Who Stare", - ] { - assert_eq!(clause_suspect(s), None, "{s} 不该被怀疑"); - } - } - - /// 分词在名词短语里完全正常，列进标志词会误伤。 - #[test] - fn a_participle_is_not_a_predicate() { - assert!(is_entity_name("equipment used by volunteers")); - assert!(is_entity_name("Gas-Burning Turbines")); - } - - /// 短名字不做从句判断：`Is` 之类可能是专名的一部分。 - #[test] - fn a_short_name_is_never_a_clause() { - assert!(is_entity_name("Was")); - assert!(is_entity_name("Is Elon")); - } - - #[test] - fn an_empty_name_is_not_a_name() { - assert!(!is_entity_name("")); - assert!(!is_entity_name(" ")); - } -} - #[cfg(test)] mod tests { - - #[test] - fn two_ranked_lists_take_turns_and_the_longer_one_finishes() { - let ids: Vec = (0..5).map(|_| Uuid::now_v7()).collect(); - let (a, b) = (vec![ids[0], ids[1], ids[2]], vec![ids[3]]); - assert_eq!( - super::interleave(a, b), - vec![ids[0], ids[3], ids[1], ids[2]] - ); - assert!(super::interleave(Vec::new(), Vec::new()).is_empty()); - } - - #[test] - fn a_name_declared_for_another_entity_is_not_an_alias() { - let (probe, project) = (Uuid::now_v7(), Uuid::now_v7()); - let declared = HashMap::from([ - ("海洋探测器1号".to_string(), probe), - ("海探1项目".to_string(), project), - ]); - assert!(name_claimed_elsewhere("海探1项目", probe, &declared)); - assert!( - name_claimed_elsewhere(" 海探1项目 ", probe, &declared), - "空白不论" - ); - assert!( - !name_claimed_elsewhere("海探1", probe, &declared), - "没人声明过的名字照收" - ); - assert!( - !name_claimed_elsewhere("海洋探测器1号", probe, &declared), - "自己的名字不算撞" - ); - } - use super::{name_claimed_elsewhere, slot_matches, span_in_quote, verify_span, SpanVerdict}; - fn declared(names: &[&str]) -> HashMap { - names - .iter() - .map(|n| (n.to_string(), Uuid::now_v7())) - .collect() - } - - /// 片段核对：#578 的两句真实错例；改绑；头衔与指代只记；正常的放行 - #[test] - fn a_span_that_names_a_description_is_not_the_entity() { - let d = declared(&[ - "OpenAI", - "Anthropic", - "Sam Altman", - "OpenAI's board of directors", - "The Verge", - "Tasha McCauley", - ]); - let quote = "Former OpenAI personnel have founded competing AI companies Anthropic, SpaceXAI, Safe Superintelligence Inc., and Thinking Machines Lab"; - // 名字后面还有词：名字只是修饰语 - assert_eq!( - verify_span(Some("Former OpenAI personnel"), "OpenAI", "", quote, &d), - SpanVerdict::Described("Former OpenAI personnel".into()) - ); - assert_eq!( - verify_span(Some("Anthropic"), "Anthropic", "", quote, &d), - SpanVerdict::Ok - ); - // 所有格：名字只是修饰语 - let q3 = "Claude bypassed Anthropic's safeguards"; - assert_eq!( - verify_span(Some("Anthropic's safeguards"), "Anthropic", "", q3, &d), - SpanVerdict::Described("Anthropic's safeguards".into()) - ); - // 另一个名字带着修饰，绑到了第三方 - assert_eq!( - verify_span( - Some("The Verge reporter"), - "Sam Altman", - "", - "a The Verge reporter wrote", - &d - ), - SpanVerdict::Described("The Verge reporter".into()) - ); - // 名字前面带了词：头衔还是另一件东西分不开，只记 - let q2 = "Over one hundred companies using OpenAI contacted Anthropic"; - assert_eq!( - verify_span(Some("companies using OpenAI"), "OpenAI", "", q2, &d), - SpanVerdict::Prefixed("companies using OpenAI".into()) - ); - assert_eq!( - verify_span( - Some("entrepreneur Tasha McCauley"), - "Tasha McCauley", - "", - "with entrepreneur Tasha McCauley on the board", - &d - ), - SpanVerdict::Prefixed("entrepreneur Tasha McCauley".into()) - ); - // 片段点了另一个声明过的实体：改绑，不丢 - assert_eq!( - verify_span( - Some("Sam Altman"), - "OpenAI", - "", - "Sam Altman announced the deal", - &d - ), - SpanVerdict::Rebind("Sam Altman".into()) - ); - // 单个词包在别的名字里不改绑（#595）：句首大写不是专名的证据。"Altman" 绑错到 - // OpenAI 时只记指代——改绑要么精确/词干命中，要么两个词以上 - assert_eq!( - verify_span( - Some("Altman"), - "OpenAI", - "", - "Altman announced the deal", - &d - ), - SpanVerdict::Coreference("Altman".into()) - ); - assert_eq!( - verify_span( - Some("Stockholders"), - "NVIDIA", - "", - "Stockholders approved the election of each of our ten (10) director nominees", - &declared(&[ - "NVIDIA", - "2026 Annual Meeting of Stockholders of NVIDIA Corporation" - ]) - ), - SpanVerdict::Coreference("Stockholders".into()) - ); - assert_eq!( - verify_span( - Some("OpenAI's board"), - "OpenAI", - "", - "OpenAI's board removed Sam Altman as CEO", - &d - ), - SpanVerdict::Rebind("OpenAI's board of directors".into()) - ); - // 指代：没有任何名字，绑定照旧、只记 - assert_eq!( - verify_span( - Some("him"), - "Sam Altman", - "", - "the board reinstated him", - &d - ), - SpanVerdict::Coreference("him".into()) - ); - // 单个普通词包在别的名字里也是指代，不改绑（"the company" ≠ "for-profit company"） - let d2 = declared(&[ - "OpenAI", - "for-profit company", - "Satya Nadella", - "Jony Ive", - "Apple", - "Microsoft", - "Helen Toner", - "Fidji Simo", - ]); - assert_eq!( - verify_span( - Some("the company"), - "OpenAI", - "", - "the company took over", - &d2 - ), - SpanVerdict::Coreference("the company".into()) - ); - // 片段以另一个名字结尾：不猜它是头（v4/v5 里猜错的多过猜对的），照描述处理 - assert_eq!( - verify_span( - Some("Microsoft chief executive Satya Nadella"), - "Microsoft", - "OpenAI", - "convinced Microsoft chief executive Satya Nadella", - &d2 - ), - SpanVerdict::Described("Microsoft chief executive Satya Nadella".into()) - ); - assert_eq!( - verify_span( - Some("Swisher and The Verge reporter Alex Heath"), - "Kara Swisher", - "", - "Swisher and The Verge reporter Alex Heath stated", - &declared(&["Kara Swisher", "Alex Heath", "The Verge"]) - ), - SpanVerdict::Described("Swisher and The Verge reporter Alex Heath".into()) - ); - // 以另一侧的名字结尾：抄错了位置，绑定照旧、只记 - assert_eq!( - verify_span( - Some("CEO of Applications: Fidji Simo"), - "OpenAI", - "Fidji Simo", - "CEO of Applications: Fidji Simo", - &d2 - ), - SpanVerdict::Misplaced("CEO of Applications: Fidji Simo".into()) - ); - // 末尾的名字是介词的补语或名单的最后一项，不是头：不改绑（v4 的四个错例） - let d3 = declared(&[ - "OpenAI", - "companies using OpenAI", - "TBPN", - "California", - "Pioneer Building", - "San Francisco", - "Anthropic", - ]); - assert_eq!( - verify_span( - Some("Over one hundred companies using OpenAI"), - "companies using OpenAI", - "Anthropic", - "Over one hundred companies using OpenAI contacted Anthropic", - &d3 - ), - SpanVerdict::Prefixed("Over one hundred companies using OpenAI".into()) - ); - assert_eq!( - verify_span( - Some("his vested equity in OpenAI"), - "vested equity", - "Sam Altman", - "forfeited his vested equity in OpenAI", - &d3 - ), - SpanVerdict::Described("his vested equity in OpenAI".into()) - ); - assert_eq!( - verify_span( - Some("TBPN, a media company in California"), - "TBPN", - "OpenAI", - "acquired TBPN, a media company in California", - &d3 - ), - SpanVerdict::Ok - ); - assert_eq!( - verify_span( - Some("the Pioneer Building in the Mission District, San Francisco"), - "Pioneer Building", - "OpenAI", - "located in the Pioneer Building in the Mission District, San Francisco", - &d3 - ), - SpanVerdict::Described( - "the Pioneer Building in the Mission District, San Francisco".into() - ) - ); - // 名字后面紧跟逗号：同位语，还是它 - assert_eq!( - verify_span( - Some("Helen Toner, strategy director for the Center for Security and Emerging Technology"), - "Helen Toner", - "OpenAI", - "and Helen Toner, strategy director for the Center for Security and Emerging Technology", - &d2 - ), - SpanVerdict::Ok - ); - // 没给片段：老行为 - assert_eq!(verify_span(None, "OpenAI", "", quote, &d), SpanVerdict::Ok); - // 片段不在引文里：只记不拦 - assert_eq!( - verify_span(Some("OpenAI staff"), "OpenAI", "", q2, &d), - SpanVerdict::NotInQuote - ); - } - - /// 模型写的名字和它 ref 指的实体打架：写 "OpenAI employees" 却指向 OpenAI - #[test] - fn a_written_name_that_describes_its_reference_is_a_description() { - use super::written_verdict; - let d = declared(&[ - "OpenAI", - "Anthropic", - "Sam Altman", - "OpenAI's board of directors", - ]); - // v5 的最后一条假边：片段 "eleven employees" 没有名字拦不住，写的名字拦得住 - assert_eq!( - written_verdict("OpenAI employees", "OpenAI", "Anthropic", true, &d), - SpanVerdict::Described("OpenAI employees".into()) - ); - assert_eq!( - written_verdict("Former OpenAI personnel", "OpenAI", "Anthropic", true, &d), - SpanVerdict::Described("Former OpenAI personnel".into()) - ); - // 写的是另一个声明过的实体：改绑 - assert_eq!( - written_verdict( - "OpenAI's board of directors", - "OpenAI", - "Sam Altman", - true, - &d - ), - SpanVerdict::Rebind("OpenAI's board of directors".into()) - ); - // 写的就是那个名字（含后缀、部分）、没有 ref、或是指代：无事 - assert_eq!( - written_verdict("OpenAI, Inc.", "OpenAI", "", true, &d), - SpanVerdict::Ok - ); - assert_eq!( - written_verdict("Altman", "Sam Altman", "", true, &d), - SpanVerdict::Ok - ); - assert_eq!( - written_verdict("OpenAI employees", "OpenAI", "", false, &d), - SpanVerdict::Ok - ); - assert_eq!( - written_verdict("the company", "OpenAI", "", true, &d), - SpanVerdict::Ok - ); - } - - /// 名字后面接着 and 再接专名，是并列的一项，不是描述（#595）；接着 and 再接小写 - /// 的描述还是描述 - #[test] - fn a_name_in_a_coordination_is_one_of_the_list() { - let d = declared(&["SB Energy", "SoftBank", "OpenAI"]); - let q = "SB Energy and SoftBank will build at least 10 GW of new energy generation"; - assert_eq!( - verify_span(Some("SB Energy and SoftBank"), "SB Energy", "", q, &d), - SpanVerdict::Ok - ); - assert_eq!( - verify_span( - Some("SB Energy & SoftBank"), - "SB Energy", - "", - "SB Energy & SoftBank will build", - &d - ), - SpanVerdict::Ok - ); - // 绑在后一项上：名字前面带词，只记 - assert_eq!( - verify_span(Some("SB Energy and SoftBank"), "SoftBank", "", q, &d), - SpanVerdict::Prefixed("SB Energy and SoftBank".into()) - ); - assert_eq!( - verify_span( - Some("OpenAI and its investors"), - "OpenAI", - "", - "OpenAI and its investors agreed", - &d - ), - SpanVerdict::Described("OpenAI and its investors".into()) - ); - } - - /// 名字后面接着专名样子的续词是同一个东西；接着小写的词就不是 - #[test] - fn a_name_continued_in_capitals_is_the_same_thing() { - let d = declared(&["Anthropic", "OpenAI"]); - assert_eq!( - verify_span( - Some("Anthropic PBC"), - "Anthropic", - "", - "Anthropic PBC filed", - &d - ), - SpanVerdict::Ok - ); - assert_eq!( - verify_span( - Some("OpenAI Global, LLC"), - "OpenAI", - "", - "OpenAI Global, LLC is the for-profit arm", - &d - ), - SpanVerdict::Ok - ); - assert_eq!( - verify_span( - Some("OpenAI employees"), - "OpenAI", - "", - "OpenAI employees left", - &d - ), - SpanVerdict::Described("OpenAI employees".into()) - ); - } - - #[test] - fn a_slot_matches_its_name_by_stem_and_suffix() { - assert!(slot_matches("OpenAI", "OpenAI, Inc.")); - assert!(slot_matches("Acme", "Acme Corp.")); - assert!(slot_matches("openai", "OpenAI")); - assert!(slot_matches("OpenAI's", "OpenAI")); - assert!(slot_matches("Altman", "Sam Altman")); - assert!(slot_matches("Anthropic", "Anthropic, PBC")); - assert!(slot_matches( - "the Center for Security and Emerging Technology", - "Center for Security and Emerging Technology" - )); - assert!(!slot_matches("Former OpenAI personnel", "OpenAI")); - assert!(!slot_matches("Anthropic", "OpenAI")); - } + use super::{incomplete_reason, resolve_handle, span_in_quote}; + use std::collections::HashMap; + use uuid::Uuid; #[test] fn a_span_is_found_in_its_quote_regardless_of_case_and_spacing() { @@ -4095,109 +382,6 @@ mod tests { assert!(!span_in_quote("", "anything")); } - use super::{ - incomplete_reason, looks_literal, no_ref_name_binding, referenced_entity, resolve_bare, - resolve_handle, validity_of, BoundEntity, NoRefNameBinding, - }; - use std::collections::HashMap; - use uuid::Uuid; - - #[test] - fn quantities_and_dates_count_as_literals() { - // 认：这些出现在宾语位上时是值，不是实体 - for yes in [ - "2015", - "2023-03", - "2024-01-15", - "1200", - "62.5", - "-3", - // 带符号与量级词的量。从前这一档不认，于是图上长出一个叫 - // 「$5 billion」的节点，同名的还并成一个，把毫不相干的两家公司连起来 - "$5 billion", - "€1.5 million", - "52%", - "35,000", - // 合同照原文写的日期（#688） - "June 23, 2020", - "17 Mar. 2020", - "2020年3月17日", - ] { - assert!(looks_literal(yes), "{yes} 该认成字面值"); - } - // 不认：判错的代价是把一个真实体降成一段文本，所以宁可漏 - for no in [ - "杭州", - "首席技术官", - "3M", - "V3", - "深蓝存储", - "", - " ", - "２０１５", // 全角数字：不是我们要处理的形态，交给实体路径 - // 尾巴上还有实词：它说的不再只是那个数 - "900 million weekly active users", - "2025 Atlantic hurricane season", - "$10 billion investment", - "8GW data center", - ] { - assert!(!looks_literal(no), "{no} 不该认成字面值"); - } - } - - /// 区间两端照合同原文写（#688）：读成日期，精度随写了几位；「unknown」仍是结束了不知哪天 - #[test] - fn a_written_start_keeps_the_precision_it_was_written_with() { - let day = validity_of(Some("June 8, 2020"), None, None); - assert_eq!( - day.from.map(|t| t.date_naive().to_string()).as_deref(), - Some("2020-06-08") - ); - assert_eq!(day.from_precision, Some("day")); - assert!(!day.has_ended()); - - let month = validity_of(Some("March 2020"), Some("17 Mar. 2021"), None); - assert_eq!(month.from_precision, Some("month")); - assert_eq!( - month.to.map(|t| t.date_naive().to_string()).as_deref(), - Some("2021-03-17") - ); - assert_eq!(month.to_precision, Some("day")); - - // 说不清几月几号的不当起点 - let ambiguous = validity_of(Some("03/04/2020"), Some("unknown"), None); - assert_eq!((ambiguous.from, ambiguous.from_precision), (None, None)); - assert_eq!(ambiguous.to_precision, Some("unknown")); - } - - #[test] - fn no_ref_names_bypass_legacy_map_only_after_two_handle_claims() { - let (a, b) = (Uuid::now_v7(), Uuid::now_v7()); - let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]); - let normalized = utopia_store::resolution::normalize_name("Zhang Wei").to_lowercase(); - - assert_eq!( - no_ref_name_binding(&entity_ids, &HashMap::new(), "Zhang Wei"), - NoRefNameBinding::Legacy(b) - ); - assert_eq!( - no_ref_name_binding( - &entity_ids, - &HashMap::from([(normalized.clone(), vec![a])]), - "Zhang Wei", - ), - NoRefNameBinding::Legacy(b) - ); - assert_eq!( - no_ref_name_binding( - &entity_ids, - &HashMap::from([(normalized, vec![a, b])]), - "Zhang Wei", - ), - NoRefNameBinding::AmbiguousHandled - ); - } - /// 这条判据存在的理由：一次网络抖动让六篇文档 60 块里只抽成 12 块， /// 六篇**全部显示"抽取完成"**，八成的内容没进图，界面上没有任何东西说出来。 #[test] @@ -4229,7 +413,7 @@ mod tests { } #[tokio::test] - async fn namesake_handles_keep_fact_attribution_and_bare_mentions_get_c() -> anyhow::Result<()> + async fn namesake_handles_keep_fact_attribution_and_a_fresh_handle_gets_c() -> anyhow::Result<()> { let Some(url) = utopia_store::test_db::url() else { return Ok(()); @@ -4264,32 +448,6 @@ mod tests { .await?; let run = async { - let mut legacy_cache = HashMap::new(); - let mut legacy_needs_adjudication = false; - let legacy_first = super::resolve( - &pool, - kb, - Some(person), - "Alice", - None, - None, - &mut legacy_cache, - &mut legacy_needs_adjudication, - ) - .await?; - let legacy_second = super::resolve( - &pool, - kb, - Some(person), - "Alice", - None, - None, - &mut legacy_cache, - &mut legacy_needs_adjudication, - ) - .await?; - assert_eq!(legacy_first, legacy_second, "legacy name cache is unchanged"); - let mut response_claims = HashMap::new(); let mut document_claims = HashMap::new(); let mut bare_cache = HashMap::new(); @@ -4329,25 +487,8 @@ mod tests { "namesakes must not wake the LLM worker" ); - let refs = HashMap::from([ - ( - "e1".to_string(), - BoundEntity { - id: a, - type_id: Some(person), - }, - ), - ( - "e2".to_string(), - BoundEntity { - id: b, - type_id: Some(person), - }, - ), - ]); - assert_eq!(referenced_entity(&refs, "e1").map(|x| x.id), Some(a)); - assert_eq!(referenced_entity(&refs, "e2").map(|x| x.id), Some(b)); - assert!(referenced_entity(&refs, "missing").is_none()); + // 同一回复里两个句柄各绑各的实体：事实跟着句柄走，不跟着名字走 + let refs = HashMap::from([("e1", a), ("e2", b)]); let (finance, platform) = (Uuid::now_v7(), Uuid::now_v7()); let works_at = Uuid::now_v7(); sqlx::query( @@ -4366,7 +507,7 @@ mod tests { .await?; } for (handle, object) in [("e1", finance), ("e2", platform)] { - let subject = referenced_entity(&refs, handle).expect("valid ref").id; + let subject = refs[handle]; utopia_store::graph::insert_fact( &pool, kb, @@ -4400,29 +541,25 @@ mod tests { assert!(labels.contains(&(a, Some("Finance".to_string())))); assert!(labels.contains(&(b, Some("Platform Engineering".to_string())))); - let mut doc_cache = HashMap::new(); - // The legacy surface map still ends on B, but a no-ref fact must not use that - // last-write-wins value once handles have claimed two distinct Zhang Weis. - let entity_ids = HashMap::from([("Zhang Wei".to_string(), b)]); - assert_eq!(entity_ids.get("Zhang Wei"), Some(&b)); - assert_eq!( - super::no_ref_name_binding(&entity_ids, &document_claims, "Zhang Wei"), - super::NoRefNameBinding::AmbiguousHandled - ); - let c = resolve_bare( + // 后一次回复给同一个名字开了一个新句柄，没挑 A 或 B：文本里没有挑的依据， + // 不许猜。事实落到一个文档级的 C 上，对 A、对 B 各一条人工审核对 + let mut later_response_claims = HashMap::new(); + let c = resolve_handle( &pool, kb, Some(person), "Zhang Wei", None, None, - &mut doc_cache, - &document_claims, + &mut later_response_claims, + &mut document_claims, &mut bare_cache, &mut needs_adjudication, &mut human_reviews, ) .await?; + assert_ne!(c, a); + assert_ne!(c, b); utopia_store::graph::insert_fact( &pool, kb, @@ -4433,20 +570,26 @@ mod tests { 0.9, ) .await?; - let c_again = resolve_bare( + // 再往后的回复再开一个新句柄，仍是那个 C：一个新拼写不是猜 A 或 B 的许可 + let mut another_response_claims = HashMap::new(); + let c_again = resolve_handle( &pool, kb, None, "Zhang Wei", None, None, - &mut doc_cache, - &document_claims, + &mut another_response_claims, + &mut document_claims, &mut bare_cache, &mut needs_adjudication, &mut human_reviews, ) .await?; + assert_eq!( + c_again, c, + "a later response cannot evade A/B ambiguity by inventing a new e-handle" + ); utopia_store::graph::insert_fact( &pool, kb, @@ -4457,12 +600,6 @@ mod tests { 0.9, ) .await?; - assert_ne!(c, a); - assert_ne!(c, b); - assert_eq!( - c_again, c, - "later bare mentions must reuse document-local C" - ); let c_objects: Vec = sqlx::query_scalar( "SELECT object_id FROM facts WHERE kb_id = $1 AND subject_id = $2 AND object_id IS NOT NULL ORDER BY object_id", @@ -4474,25 +611,6 @@ mod tests { assert_eq!(c_objects.len(), 2); assert!(c_objects.contains(&finance)); assert!(c_objects.contains(&platform)); - let mut later_response_claims = HashMap::new(); - let c_via_fresh_handle = resolve_handle( - &pool, - kb, - Some(person), - "Zhang Wei", - None, - None, - &mut later_response_claims, - &mut document_claims, - &mut bare_cache, - &mut needs_adjudication, - &mut human_reviews, - ) - .await?; - assert_eq!( - c_via_fresh_handle, c, - "a later response cannot evade A/B ambiguity by inventing a new e-handle" - ); let reviews = utopia_store::resolution::list_reviews( &pool, @@ -4520,11 +638,10 @@ mod tests { run } - /// #270：跨文档的裸 mention 同名并列——不靠 handle，靠画像相似度。库里已有两个 - /// 「张伟」，画像质心一模一样（同一 chunk 播的种）。新 mention 对两人打出同一个分。 - /// 旧路径在最高分 ≥ SIM_ATTACH 时静默 attach 到先遇到的那个；修好之后 `resolve` - /// 走的这条路要：新建第三个实体、两条**人工**审核对、且绝不唤醒 LLM 裁决器 - /// （否则两条几乎相同的画像会被自动并掉，正是要防的）。 + /// #270：跨文档的同名并列——不靠句柄，靠画像相似度。库里已有两个「张伟」，画像质心 + /// 一模一样（同一 chunk 播的种）。新提及对两人打出同一个分。旧路径在最高分 ≥ SIM_ATTACH + /// 时静默 attach 到先遇到的那个；修好之后走句柄这条路要：新建第三个实体、两条**人工** + /// 审核对、且绝不唤醒 LLM 裁决器（否则两条几乎相同的画像会被自动并掉，正是要防的）。 #[tokio::test] async fn a_profile_tie_across_documents_files_human_reviews_not_an_attach() -> anyhow::Result<()> { @@ -4578,17 +695,22 @@ mod tests { // 与两人画像都一致的上下文：打出的余弦相同 → 分不开 let ctx: Vec = vec![1.0, 0.0, 0.0]; - let mut doc_cache = HashMap::new(); - let mut needs_adjudication = false; - let c = super::resolve( + let mut response_claims = HashMap::new(); + let mut document_claims = HashMap::new(); + let mut bare_cache = HashMap::new(); + let (mut needs_adjudication, mut human_reviews) = (false, false); + let c = resolve_handle( &pool, kb, Some(person), "Zhang Wei", Some(&ctx), None, - &mut doc_cache, + &mut response_claims, + &mut document_claims, + &mut bare_cache, &mut needs_adjudication, + &mut human_reviews, ) .await?; @@ -4629,36 +751,3 @@ mod tests { run } } - -#[cfg(test)] -mod undeclared_beside_value_tests { - use super::undeclared_beside_value; - use std::collections::HashMap; - use uuid::Uuid; - - #[test] - fn a_phrase_beside_a_value_is_not_an_entity() { - let mut declared = HashMap::new(); - declared.insert("SB Energy".to_string(), Uuid::nil()); - declared.insert("Microsoft".to_string(), Uuid::nil()); - // 回复里的真形状：没句柄、没声明 → 值落下，短语进谓词 - assert!(undeclared_beside_value( - None, - "new energy generation", - &declared - )); - assert!(undeclared_beside_value( - Some(" "), - "new regional grid infrastructure", - &declared - )); - // 宾语有句柄，或者是认下的名字（大小写不计）→ 是实体，走边 - assert!(!undeclared_beside_value( - Some("e2"), - "new energy generation", - &declared - )); - assert!(!undeclared_beside_value(None, "microsoft", &declared)); - assert!(!undeclared_beside_value(None, " ", &declared)); - } -} diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index 92436221e..c31f1517c 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -24,6 +24,7 @@ use crate::extraction::{ span_in_quote, }; use crate::state::AppState; +use sqlx::PgPool; use std::collections::{HashMap, HashSet}; use utopia_core::models::{Document, KnowledgeBase, LlmSettings, Proposer}; use utopia_store::extraction_drops::reason; @@ -76,6 +77,46 @@ fn name_key(name: &str) -> String { .to_lowercase() } +/// 陈述里写的名字 → 实体。先看这一块列出的，再看本文档前面认下的（提示词里的清单）。 +/// +/// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`：实测一个 25 篇的库里 +/// 848 个实体有 616 个是被描述的，其中 181 个没挂任何陈述，还有 155 字的从句。没人指着的 +/// 描述不是实体，只是一句话的一部分——所以描述先记在 `deferred` 里，第一条指到它的陈述 +/// 才把它建出来；同一篇里同一段描述只建一次（`described`） +async fn place( + pool: &PgPool, + kb_id: Uuid, + local: &mut HashMap, + known: &HashMap, + deferred: &HashMap, + described: &mut HashMap, + name: &str, +) -> anyhow::Result> { + let key = name_key(name); + if let Some(id) = local.get(&key).or_else(|| known.get(&key)) { + return Ok(Some(*id)); + } + let Some((text, kind)) = deferred.get(&key) else { + return Ok(None); + }; + let id = match described.get(&key) { + Some(id) => *id, + None => { + let id = utopia_store::resolution::create_described( + pool, + kb_id, + text, + (!kind.is_empty()).then_some(kind.as_str()), + ) + .await?; + described.insert(key.clone(), id); + id + } + }; + local.insert(key, id); + Ok(Some(id)) +} + /// `await_nod`：这是记忆日志（0015）——陈述不直接落库，原样进待确认表，人点头时才成为开放陈述。 /// `proposer`：那句话是谁、经哪枚令牌说的（0026），随待确认项一起记 #[allow(clippy::too_many_arguments)] @@ -195,6 +236,8 @@ pub(crate) async fn run_open( // ---- 东西：有名字的走身份消解，被描述的建成没有名字事实的实体 ---- let mut response_claims: HashMap> = HashMap::new(); let mut local: HashMap = HashMap::new(); + // 被描述的东西：先记下名字和类别词，等陈述指到它再建（见 `place`） + let mut deferred: HashMap = HashMap::new(); for e in &extraction.entities { let name = e.name.trim(); if name.is_empty() { @@ -248,28 +291,11 @@ pub(crate) async fn run_open( } id } else { - match described.get(&key) { - Some(id) => *id, - None => { - let id = utopia_store::resolution::create_described( - pool, - kb_id, - name, - (!kind.is_empty()).then_some(kind), - ) - .await?; - described.insert(key.clone(), id); - id - } - } + deferred.insert(key, (name.to_string(), kind.to_string())); + continue; }; local.insert(key, id); } - // 陈述里写的名字 → 实体：先看这一块列出的，再看本文档前面认下的（提示词里的清单） - let resolve_name = |name: &str| -> Option { - let key = name_key(name); - local.get(&key).or_else(|| known_by_name.get(&key)).copied() - }; // ---- 陈述 ---- for s in &extraction.statements { @@ -277,7 +303,17 @@ pub(crate) async fn run_open( if phrase.is_empty() { continue; } - let Some(subject) = resolve_name(&s.subject) else { + let Some(subject) = place( + pool, + kb_id, + &mut local, + &known_by_name, + &deferred, + &mut described, + &s.subject, + ) + .await? + else { drop_signal( state, kb_id, @@ -299,7 +335,17 @@ pub(crate) async fn run_open( .map(str::trim) .filter(|v| v.chars().any(char::is_alphanumeric)); let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) { - Some(name) => match resolve_name(name) { + Some(name) => match place( + pool, + kb_id, + &mut local, + &known_by_name, + &deferred, + &mut described, + name, + ) + .await? + { Some(id) => Some(id), None => { drop_signal( @@ -376,7 +422,17 @@ pub(crate) async fn run_open( if role.is_empty() || text.is_empty() { continue; } - match resolve_name(text) { + match place( + pool, + kb_id, + &mut local, + &known_by_name, + &deferred, + &mut described, + text, + ) + .await? + { Some(id) => qualifiers.push((role, None, Some(id))), None => qualifiers.push((role, Some(serde_json::json!(text)), None)), } @@ -500,7 +556,17 @@ pub(crate) async fn run_open( if name.is_empty() { continue; } - let Some(id) = resolve_name(&n.entity) else { + let Some(id) = place( + pool, + kb_id, + &mut local, + &known_by_name, + &deferred, + &mut described, + &n.entity, + ) + .await? + else { drop_signal( state, kb_id, diff --git a/crates/utopia-server/src/ontology_index.rs b/crates/utopia-server/src/ontology_index.rs index b3e05bce5..7b1ef7622 100644 --- a/crates/utopia-server/src/ontology_index.rs +++ b/crates/utopia-server/src/ontology_index.rs @@ -64,64 +64,6 @@ pub async fn refresh(state: &AppState, kb_id: Uuid) -> anyhow::Result { refresh_scoped(state, kb_id, None).await } -/// 抽取前的门控（#526）。**走这条路径的抽取器必须先问这个再继续**—— -/// 不问而直接抽，要么抽出来的图是基于半个本体的、要么抽取 worker 在 -/// `refresh` 锁上空转把 32 个并发槽占死。 -/// -/// 返回值语义： -/// - `Ok(false)`：本体小，全铺就行；本体大但已经在向量里了；或者没配嵌入模型 -/// （照旧全铺）。继续抽。 -/// - `Err(anyhow::Error::context(Deferred{30s}))`：本体超出预算且向量还没补齐， -/// 调用方应该把这个错误原样往上抛——`jobs::mark_failed` 认 `Deferred`，把任务 -/// 挂回 `queued` 等 30s。worker 槽立刻空出来，下一轮再试。 -pub async fn gate_required(state: &AppState, kb_id: Uuid) -> anyhow::Result { - let etypes = utopia_store::graph::entity_types(&state.pool, kb_id).await?; - let mut rtypes = utopia_store::graph::relation_types(&state.pool, kb_id).await?; - // 名字属性不进本体向量索引：检索出来就会被当成一条可抽的属性递给模型 - rtypes.retain(|r| !utopia_store::names::is_name_attribute(r)); - let budget = utopia_store::access::ontology_prompt_budget(&state.pool).await?; - let chars = crate::extraction::full_ontology_chars(&etypes, &rtypes); - if chars <= budget { - return Ok(false); - } - // 超出预算：需要检索候选。检索候选要先有向量——向量是否就绪取决于 - // `types_needing_embedding`，而它要求一个具体的嵌入模型名。 - // - // KB 拿不到就不再静默当成「没事」——`run()` 后面还会再读一次同样的字段， - // 让那次错误冒上来但归因不清。`Err` 直接归到这里，运维日志一眼能找到。 - let kb = utopia_store::kbs::get(&state.pool, kb_id).await?; - let Some(settings) = utopia_store::settings::get(&state.pool, kb.workspace_id).await? else { - // 没设模型就走全铺那条路——和原行为一致。配置阶段不该在这里就报错 - return Ok(false); - }; - // 与 `refresh_scoped` 同一个判据：客户端与模型名缺一个，补齐就什么都不做， - // 等下去只会空等到期限 - let (Some(_), Some(embed_model)) = ( - llm_util::embed_client(&settings), - settings.embed_model.clone(), - ) else { - // 超预算 + 没配嵌入模型：照旧按全量本体抽。那种部署本来就没有检索， - // 没有什么可等；判成失败会让只配了对话模型的库一篇都抽不出来 - return Ok(false); - }; - let stale = - utopia_store::ontology::types_needing_embedding(&state.pool, kb_id, &embed_model, None) - .await?; - if stale.is_empty() { - // 超预算但所有类型都已嵌好——这个组合很罕见，意味着库被改大后又改小了， - // 但代码路径是合法的。继续抽 - return Ok(false); - } - // 入队补齐任务——同库已排着的不重复（`enqueue_unless_queued` 的契约） - utopia_store::jobs::enqueue_unless_queued( - &state.pool, - "embed_ontology", - serde_json::json!({ "kb_id": kb_id }), - ) - .await?; - Ok(true) -} - /// 只补一半。调用方清楚自己要哪一半时用它——类型消解只用类， /// 等关系嵌完是白等。补漏的那一半有后台任务兜着。 pub async fn refresh_scoped( diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs index 0ba9cad0d..15cf80211 100644 --- a/crates/utopia-store/src/extraction_drops.rs +++ b/crates/utopia-store/src/extraction_drops.rs @@ -13,91 +13,24 @@ use utopia_core::AppResult; use uuid::Uuid; /// 原因码。前端按这个查文案，所以是稳定契约，不要改字面量。 +/// 都由开放抽取那条路（0044）发出；类型化那条路的原因码随它一起退场了 pub mod reason { - /// 主语没在 entities 里声明 → 类型不明，属性无法校验 domain - pub const SUBJECT_NOT_DECLARED: &str = "subject_not_declared"; - /// 属性挂在了不该挂的类上（salary 挂到 Organization） - pub const ATTR_DOMAIN_MISMATCH: &str = "attr_domain_mismatch"; - /// 属性事实既没给 value 也没给 object - pub const ATTR_NO_VALUE: &str = "attr_no_value"; - /// 值不合 datatype，归一化失败 - pub const ATTR_DATATYPE: &str = "attr_datatype"; - /// 模型在边上写了一个这条关系没声明过的属性 key（0037） - pub const QUALIFIER_UNKNOWN: &str = "qualifier_unknown"; - /// 边上属性的值换不成它声明的 datatype - pub const QUALIFIER_DATATYPE: &str = "qualifier_datatype"; - /// 同一条边再听到一次，属性值与已记的不一致——先记下，不覆盖 - pub const QUALIFIER_CONFLICT: &str = "qualifier_conflict"; - /// 模型自报置信度低于阈值 - pub const LOW_CONFIDENCE: &str = "low_confidence"; - /// 关系事实缺宾语 + /// 陈述缺宾语也缺值 pub const OBJECT_MISSING: &str = "object_missing"; - /// 模型给的这一条不合结构（缺 predicate 之类）→ 只跳这一条，不牵连整块 + /// 模型给的这一条不合结构（缺短语之类）→ 只跳这一条，不牵连整块 pub const MALFORMED_ITEM: &str = "malformed_item"; - /// 主语的类型对不上关系声明的 domain，**且对调也不合法**——那是选错了关系 - /// 或类型判错，不是方向问题。照原样落库 + 记信号，交给人看，不猜 - pub const DOMAIN_MISMATCH: &str = "domain_mismatch"; - /// 模型给的"实体名"其实是一整句话或从句——不是一个东西的名字。 - /// 这类东西永远匹配不到别处的提及，在图上是孤点，还会拖累消解 - pub const NOT_AN_ENTITY_NAME: &str = "not_an_entity_name"; - /// 主语违反 domain 而宾语符合，已按本体声明的方向把主宾掰正。 - /// **动作必须留痕**：自动的、看不见的改写才是 0001 反对的那种 - pub const DIRECTION_CORRECTED: &str = "direction_corrected"; /// 模型输出被截断（撞上 max_tokens）→ 已完整的那些留下，尾巴丢掉 pub const TRUNCATED_REPLY: &str = "truncated_reply"; - /// 守卫放行了、结构却像从句（限定词起头的长串、句中的关系词）。**只记不挡**： - /// 实体照常落库，例句留下来——#193 要的是一份跨语料的标注集，再决定哪条升成硬规则 - pub const CLAUSE_SUSPECT: &str = "clause_suspect"; - /// 主语是「跟 X 有关的一群人」而写成了 X（#578）："former OpenAI personnel" 不是 - /// OpenAI。事实不落，例句是那个短语；这一类该由提示词的规则改写到有名字的一侧， - /// 这里只量它还错多少 - pub const SUBJECT_SHORTENED: &str = "subject_shortened"; - /// 主宾片段不在引文里（#582）：模型没照抄。事实照旧处理，只记下来 - pub const SPAN_NOT_IN_QUOTE: &str = "span_not_in_quote"; /// 模型报的别名（或它的引文）不在这一块原文里（0041 决定 2）：不记这个名字。 /// 名字是召回的桥，一座凭空的桥会把两个不相干的实体接到一起 pub const NAME_NOT_IN_TEXT: &str = "name_not_in_text"; /// 模型报的别名，这次回复（或本文档前面几块）里已经是另一个实体的名字（0041 决定 2）： /// 一个名字不会同时是两样东西的名字。「海探1项目」声明成了一个机构，就不是探测器的别名 pub const NAME_CLAIMED_BY_ANOTHER: &str = "name_claimed_by_another"; - /// 主语片段是个描述，不是任何声明过的实体的名字：事实不落（#582，取代 #578 的词表） - pub const SUBJECT_DESCRIBED: &str = "subject_described"; - /// 宾语片段是个描述：事实照落，宾语落成字面值（#582） - pub const OBJECT_DESCRIBED: &str = "object_described"; - /// 片段点的是另一个声明过的实体：改绑到它（#582） - pub const SPAN_REBOUND: &str = "span_rebound"; - /// 片段是所绑名字前面带了别的词（"entrepreneur Tasha McCauley" / "companies using - /// OpenAI"）：头衔还是另一件东西，机器分不开，绑定照旧，只记（#582） - pub const SPAN_PREFIXED: &str = "span_prefixed"; - /// 片段里没有任何声明过的名字（"him" / "the company"）：模型消解了指代，无从核对， - /// 绑定照旧，只记（#582） - pub const SPAN_COREFERENCE: &str = "span_coreference"; - /// 片段抄的是事实**另一侧**的名字（宾语片段写成了主语）：抄错了位置，不是绑错了 - /// 实体。绑定照旧，只记（#582） - pub const SPAN_MISPLACED: &str = "span_misplaced"; - /// 宾语既没在 entities 里声明、库里也没有叫这个名字的东西：事实照落， + /// 宾语既没在回复里列出、库里也没有叫这个名字的东西：陈述照落， /// 宾语落成字面值而不是节点（#559）。记下来是为了量：这一类里有多少 /// 本该是实体（模型漏报），有多少本来就是描述 pub const OBJECT_UNDECLARED: &str = "object_undeclared"; - /// 以下都来自 `utopia_extract::normalize`：只看结构、不看词的形状检查 - /// 值只有破折号（`—`）：表里的「无」，不落 - pub const NO_VALUE: &str = "no_value"; - /// 值后面有一截引文里没有的字：只留引文里有的那段。只记 - pub const VALUE_TRIMMED: &str = "value_trimmed"; - /// 没有宾语也没有值、只带边属性：属性落成主语上的值事实。只记 - pub const QUALIFIERS_WITHOUT_OBJECT: &str = "qualifiers_without_object"; - /// 宾语是契约格式的日期：数落成值、日期进有效期；没带数的把写出来的那段落成值。只记 - pub const TIME_AS_OBJECT: &str = "time_as_object"; - /// 主语是契约格式的日期：数是谁的回复里没说，不落 - pub const TIME_AS_SUBJECT: &str = "time_as_subject"; - /// 宾语名字包住另一个声明实体、同句已有指向本尊的边：可能是描述，也可能是另一个 - /// 东西（每股收益包住了净利润）。只记，不删 - pub const OBJECT_DESCRIBES_DECLARED: &str = "object_describes_declared"; - /// 上面几条去掉事实后没人引用的声明：不建 - pub const ORPHAN_DECLARATION: &str = "orphan_declaration"; - /// 引文抄自提示词里附的文件开头、不在这一块：证据会挂错出处，不落 - pub const QUOTE_FROM_OPENING: &str = "quote_from_opening"; - /// 以下三条来自开放抽取那条路（0044 第一刀，#729） /// 模型的引文在这一块里找不到原样的一句：陈述照落、引文照记，只是没有偏移 /// （`quote_start` / `quote_end` 留空）。这个信号数的是有多少条没定位到 pub const QUOTE_NOT_IN_CHUNK: &str = "quote_not_in_chunk"; diff --git a/crates/utopia-store/src/kbs.rs b/crates/utopia-store/src/kbs.rs index 14077b094..af2baec83 100644 --- a/crates/utopia-store/src/kbs.rs +++ b/crates/utopia-store/src/kbs.rs @@ -110,7 +110,6 @@ pub async fn update( auto_type_resolution: Option, governance: Option, data_conventions: Option<&str>, - open_extraction: Option, ) -> AppResult { // 改语言不回头重写已有的类——它们已经是这个库的数据，可能有人手工调过。 // 这一列往后管的是**新**描述（自动扩本体、AI 建议）写成什么语言 @@ -152,8 +151,6 @@ pub async fn update( ELSE governance_since END, -- 人写的约定；清空要送空串，送 null 等于不改（与 description 同一约定） data_conventions = COALESCE($11, data_conventions), - -- 开放抽取（0044 第一刀）：抽取只写开放图谱；关着走类型化那条路 - open_extraction = COALESCE($12, open_extraction), updated_at = now() WHERE id = $1 RETURNING *", ) @@ -168,7 +165,6 @@ pub async fn update( .bind(auto_type_resolution) .bind(governance) .bind(data_conventions) - .bind(open_extraction) .fetch_optional(pool) .await? .ok_or(AppError::NotFound) diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index 514a29ce7..10f7dd29b 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -261,7 +261,10 @@ pub async fn resolve_mention( WHERE (f.subject_id = e.id OR f.object_id = e.id) AND f.invalidated_at IS NULL AND {not_name}) AS degree FROM entities e - WHERE e.kb_id = $1 AND e.type_id = $2 AND e.merged_into IS NULL + -- IS NOT DISTINCT FROM 而不是 =（0009 的那个陷阱）：开放图谱里的实体都没有类 + -- （类由对齐来定），`type_id = NULL` 永远不成立，同名的它就永远撞不上—— + -- 实测一个库里 Securities and Exchange Commission 与它的全大写写法成了两个实体 + WHERE e.kb_id = $1 AND e.type_id IS NOT DISTINCT FROM $2 AND e.merged_into IS NULL -- 被描述的东西没有名字（0044）：它的 canonical_name 只是显示用的描述， -- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起 AND e.description IS NULL diff --git a/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs new file mode 100644 index 000000000..afce5bed7 --- /dev/null +++ b/crates/utopia-store/tests/an_untyped_name_meets_its_namesake.rs @@ -0,0 +1,78 @@ +//! 没有类的实体也要撞得上同名者（0044 第一刀之后的实测）。 +//! +//! 开放图谱里的实体都没有类：类由对齐来定，抽取只记文档的字。`resolve_mention` 的候选 +//! 查询从前写的是 `e.type_id = $2`，传 NULL 时这个条件永远不成立——于是同一个库里 +//! 「Securities and Exchange Commission」和它的全大写写法成了两个实体，只有建实体时 +//! 的精确同名匹配还兜得住。这里守的是：两次没有类的提及，写法只差大小写，落到同一个实体。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过。自建自拆，绝不碰已有的库。 + +use sqlx::PgPool; +use uuid::Uuid; + +#[tokio::test] +async fn an_untyped_mention_attaches_to_its_untyped_namesake() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'untyped-namesake-test')") + .bind(org) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'untyped-namesake-test')", + ) + .bind(ws) + .bind(org) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'untyped-namesake-test')", + ) + .bind(kb) + .bind(ws) + .execute(&pool) + .await?; + + let first = utopia_store::resolution::resolve_mention( + &pool, + kb, + None, + "Securities and Exchange Commission", + None, + None, + &[], + ) + .await?; + assert!(first.created, "第一次提及要建实体"); + let again = utopia_store::resolution::resolve_mention( + &pool, + kb, + None, + "SECURITIES AND EXCHANGE COMMISSION", + None, + None, + &[], + ) + .await?; + assert_eq!( + again.entity_id, first.entity_id, + "只差大小写的同名提及要落到同一个没有类的实体上" + ); + assert!(!again.created); + let n: i64 = sqlx::query_scalar( + "SELECT count(*) FROM entities WHERE kb_id = $1 AND merged_into IS NULL", + ) + .bind(kb) + .fetch_one(&pool) + .await?; + assert_eq!(n, 1); + + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + Ok(()) +} diff --git a/crates/utopia-store/tests/exploration_describes_the_data.rs b/crates/utopia-store/tests/exploration_describes_the_data.rs index fdf1dcc30..20d564eac 100644 --- a/crates/utopia-store/tests/exploration_describes_the_data.rs +++ b/crates/utopia-store/tests/exploration_describes_the_data.rs @@ -51,7 +51,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an None, None, Some("Amounts are in cents.\nis_test = 1 is excluded."), - None, ) .await?; // 探索写描述与问题 @@ -102,7 +101,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an None, None, Some("cents only"), - None, ) .await?; let got = utopia_store::kbs::get(&pool, kb).await?; @@ -121,7 +119,6 @@ async fn the_description_and_the_conventions_do_not_overwrite_each_other() -> an None, None, no, - None, ) .await?; assert_eq!( diff --git a/migrations/0063_extraction_reads_only_the_open_graph.sql b/migrations/0063_extraction_reads_only_the_open_graph.sql new file mode 100644 index 000000000..a648c5ee2 --- /dev/null +++ b/migrations/0063_extraction_reads_only_the_open_graph.sql @@ -0,0 +1,10 @@ +-- 抽取只写开放图谱（0044 决定 2）：开关退场。 +-- +-- 0061 加这一列时给带本体的老路留了一个关掉的口子——按当时的决定 3，它是已批准本体下 +-- 的可选第二路，等对齐追平就退场。现在定了：没有遗留层，也没有可选的类型化第二路。 +-- 每篇文档（记忆日志也在内）都走 `extraction_open::run_open`，本体不进提示词；类型化的 +-- 事实由对齐从开放陈述算出来，不由抽取写。一个再也没有第二个取值的开关不该留在表里： +-- 留着，读的人会以为还有一条路可以切回去。 +-- +-- 没有已发布的版本，直接删列；已有的 `layer = 'typed'` 行不动，它们是对齐的输入。 +ALTER TABLE knowledge_bases DROP COLUMN open_extraction; diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 7449ae1df..39a1cba21 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -65,13 +65,12 @@ node scripts/bench/fetch-sec-filings.mjs # 一次就够 node scripts/bench/recall.mjs --kb # 一轮：清空 → 重抽 → 打分 node scripts/bench/recall.mjs --kb --score # 只打分 node scripts/bench/recall.mjs --kb --reprocess # 改了解析器：连分块一起重来 -node scripts/bench/recall.mjs --kb --typed # 带本体的老路（可选第二路）；缺省走开放图谱（0044 第 1 刀） node scripts/bench/judge_open.mjs --kb --sample 200 # 开放陈述有多少不是原文说的（门槛 2%） ``` -缺省（库的 `open_extraction` 开着）抽的是开放图谱：陈述按原文短语落成 `layer='open'` 的事实行， -限定挂在 `statement_qualifiers`，时间词进 `time_mentions`；打分口径不变（谓词取 -`proposed_predicate`，限定也拼进那一行），所以 `--typed` 那条老路和它的分数直接可比。`judge_open.mjs` 是 +抽取写的是开放图谱（0044 决定 2，没有带本体的第二条路）：陈述按原文短语落成 `layer='open'` 的 +事实行，限定挂在 `statement_qualifiers`，时间词进 `time_mentions`；打分口径照旧（谓词取 +`proposed_predicate`，限定也拼进那一行），所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是 反面的尺子：抽样让一个裁判模型读原文判 stated / misworded / not_stated；裁判端点用 `BENCH_JUDGE_BASE / BENCH_JUDGE_KEY / BENCH_JUDGE_MODEL`，不给就用工作区的对话模型 （和抽取同一个模型，数字要打折看）。 diff --git a/scripts/bench/recall.mjs b/scripts/bench/recall.mjs index 18d93fc89..cc9f85b9f 100644 --- a/scripts/bench/recall.mjs +++ b/scripts/bench/recall.mjs @@ -16,7 +16,6 @@ // node scripts/bench/recall.mjs --kb # 已有库（本体向量已就绪） // node scripts/bench/recall.mjs --kb --score # 只打分，不重抽 // node scripts/bench/recall.mjs --kb --reprocess # 改了解析器：连分块一起重来 -// node scripts/bench/recall.mjs --kb --typed # 带本体的老路（可选第二路）；缺省是开放图谱（0044 第 1 刀） // // 环境变量：BENCH_BASE / BENCH_EMAIL / BENCH_PASSWORD / BENCH_PSQL（同 run.mjs）。 // @@ -212,8 +211,7 @@ psql(`DELETE FROM ontology_misses WHERE kb_id='${KB}'`); if (packTs) psql(`DELETE FROM relation_types WHERE kb_id='${KB}' AND created_at > '${packTs}'::timestamptz + interval '1 second'`); // 会在抽取之后改本体、增派生事实的开关关掉，两轮的本体与打分口径才一样。治理开着： // 库生下来就开着它（0050），量的是产品本来的样子 -psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE, open_extraction=${args.typed ? "FALSE" : "TRUE"} WHERE id='${KB}'`); -console.log(args.typed ? "带本体的老路：本体进提示词" : "开放图谱：本体不进提示词，陈述按原文短语落库"); +psql(`UPDATE knowledge_bases SET auto_extend_ontology=FALSE, auto_type_resolution=FALSE, materialize_inferences=FALSE, governance=TRUE WHERE id='${KB}'`); psql(`UPDATE chunks SET extracted_at=NULL WHERE document_id IN (SELECT id FROM documents WHERE kb_id='${KB}' AND filename IN (${names}))`); console.log(`本体 ${num(`SELECT count(*) FROM relation_types WHERE kb_id='${KB}'`)} 个关系 / ${num(`SELECT count(*) FROM entity_types WHERE kb_id='${KB}'`)} 个类`); diff --git a/scripts/bench/temporal.mjs b/scripts/bench/temporal.mjs index 9debb3301..4a7dc0a1b 100644 --- a/scripts/bench/temporal.mjs +++ b/scripts/bench/temporal.mjs @@ -437,6 +437,7 @@ async function acceptUniqueness(kb, axioms) { const byId = new Map(relation_types.map((r) => [r.id, r])); const rows = []; for (const c of candidates) { + // 候选来自类型化的事实（predicate_id）；开放陈述带的是 `phrase`、没有 predicate_id，对不上的跳过 const r = byId.get(c.predicate_id); if (!r) continue; const declared = wanted.get(c.key); diff --git a/web/src/api.ts b/web/src/api.ts index a193ae770..9f4a110aa 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -102,9 +102,6 @@ export interface Kb { /** 抽取遇到本体外的说法时，是否允许系统自动补进本体并改写等它的事实。 关掉不影响"留意"：未匹配统计照常累积可见，只是变成你点一下的提案 */ auto_extend_ontology: boolean; - /** 开放抽取（0044 第一刀）：开着，抽取只写开放图谱——陈述照文档的字落库， - * 不读本体；关着走今天的类型化那条路。缺省关；这一刀没有开关界面 */ - open_extraction: boolean; /** 把推出来的事实写进账本（R1）。**缺省关**——推理往图里加东西， * 而声明可能是错的，不该在用户没表态时就按它改图 */ materialize_inferences: boolean; diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index ba0421cca..a2a484509 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -503,20 +503,9 @@ export const en = { "Each line says why, and how many.", dropsExample: "e.g.", dropReason: { - attr_domain_mismatch: "Attribute on the wrong class", - subject_not_declared: "Subject type unknown", - attr_no_value: "Attribute had no value", - attr_datatype: "Value did not match the datatype", - low_confidence: "Below the confidence threshold", object_missing: "Relation had no object", malformed_item: "The model's item did not fit the schema", truncated_reply: "The model's reply was cut off", - domain_mismatch: - "The subject does not fit the relation, and swapping would not help", - not_an_entity_name: "That name is a sentence, not a thing", - clause_suspect: "Kept, but the name reads like a clause: a sample for the guard", - direction_corrected: - "Subject and object were swapped to match the signature", quote_not_in_chunk: "Kept, but the quoted sentence is not in the text verbatim", time_not_in_quote: "A time mention's words are not in the text", unknown_ref: "The item points at nothing in the reply", @@ -2023,7 +2012,6 @@ export const en = { conflictReason: { no_time: "new fact has no date", simultaneous: "same start date", - low_confidence: "low confidence", } as Record, conflictVs: "vs", conflictSince: (d: string) => `since ${d}`, diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 6b5c03ce7..ec8fe671d 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -454,18 +454,9 @@ export const zh: Strings = { "这些是从文档里抽出来的，但在写入途中被挡下了。每行说明原因与条数。", dropsExample: "例如", dropReason: { - attr_domain_mismatch: "属性挂在了错误的类上", - subject_not_declared: "主语类型未知", - attr_no_value: "属性没有取值", - attr_datatype: "取值与数据类型不符", - low_confidence: "低于置信度阈值", object_missing: "关系缺少宾语", malformed_item: "模型给的这一条结构不合", truncated_reply: "模型的输出被截断", - domain_mismatch: "主语对不上这个关系，对调也不行", - not_an_entity_name: "那个名字是一句话，不是一个东西", - clause_suspect: "已保留，但这个名字像从句：给守卫攒的样本", - direction_corrected: "已按签名把主宾掰正", quote_not_in_chunk: "已保留，但引文不是原文原样", time_not_in_quote: "时间词不在原文里", unknown_ref: "这一条指向的东西不在回复里", @@ -1764,7 +1755,6 @@ export const zh: Strings = { conflictReason: { no_time: "新事实没有日期", simultaneous: "起始日期相同", - low_confidence: "置信度低", }, conflictVs: "对", conflictSince: (d: string) => `自 ${d} 起`,