From fac5abdaa084d5069b6ea331f125b41ca94cf96b Mon Sep 17 00:00:00 2001 From: juhy0987 Date: Fri, 1 May 2026 21:16:50 +0900 Subject: [PATCH 1/3] =?UTF-8?q?[REFAC]:=20internal/parser=20=E2=86=92=20in?= =?UTF-8?q?ternal/processor/parser=20=EC=9D=B4=EB=8F=99=20(=EC=9D=B4?= =?UTF-8?q?=EC=8A=88=20#204,=20=EB=A9=94=ED=83=80=20#195=20=EC=99=84?= =?UTF-8?q?=EB=A3=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 이동: - internal/parser → internal/processor/parser - parser.go (도메인 중립 인터페이스) - rule/{,llmgen,pathinfer,refiner} - worker/ (Claim Check ParserWorker + RawContentCleaner) - 16개 .go 파일 import 경로 sed 일괄 치환 (issuetracker/internal/parser → issuetracker/internal/processor/parser) - internal/processor/fetcher/domain/general/types.go, internal/locks/processing_lock.go 의 doc 참조 갱신 - migrations/up/007_seed_parsing_rules.sql comment 갱신 테스트 mirror: test/internal/parser → test/internal/processor/parser 내부 동작 / 패키지명 / DB 스키마 / Kafka topic / Redis 동작 변경 없음 — 단순 path prefix 교체. git mv 로 history 보존. 28개 패키지 race test 모두 통과. 본 PR 머지 시 internal/parser/ 가 top-level 에서 사라지고, 모든 분리 worker 프로세스 (fetcher/parser/validate) 가 internal/processor// 하위로 일관 정렬됨. 메타 이슈 #195 완료. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/issuetracker/main.go | 8 ++++---- .../internal/{ => processor}/parser/README.md | 0 docs/architecture/internal/{ => processor}/parser/rule.md | 0 internal/locks/processing_lock.go | 4 ++-- internal/processor/fetcher/domain/general/convert.go | 2 +- internal/processor/fetcher/domain/general/types.go | 2 +- internal/{ => processor}/parser/parser.go | 0 internal/{ => processor}/parser/rule/discovery.go | 2 +- internal/{ => processor}/parser/rule/errors.go | 0 internal/{ => processor}/parser/rule/llmgen/dedup.go | 0 internal/{ => processor}/parser/rule/llmgen/generator.go | 2 +- internal/{ => processor}/parser/rule/llmgen/prompt.go | 0 internal/{ => processor}/parser/rule/parser.go | 2 +- internal/{ => processor}/parser/rule/pathinfer/llm.go | 0 .../{ => processor}/parser/rule/pathinfer/pathinfer.go | 0 .../{ => processor}/parser/rule/refiner/llm_adapter.go | 2 +- internal/{ => processor}/parser/rule/refiner/metrics.go | 0 internal/{ => processor}/parser/rule/refiner/refiner.go | 6 +++--- internal/{ => processor}/parser/rule/resolver.go | 0 internal/{ => processor}/parser/worker/cleanup.go | 0 internal/{ => processor}/parser/worker/parser_worker.go | 6 +++--- migrations/up/007_seed_parsing_rules.sql | 2 +- .../{ => processor}/parser/rule/discovery_test.go | 2 +- .../{ => processor}/parser/rule/llmgen/generator_test.go | 4 ++-- test/internal/{ => processor}/parser/rule/parser_test.go | 2 +- .../{ => processor}/parser/rule/pathinfer/llm_test.go | 2 +- .../parser/rule/pathinfer/pathinfer_test.go | 2 +- .../{ => processor}/parser/rule/refiner/refiner_test.go | 6 +++--- .../internal/{ => processor}/parser/rule/resolver_test.go | 2 +- .../{ => processor}/parser/worker/cleanup_test.go | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) rename docs/architecture/internal/{ => processor}/parser/README.md (100%) rename docs/architecture/internal/{ => processor}/parser/rule.md (100%) rename internal/{ => processor}/parser/parser.go (100%) rename internal/{ => processor}/parser/rule/discovery.go (99%) rename internal/{ => processor}/parser/rule/errors.go (100%) rename internal/{ => processor}/parser/rule/llmgen/dedup.go (100%) rename internal/{ => processor}/parser/rule/llmgen/generator.go (99%) rename internal/{ => processor}/parser/rule/llmgen/prompt.go (100%) rename internal/{ => processor}/parser/rule/parser.go (99%) rename internal/{ => processor}/parser/rule/pathinfer/llm.go (100%) rename internal/{ => processor}/parser/rule/pathinfer/pathinfer.go (100%) rename internal/{ => processor}/parser/rule/refiner/llm_adapter.go (97%) rename internal/{ => processor}/parser/rule/refiner/metrics.go (100%) rename internal/{ => processor}/parser/rule/refiner/refiner.go (98%) rename internal/{ => processor}/parser/rule/resolver.go (100%) rename internal/{ => processor}/parser/worker/cleanup.go (100%) rename internal/{ => processor}/parser/worker/parser_worker.go (99%) rename test/internal/{ => processor}/parser/rule/discovery_test.go (99%) rename test/internal/{ => processor}/parser/rule/llmgen/generator_test.go (99%) rename test/internal/{ => processor}/parser/rule/parser_test.go (99%) rename test/internal/{ => processor}/parser/rule/pathinfer/llm_test.go (99%) rename test/internal/{ => processor}/parser/rule/pathinfer/pathinfer_test.go (99%) rename test/internal/{ => processor}/parser/rule/refiner/refiner_test.go (99%) rename test/internal/{ => processor}/parser/rule/resolver_test.go (99%) rename test/internal/{ => processor}/parser/worker/cleanup_test.go (97%) diff --git a/cmd/issuetracker/main.go b/cmd/issuetracker/main.go index 3d822479..49bc28a4 100644 --- a/cmd/issuetracker/main.go +++ b/cmd/issuetracker/main.go @@ -11,15 +11,15 @@ import ( "github.com/prometheus/client_golang/prometheus" "issuetracker/internal/locks" - "issuetracker/internal/parser/rule" - "issuetracker/internal/parser/rule/llmgen" - "issuetracker/internal/parser/rule/refiner" - parserWorker "issuetracker/internal/parser/worker" "issuetracker/internal/processor/fetcher/core" "issuetracker/internal/processor/fetcher/domain/general/sources/kr" "issuetracker/internal/processor/fetcher/domain/general/sources/us" "issuetracker/internal/processor/fetcher/handler" crawlerWorker "issuetracker/internal/processor/fetcher/worker" + "issuetracker/internal/processor/parser/rule" + "issuetracker/internal/processor/parser/rule/llmgen" + "issuetracker/internal/processor/parser/rule/refiner" + parserWorker "issuetracker/internal/processor/parser/worker" "issuetracker/internal/processor/validate" "issuetracker/internal/publisher" "issuetracker/internal/scheduler" diff --git a/docs/architecture/internal/parser/README.md b/docs/architecture/internal/processor/parser/README.md similarity index 100% rename from docs/architecture/internal/parser/README.md rename to docs/architecture/internal/processor/parser/README.md diff --git a/docs/architecture/internal/parser/rule.md b/docs/architecture/internal/processor/parser/rule.md similarity index 100% rename from docs/architecture/internal/parser/rule.md rename to docs/architecture/internal/processor/parser/rule.md diff --git a/internal/locks/processing_lock.go b/internal/locks/processing_lock.go index f2e40c20..c3e2254c 100644 --- a/internal/locks/processing_lock.go +++ b/internal/locks/processing_lock.go @@ -42,7 +42,7 @@ type ProcessingLock interface { // ProcessingLock stage 표준 상수 — 호출자가 stage 를 직접 string literal 로 쓰지 않고 // 이 상수를 통해 일관된 키를 만들도록 강제 (typo 방지 + 검색 용이). // -// 외부 패키지 (internal/parser/worker, internal/processor/validate) 도 import 후 사용. +// 외부 패키지 (internal/processor/parser/worker, internal/processor/validate) 도 import 후 사용. const ( StageFetcher = "fetcher" StageParser = "parser" @@ -56,7 +56,7 @@ const ( // // stage 표준 값 (StageFetcher / StageParser / StageValidator 상수를 통해 일관된 키 생성 보장): // - "fetcher" — crawler worker (raw HTML fetch, internal/processor/fetcher/worker) -// - "parser" — parser worker (rule.Parser.ParsePage / ParseLinks, internal/parser/worker) +// - "parser" — parser worker (rule.Parser.ParsePage / ParseLinks, internal/processor/parser/worker) // - "validator" — validate worker (Content 검증, internal/processor/validate) // // 외부 패키지도 본 worker 패키지를 import 하여 동일 상수와 ProcessingKey 헬퍼를 사용 — diff --git a/internal/processor/fetcher/domain/general/convert.go b/internal/processor/fetcher/domain/general/convert.go index 2d3dcfa5..88e8f708 100644 --- a/internal/processor/fetcher/domain/general/convert.go +++ b/internal/processor/fetcher/domain/general/convert.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "issuetracker/internal/parser" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser" ) // ConvertPage 는 parser.Page + RawContent 를 core.Content 로 변환합니다. diff --git a/internal/processor/fetcher/domain/general/types.go b/internal/processor/fetcher/domain/general/types.go index 282954c4..7b2c8f30 100644 --- a/internal/processor/fetcher/domain/general/types.go +++ b/internal/processor/fetcher/domain/general/types.go @@ -3,7 +3,7 @@ // // Package general provides domain-neutral abstractions for crawling arbitrary web pages // (news, blogs, product pages, generic documents). 기존 news 도메인을 흡수합니다 — -// 모든 사이트는 본 패키지의 인터페이스를 구현하고, 파싱은 internal/parser/rule +// 모든 사이트는 본 패키지의 인터페이스를 구현하고, 파싱은 internal/processor/parser/rule // 의 단일 DB 규칙 기반 엔진이 처리합니다. // // Source-specific parsers (NaverParser/CNNParser/...) 는 더 이상 존재하지 않습니다. diff --git a/internal/parser/parser.go b/internal/processor/parser/parser.go similarity index 100% rename from internal/parser/parser.go rename to internal/processor/parser/parser.go diff --git a/internal/parser/rule/discovery.go b/internal/processor/parser/rule/discovery.go similarity index 99% rename from internal/parser/rule/discovery.go rename to internal/processor/parser/rule/discovery.go index 85bffae0..effaa626 100644 --- a/internal/parser/rule/discovery.go +++ b/internal/processor/parser/rule/discovery.go @@ -6,8 +6,8 @@ import ( "net/url" "regexp" - "issuetracker/internal/parser" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser" "issuetracker/internal/storage" "issuetracker/pkg/links" ) diff --git a/internal/parser/rule/errors.go b/internal/processor/parser/rule/errors.go similarity index 100% rename from internal/parser/rule/errors.go rename to internal/processor/parser/rule/errors.go diff --git a/internal/parser/rule/llmgen/dedup.go b/internal/processor/parser/rule/llmgen/dedup.go similarity index 100% rename from internal/parser/rule/llmgen/dedup.go rename to internal/processor/parser/rule/llmgen/dedup.go diff --git a/internal/parser/rule/llmgen/generator.go b/internal/processor/parser/rule/llmgen/generator.go similarity index 99% rename from internal/parser/rule/llmgen/generator.go rename to internal/processor/parser/rule/llmgen/generator.go index efddce6f..a9152865 100644 --- a/internal/parser/rule/llmgen/generator.go +++ b/internal/processor/parser/rule/llmgen/generator.go @@ -33,8 +33,8 @@ import ( "github.com/PuerkitoBio/goquery" - "issuetracker/internal/parser/rule" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser/rule" "issuetracker/internal/storage" "issuetracker/pkg/llm" "issuetracker/pkg/logger" diff --git a/internal/parser/rule/llmgen/prompt.go b/internal/processor/parser/rule/llmgen/prompt.go similarity index 100% rename from internal/parser/rule/llmgen/prompt.go rename to internal/processor/parser/rule/llmgen/prompt.go diff --git a/internal/parser/rule/parser.go b/internal/processor/parser/rule/parser.go similarity index 99% rename from internal/parser/rule/parser.go rename to internal/processor/parser/rule/parser.go index 0c88de76..32596206 100644 --- a/internal/parser/rule/parser.go +++ b/internal/processor/parser/rule/parser.go @@ -9,8 +9,8 @@ import ( "github.com/PuerkitoBio/goquery" - "issuetracker/internal/parser" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser" "issuetracker/internal/storage" ) diff --git a/internal/parser/rule/pathinfer/llm.go b/internal/processor/parser/rule/pathinfer/llm.go similarity index 100% rename from internal/parser/rule/pathinfer/llm.go rename to internal/processor/parser/rule/pathinfer/llm.go diff --git a/internal/parser/rule/pathinfer/pathinfer.go b/internal/processor/parser/rule/pathinfer/pathinfer.go similarity index 100% rename from internal/parser/rule/pathinfer/pathinfer.go rename to internal/processor/parser/rule/pathinfer/pathinfer.go diff --git a/internal/parser/rule/refiner/llm_adapter.go b/internal/processor/parser/rule/refiner/llm_adapter.go similarity index 97% rename from internal/parser/rule/refiner/llm_adapter.go rename to internal/processor/parser/rule/refiner/llm_adapter.go index f3c5770c..92875fcc 100644 --- a/internal/parser/rule/refiner/llm_adapter.go +++ b/internal/processor/parser/rule/refiner/llm_adapter.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "issuetracker/internal/parser/rule/pathinfer" + "issuetracker/internal/processor/parser/rule/pathinfer" "issuetracker/pkg/llm" ) diff --git a/internal/parser/rule/refiner/metrics.go b/internal/processor/parser/rule/refiner/metrics.go similarity index 100% rename from internal/parser/rule/refiner/metrics.go rename to internal/processor/parser/rule/refiner/metrics.go diff --git a/internal/parser/rule/refiner/refiner.go b/internal/processor/parser/rule/refiner/refiner.go similarity index 98% rename from internal/parser/rule/refiner/refiner.go rename to internal/processor/parser/rule/refiner/refiner.go index eede82fb..8cacbb5e 100644 --- a/internal/parser/rule/refiner/refiner.go +++ b/internal/processor/parser/rule/refiner/refiner.go @@ -29,9 +29,9 @@ import ( "sync/atomic" "time" - "issuetracker/internal/parser/rule" - "issuetracker/internal/parser/rule/llmgen" - "issuetracker/internal/parser/rule/pathinfer" + "issuetracker/internal/processor/parser/rule" + "issuetracker/internal/processor/parser/rule/llmgen" + "issuetracker/internal/processor/parser/rule/pathinfer" "issuetracker/internal/storage" "issuetracker/pkg/logger" ) diff --git a/internal/parser/rule/resolver.go b/internal/processor/parser/rule/resolver.go similarity index 100% rename from internal/parser/rule/resolver.go rename to internal/processor/parser/rule/resolver.go diff --git a/internal/parser/worker/cleanup.go b/internal/processor/parser/worker/cleanup.go similarity index 100% rename from internal/parser/worker/cleanup.go rename to internal/processor/parser/worker/cleanup.go diff --git a/internal/parser/worker/parser_worker.go b/internal/processor/parser/worker/parser_worker.go similarity index 99% rename from internal/parser/worker/parser_worker.go rename to internal/processor/parser/worker/parser_worker.go index 59bab808..0a770382 100644 --- a/internal/parser/worker/parser_worker.go +++ b/internal/processor/parser/worker/parser_worker.go @@ -26,11 +26,11 @@ import ( "time" "issuetracker/internal/locks" - "issuetracker/internal/parser" - "issuetracker/internal/parser/rule" - "issuetracker/internal/parser/rule/llmgen" "issuetracker/internal/processor/fetcher/core" "issuetracker/internal/processor/fetcher/domain/general" + "issuetracker/internal/processor/parser" + "issuetracker/internal/processor/parser/rule" + "issuetracker/internal/processor/parser/rule/llmgen" "issuetracker/internal/storage" "issuetracker/internal/storage/service" "issuetracker/pkg/logger" diff --git a/migrations/up/007_seed_parsing_rules.sql b/migrations/up/007_seed_parsing_rules.sql index bc56ecee..3d0f937c 100644 --- a/migrations/up/007_seed_parsing_rules.sql +++ b/migrations/up/007_seed_parsing_rules.sql @@ -2,7 +2,7 @@ -- parsing_rules row 로 이전 (이슈 #100 / #139 follow-up). -- -- 본 migration 적용 후, 사이트별 parser.go (NaverParser/DaumParser/YonhapParser/CNNParser) 가 --- 제거되고 internal/parser/rule.Parser 가 단일 엔진으로 동작합니다. +-- 제거되고 internal/processor/parser/rule.Parser 가 단일 엔진으로 동작합니다. -- -- 운영자 노트: -- - 사이트가 selector 를 바꾼 경우 본 row 의 selectors JSONB 만 UPDATE 하면 된다 (재배포 X). diff --git a/test/internal/parser/rule/discovery_test.go b/test/internal/processor/parser/rule/discovery_test.go similarity index 99% rename from test/internal/parser/rule/discovery_test.go rename to test/internal/processor/parser/rule/discovery_test.go index 87d50e09..4e993436 100644 --- a/test/internal/parser/rule/discovery_test.go +++ b/test/internal/processor/parser/rule/discovery_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule" + "issuetracker/internal/processor/parser/rule" "issuetracker/internal/storage" ) diff --git a/test/internal/parser/rule/llmgen/generator_test.go b/test/internal/processor/parser/rule/llmgen/generator_test.go similarity index 99% rename from test/internal/parser/rule/llmgen/generator_test.go rename to test/internal/processor/parser/rule/llmgen/generator_test.go index 90e9bdd3..bfc4fc8c 100644 --- a/test/internal/parser/rule/llmgen/generator_test.go +++ b/test/internal/processor/parser/rule/llmgen/generator_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule" - "issuetracker/internal/parser/rule/llmgen" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser/rule" + "issuetracker/internal/processor/parser/rule/llmgen" "issuetracker/internal/storage" "issuetracker/pkg/llm" "issuetracker/pkg/logger" diff --git a/test/internal/parser/rule/parser_test.go b/test/internal/processor/parser/rule/parser_test.go similarity index 99% rename from test/internal/parser/rule/parser_test.go rename to test/internal/processor/parser/rule/parser_test.go index 67188b9e..b40d1263 100644 --- a/test/internal/parser/rule/parser_test.go +++ b/test/internal/processor/parser/rule/parser_test.go @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule" "issuetracker/internal/processor/fetcher/core" + "issuetracker/internal/processor/parser/rule" "issuetracker/internal/storage" ) diff --git a/test/internal/parser/rule/pathinfer/llm_test.go b/test/internal/processor/parser/rule/pathinfer/llm_test.go similarity index 99% rename from test/internal/parser/rule/pathinfer/llm_test.go rename to test/internal/processor/parser/rule/pathinfer/llm_test.go index c2a894a4..ccd4ef2f 100644 --- a/test/internal/parser/rule/pathinfer/llm_test.go +++ b/test/internal/processor/parser/rule/pathinfer/llm_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule/pathinfer" + "issuetracker/internal/processor/parser/rule/pathinfer" ) // ───────────────────────────────────────────────────────────────────────────── diff --git a/test/internal/parser/rule/pathinfer/pathinfer_test.go b/test/internal/processor/parser/rule/pathinfer/pathinfer_test.go similarity index 99% rename from test/internal/parser/rule/pathinfer/pathinfer_test.go rename to test/internal/processor/parser/rule/pathinfer/pathinfer_test.go index 6408bc13..00d4c69d 100644 --- a/test/internal/parser/rule/pathinfer/pathinfer_test.go +++ b/test/internal/processor/parser/rule/pathinfer/pathinfer_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule/pathinfer" + "issuetracker/internal/processor/parser/rule/pathinfer" ) // 결과 regex 가 입력 sample 들 모두 매칭하는지 확인하는 헬퍼. diff --git a/test/internal/parser/rule/refiner/refiner_test.go b/test/internal/processor/parser/rule/refiner/refiner_test.go similarity index 99% rename from test/internal/parser/rule/refiner/refiner_test.go rename to test/internal/processor/parser/rule/refiner/refiner_test.go index 890b2896..cef99f8c 100644 --- a/test/internal/parser/rule/refiner/refiner_test.go +++ b/test/internal/processor/parser/rule/refiner/refiner_test.go @@ -14,9 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule" - "issuetracker/internal/parser/rule/llmgen" - "issuetracker/internal/parser/rule/refiner" + "issuetracker/internal/processor/parser/rule" + "issuetracker/internal/processor/parser/rule/llmgen" + "issuetracker/internal/processor/parser/rule/refiner" "issuetracker/internal/storage" "issuetracker/pkg/logger" ) diff --git a/test/internal/parser/rule/resolver_test.go b/test/internal/processor/parser/rule/resolver_test.go similarity index 99% rename from test/internal/parser/rule/resolver_test.go rename to test/internal/processor/parser/rule/resolver_test.go index cdd6ae99..ec17fbf9 100644 --- a/test/internal/parser/rule/resolver_test.go +++ b/test/internal/processor/parser/rule/resolver_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "issuetracker/internal/parser/rule" + "issuetracker/internal/processor/parser/rule" "issuetracker/internal/storage" ) diff --git a/test/internal/parser/worker/cleanup_test.go b/test/internal/processor/parser/worker/cleanup_test.go similarity index 97% rename from test/internal/parser/worker/cleanup_test.go rename to test/internal/processor/parser/worker/cleanup_test.go index c506378b..81e87e3b 100644 --- a/test/internal/parser/worker/cleanup_test.go +++ b/test/internal/processor/parser/worker/cleanup_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - parserWorker "issuetracker/internal/parser/worker" "issuetracker/internal/processor/fetcher/core" + parserWorker "issuetracker/internal/processor/parser/worker" "issuetracker/internal/storage" "issuetracker/pkg/logger" ) From 24e5a0bd0f3d05e2f6f0012070f869a475d4453c Mon Sep 17 00:00:00 2001 From: juhy0987 Date: Fri, 1 May 2026 21:17:02 +0900 Subject: [PATCH 2/3] =?UTF-8?q?[DOCS]:=20parser=20=E2=86=92=20processor/pa?= =?UTF-8?q?rser=20=EC=9D=B4=EB=8F=99=EC=97=90=20=EB=A7=9E=EC=B6=B0=20?= =?UTF-8?q?=EB=AC=B8=EC=84=9C=20=EA=B0=B1=EC=8B=A0=20(=EC=9D=B4=EC=8A=88?= =?UTF-8?q?=20#204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docs/architecture/internal/parser/ → docs/architecture/internal/processor/parser/ (디렉토리 이동) - 모든 .md 의 path 참조 갱신 (internal/parser → internal/processor/parser) - 이동된 docs 의 sibling-link 깊이 +1 보정 (../locks/ → ../../locks/, 등) - 다른 docs 의 inverse cross-link 갱신: - internal/processor/fetcher/{domain,core,README}.md 의 ../../parser/ → ../parser/ (sibling) - internal/storage/service.md / internal/locks/README.md 의 ../parser/ → ../processor/parser/ - publisher.md / internal/README.md 의 (parser/README.md) → (processor/parser/README.md) - docs/architecture/README.md 디렉토리 트리 — parser 를 processor 하위로 이동 - .claude/rules/01-architecture.md 디렉토리 트리 동일 정렬 + test mirror 갱신 - CLAUDE.md cmd/ 줄 — 실제 바이너리 목록 반영 (#198 PR 에서 누락된 분 함께 commit) Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/rules/01-architecture.md | 19 ++++---- CLAUDE.md | 2 +- docs/architecture/README.md | 14 +++--- docs/architecture/cmd/issuetracker.md | 8 ++-- docs/architecture/internal/README.md | 2 +- docs/architecture/internal/locks/README.md | 2 +- .../internal/processor/fetcher/README.md | 4 +- .../internal/processor/fetcher/core.md | 4 +- .../internal/processor/fetcher/domain.md | 2 +- .../internal/processor/parser/README.md | 24 +++++----- .../internal/processor/parser/rule.md | 46 +++++++++---------- docs/architecture/internal/publisher.md | 2 +- docs/architecture/internal/storage/service.md | 4 +- docs/architecture/pkg/config.md | 2 +- docs/architecture/pkg/links.md | 2 +- docs/architecture/pkg/llm.md | 4 +- docs/architecture/pkg/metrics.md | 4 +- docs/architecture/pkg/queue.md | 2 +- 18 files changed, 73 insertions(+), 74 deletions(-) diff --git a/.claude/rules/01-architecture.md b/.claude/rules/01-architecture.md index 30be1050..149de0e2 100644 --- a/.claude/rules/01-architecture.md +++ b/.claude/rules/01-architecture.md @@ -104,17 +104,10 @@ issuetracker/ │ └── main.go │ ├── internal/ # Private application code -│ ├── parser/ # ✅ Domain-agnostic parser + DB-driven rule engine (이슈 #100, #196) -│ │ ├── parser.go # ContentParser / LinkListParser interfaces + Page model -│ │ ├── rule/ # parsing_rules 기반 단일 engine -│ │ │ ├── llmgen/ # LLM 기반 selector 자동 생성 (이슈 #149) -│ │ │ ├── pathinfer/ # path_pattern 추론 알고리즘 (이슈 #173) -│ │ │ └── refiner/ # path_pattern 정밀화 polling -│ │ └── worker/ # Claim Check 기반 ParserWorker (Kafka consumer) │ ├── locks/ # ✅ 단계 무관 distributed lock — fetcher/parser/validator 공유 (이슈 #197) │ │ ├── ingestion_lock.go # IngestionLock (Publisher 가 Kafka enqueue 직전 사용) │ │ └── processing_lock.go # ProcessingLock + ProcessingKey(stage, url) -│ ├── processor/ # ✅ 파이프라인 단계별 정렬 (이슈 #195) +│ ├── processor/ # ✅ 파이프라인 단계별 정렬 — 모든 stage 가 본 디렉토리 하위 (이슈 #195) │ │ ├── fetcher/ # ✅ Web fetch + DB-driven parse 라우팅 + worker pool (이슈 #198) │ │ │ ├── core/ # 인터페이스 + 모델 + 에러 + HTTP client + retry │ │ │ ├── handler/ # crawler_name → Handler registry @@ -122,6 +115,13 @@ issuetracker/ │ │ │ ├── domain/ # 사이트 chain handler + 사이트별 등록 (sources/) │ │ │ ├── rate_limiter/ # IP 단위 token bucket │ │ │ └── worker/ # PoolManager + KafkaConsumerPool + RetryScheduler + CircuitBreaker +│ │ ├── parser/ # ✅ Domain-agnostic parser + DB-driven rule engine + ParserWorker (이슈 #100, #196, #204) +│ │ │ ├── parser.go # ContentParser / LinkListParser interfaces + Page model +│ │ │ ├── rule/ # parsing_rules 기반 단일 engine +│ │ │ │ ├── llmgen/ # LLM 기반 selector 자동 생성 (이슈 #149) +│ │ │ │ ├── pathinfer/ # path_pattern 추론 알고리즘 (이슈 #173) +│ │ │ │ └── refiner/ # path_pattern 정밀화 polling +│ │ │ └── worker/ # Claim Check 기반 ParserWorker (Kafka consumer) + RawContentCleaner │ │ └── validate/ # Validate worker (Validation logic) │ ├── embedding/ # Embedding & ML (planned) │ │ ├── model/ # Embedding models @@ -142,8 +142,7 @@ issuetracker/ │ ├── internal/ # ✅ internal/ 패키지 테스트 │ │ ├── classifier/ # ← internal/classifier/ │ │ ├── locks/ # ← internal/locks/ -│ │ ├── parser/ # ← internal/parser/ -│ │ ├── processor/ # ← internal/processor/ (fetcher/{core,worker,...} + validate) +│ │ ├── processor/ # ← internal/processor/ (fetcher/{core,worker,...} + parser/{rule,worker} + validate) │ │ └── storage/ # ← internal/storage/ │ └── pkg/ # ✅ pkg/ 패키지 테스트 │ ├── config/ # ← pkg/config/ diff --git a/CLAUDE.md b/CLAUDE.md index 6bc82484..b6423fd4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ make fmt # gofmt ## 디렉토리 구조 (요약) ``` -cmd/ → 실행 바이너리 (crawler, processor, issuetracker, migrate) +cmd/ → 실행 바이너리 (issuetracker, processor, api, migrate, migrate-down, rldebug) internal/ → 비공개 비즈니스 로직 pkg/ → 공개 유틸리티 (logger, config, queue, redis) test/ → 테스트 (internal/, pkg/ 미러링) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 7db58965..6d3737f9 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -37,9 +37,6 @@ docs/architecture/ │ │ └── http.md │ ├── locks/ ← 단계 무관 distributed lock (이슈 #197) │ │ └── README.md ← ProcessingLock + IngestionLock (Redis SETNX) -│ ├── parser/ -│ │ ├── README.md ← Domain-Agnostic Parser + Claim Check Worker -│ │ └── rule.md ← rule.Parser (DB-driven) + llmgen + pathinfer + refiner │ ├── processor/ ← 파이프라인 단계별 정렬 (이슈 #195) │ │ ├── README.md │ │ ├── fetcher/ ← Web fetch + DB-driven parse + rate limit + worker pool (이슈 #198) @@ -50,6 +47,9 @@ docs/architecture/ │ │ │ ├── implementation.md ← chromedp / goquery │ │ │ ├── rate_limiter.md │ │ │ └── worker.md ← PoolManager + RetryScheduler + CircuitBreaker +│ │ ├── parser/ ← Domain-Agnostic Parser + Claim Check Worker (이슈 #204) +│ │ │ ├── README.md +│ │ │ └── rule.md ← rule.Parser (DB-driven) + llmgen + pathinfer + refiner │ │ └── validate.md ← news/community Validator │ ├── publisher.md ← chained job 발행 + IngestionLock │ ├── scheduler.md ← seed job 주기적 발행 @@ -108,7 +108,7 @@ docs/architecture/ ▼ ┌─────────────────────┐ rule.Parser (DB-driven, parsing_rules 테이블) │ ParserWorker │ ├─ ErrNoRule → llmgen.Generator (async, enabled=false) -│ internal/parser/ │ └─ list 페이지 → Publisher.PublishBatch (chained CrawlJob) +│ internal/processor/parser/ │ └─ list 페이지 → Publisher.PublishBatch (chained CrawlJob) │ worker │ └──────┬──────────────┘ │ Content → ContentService.Store → ContentRef @@ -130,7 +130,7 @@ docs/architecture/ └──────────────────────────────────────────────────────────────────────────┘ 부수 흐름: - • Refiner (internal/parser/rule/refiner) — 주기적 polling 으로 llm-auto 규칙의 + • Refiner (internal/processor/parser/rule/refiner) — 주기적 polling 으로 llm-auto 규칙의 path_pattern 을 sample_urls 기반으로 정밀화 (이슈 #173). • Classifier (internal/classifier) — gRPC 기본 + HTTP fallback 으로 ELArchive Classifier 호출 — 향후 enrich 단계에서 사용. @@ -149,7 +149,7 @@ docs/architecture/ | `issuetracker.crawl.normal` | [scheduler](../../internal/scheduler/) / publisher | `issuetracker-crawler-workers` (동일) | | `issuetracker.crawl.low` | [scheduler](../../internal/scheduler/) / publisher | `issuetracker-crawler-workers` (동일) | | `issuetracker.fetched` | [processor/fetcher/worker](../../internal/processor/fetcher/worker/) | `issuetracker-parsers` (`GroupParsers`) | -| `issuetracker.normalized` | [parser/worker](../../internal/parser/worker/) | `issuetracker-validators` (`GroupValidators`) | +| `issuetracker.normalized` | [parser/worker](../../internal/processor/parser/worker/) | `issuetracker-validators` (`GroupValidators`) | | `issuetracker.validated` | [processor/validate](../../internal/processor/validate/) | (downstream — TBD) | | `issuetracker.dlq` | 모든 stage 실패 분기 | (운영 모니터링) | @@ -196,7 +196,7 @@ docs/architecture/ 1. [docs/architecture/cmd/issuetracker.md](cmd/issuetracker.md) — `main()` 가 어떤 모듈을 어떤 순서로 wire 하는지 2. [docs/architecture/internal/processor/fetcher/worker.md](internal/processor/fetcher/worker.md) — Kafka consumer + lock + retry 의 핵심 패턴 -3. [docs/architecture/internal/parser/README.md](internal/parser/README.md) — Claim Check + rule-driven 파싱 +3. [docs/architecture/internal/processor/parser/README.md](internal/processor/parser/README.md) — Claim Check + rule-driven 파싱 4. [docs/architecture/internal/storage/README.md](internal/storage/README.md) — Repository → Service 분리 원칙 5. [docs/architecture/pkg/queue.md](pkg/queue.md) — 토픽 상수와 Kafka producer/consumer 추상 diff --git a/docs/architecture/cmd/issuetracker.md b/docs/architecture/cmd/issuetracker.md index 9e6f08c2..517c3147 100644 --- a/docs/architecture/cmd/issuetracker.md +++ b/docs/architecture/cmd/issuetracker.md @@ -40,8 +40,8 @@ - [internal/processor/fetcher/worker.md](../internal/processor/fetcher/worker.md) — PoolManager, RetryScheduler - [internal/locks/README.md](../internal/locks/README.md) — ProcessingLock, IngestionLock -- [internal/parser/rule.md](../internal/parser/rule.md) — rule.Parser, llmgen, refiner -- [internal/parser/README.md](../internal/parser/README.md) — ParserWorker (Claim Check) +- [internal/processor/parser/rule.md](../internal/processor/parser/rule.md) — rule.Parser, llmgen, refiner +- [internal/processor/parser/README.md](../internal/processor/parser/README.md) — ParserWorker (Claim Check) - [internal/processor/validate.md](../internal/processor/validate.md) — Validator - [internal/scheduler.md](../internal/scheduler.md) — seed job 발행 @@ -52,8 +52,8 @@ | 함수 | 책임 | 실패 동작 | |--------------------------|------------------------------------------------------------|------------------| | `buildLLMProvider()` | `LLM_ENABLED` + API key 검증 → [`pkg/llm`](../../../pkg/llm/) chain provider 구성 | nil 반환 (LLM 비활성) | -| `buildLLMGenerator()` | provider nil 이면 nil — 아니면 [`llmgen.New`](../../../internal/parser/rule/llmgen/) | nil 허용 | -| `buildRefiner()` | `REFINEMENT_ENABLED` + provider 옵션 결합 → [`refiner.New`](../../../internal/parser/rule/refiner/) | nil 반환 (정밀화 비활성) | +| `buildLLMGenerator()` | provider nil 이면 nil — 아니면 [`llmgen.New`](../../../internal/processor/parser/rule/llmgen/) | nil 허용 | +| `buildRefiner()` | `REFINEMENT_ENABLED` + provider 옵션 결합 → [`refiner.New`](../../../internal/processor/parser/rule/refiner/) | nil 반환 (정밀화 비활성) | | `verifyParsingRulesSeeded()` | 기본 파싱 규칙(seed data) 이 DB 에 정상 로드됐는지 검증 | **fail-fast (Fatal)** |
diff --git a/docs/architecture/internal/README.md b/docs/architecture/internal/README.md index b31b5219..6c8e7513 100644 --- a/docs/architecture/internal/README.md +++ b/docs/architecture/internal/README.md @@ -14,7 +14,7 @@ Go 의 `internal` 규칙에 의해 외부 모듈에서 import 할 수 없으므 | 디렉토리 | 역할 | 문서 | |-----------------------------------------------|-----------------------------------------------------------|-----------------------------------------------------------| | [`internal/processor/fetcher/`](../../../internal/processor/fetcher/) | 웹 크롤링 — fetch + DB-driven parse + rate limit + worker pool | [processor/fetcher/README.md](processor/fetcher/README.md) | -| [`internal/parser/`](../../../internal/parser/) | Claim Check 기반 ParserWorker (TopicFetched 소비) | [parser/README.md](parser/README.md) | +| [`internal/processor/parser/`](../../../internal/processor/parser/) | Claim Check 기반 ParserWorker (TopicFetched 소비) | [parser/README.md](processor/parser/README.md) | | [`internal/processor/`](../../../internal/processor/) | 검증 stage (news/community Validator) | [processor/README.md](processor/README.md) | | [`internal/classifier/`](../../../internal/classifier/) | ELArchive Classifier client (gRPC + HTTP fallback) | [classifier/README.md](classifier/README.md) | | [`internal/publisher/`](../../../internal/publisher/) | chained CrawlJob 발행 + IngestionLock | [publisher.md](publisher.md) | diff --git a/docs/architecture/internal/locks/README.md b/docs/architecture/internal/locks/README.md index 9ae0fd86..d25f5f21 100644 --- a/docs/architecture/internal/locks/README.md +++ b/docs/architecture/internal/locks/README.md @@ -57,7 +57,7 @@ if !acquired { - [`internal/publisher`](../publisher.md) — `IngestionLock` (Kafka enqueue 직전 dedup) - [`internal/processor/fetcher/worker`](../processor/fetcher/worker.md) — fetcher worker pool 의 `ProcessingLock(StageFetcher)` -- [`internal/parser/worker`](../parser/README.md) — parser worker 의 `ProcessingLock(StageParser)` +- [`internal/processor/parser/worker`](../processor/parser/README.md) — parser worker 의 `ProcessingLock(StageParser)` - [`internal/processor/validate`](../processor/validate.md) — validator 의 `ProcessingLock(StageValidator)` ## 외부 시스템 diff --git a/docs/architecture/internal/processor/fetcher/README.md b/docs/architecture/internal/processor/fetcher/README.md index d8916e06..40c3164c 100644 --- a/docs/architecture/internal/processor/fetcher/README.md +++ b/docs/architecture/internal/processor/fetcher/README.md @@ -50,7 +50,7 @@ worker ──→ handler ──→ (domain/general → fetcher) ──→ implem └──→ core (interfaces, models, errors) ──── 모든 패키지가 의존 ``` -> Parser engine 은 별도 패키지 (`internal/parser/`) 로 분리됨 (이슈 #196). 상세는 [`internal/parser/README.md`](../../parser/README.md). +> Parser engine 은 별도 패키지 (`internal/processor/parser/`) 로 분리됨 (이슈 #196). 상세는 [`internal/processor/parser/README.md`](../parser/README.md). `core` 가 sink 노드 (다른 어떤 서브패키지도 의존하지 않음 — 가장 안정적인 인터페이스 계층). @@ -83,7 +83,7 @@ worker ──→ handler ──→ (domain/general → fetcher) ──→ implem 8. RawContentRef 를 issuetracker.fetched 토픽에 발행 │ ▼ - (parser stage 로 핸드오프 — internal/parser/README.md) + (parser stage 로 핸드오프 — internal/processor/parser/README.md) ``` 각 단계의 자세한 책임은 위 패키지별 문서 참조. diff --git a/docs/architecture/internal/processor/fetcher/core.md b/docs/architecture/internal/processor/fetcher/core.md index 9364e4d8..6bec2232 100644 --- a/docs/architecture/internal/processor/fetcher/core.md +++ b/docs/architecture/internal/processor/fetcher/core.md @@ -12,7 +12,7 @@ | 인터페이스 | 위치 | 책임 | |--------------------|------------------------------------------------------------------|-------------------------------------------------------| | `Crawler` | [crawler.go](../../../../internal/processor/fetcher/core/crawler.go) | Initialize / Start / Stop / Fetch / HealthCheck | -| `Parser` | (동일 파일) | RawContent → 도메인 모델 (별도 [parser](../../parser/rule.md) 가 구현) | +| `Parser` | (동일 파일) | RawContent → 도메인 모델 (별도 [parser](../parser/rule.md) 가 구현) | | `RateLimiter` | (동일 파일) | Wait(ctx) — 토큰 buckets 등으로 호출 빈도 제한 | | `URLRateLimiter` | (동일 파일) | URL → IP 매핑 + per-IP rate limit (DNS 해석 필요) | | `HTTPClient` | [http_client.go](../../../../internal/processor/fetcher/core/http_client.go) | Do(req) — 표준 `*http.Client` wrapper | @@ -61,7 +61,7 @@ ## 의존 관계 - **누구도 import 하지 않는** 외부 의존 외엔 standard lib + zerolog 정도 -- 본 패키지를 import 하는 패키지: `domain/`, `handler/`, `worker/`, `parser/`, `rate_limiter/`, 그리고 [`internal/parser`](../../parser/README.md), [`internal/scheduler`](../../scheduler.md), [`internal/storage/service`](../../storage/service.md) 등 — 사실상 거의 모든 곳 +- 본 패키지를 import 하는 패키지: `domain/`, `handler/`, `worker/`, `parser/`, `rate_limiter/`, 그리고 [`internal/processor/parser`](../parser/README.md), [`internal/scheduler`](../../scheduler.md), [`internal/storage/service`](../../storage/service.md) 등 — 사실상 거의 모든 곳 본 패키지가 변경되면 광범위한 영향이 있으므로 신중히. 새 인터페이스 추가는 자유롭지만 기존 인터페이스 시그니처 변경은 큰 PR 이 됩니다. diff --git a/docs/architecture/internal/processor/fetcher/domain.md b/docs/architecture/internal/processor/fetcher/domain.md index a4d5f1d1..2a982da6 100644 --- a/docs/architecture/internal/processor/fetcher/domain.md +++ b/docs/architecture/internal/processor/fetcher/domain.md @@ -100,5 +100,5 @@ us.Register(registry, core.DefaultConfig(), rawSvc, crawlerProducer, log) 2. `sources//registry.go` 의 `Register` 에 site 추가 3. [`internal/scheduler.DefaultEntries`](../../scheduler.md) 에 카테고리 seed entry 추가 4. `parsing_rules` 에 host_pattern + path_pattern + selectors 시드 (마이그레이션 또는 운영자 INSERT) - — `path_pattern` 은 빈 문자열이면 catch-all 로 동작하며, [refiner](../../parser/rule.md) 가 누적 sample + — `path_pattern` 은 빈 문자열이면 catch-all 로 동작하며, [refiner](../parser/rule.md) 가 누적 sample 기반으로 정밀화합니다 (이슈 #173). diff --git a/docs/architecture/internal/processor/parser/README.md b/docs/architecture/internal/processor/parser/README.md index fccf052b..38a4f787 100644 --- a/docs/architecture/internal/processor/parser/README.md +++ b/docs/architecture/internal/processor/parser/README.md @@ -1,11 +1,11 @@ -# internal/parser — Domain-Agnostic Parser + Claim Check Worker +# internal/processor/parser — Domain-Agnostic Parser + Claim Check Worker -소스: [`internal/parser/`](../../../../internal/parser/) +소스: [`internal/processor/parser/`](../../../../internal/processor/parser/) 본 패키지는 두 layer 로 구성됩니다 (이슈 #196 통합): 1. **Rule engine** ([rule.md](rule.md)) — `parser.go` (도메인 중립 인터페이스) + `rule/` (DB-driven engine, llmgen, pathinfer, refiner) -2. **Kafka worker** ([worker/](../../../../internal/parser/worker/)) — Claim Check 패턴으로 raw 로드 + rule engine 호출 + content 저장 +2. **Kafka worker** ([worker/](../../../../internal/processor/parser/worker/)) — Claim Check 패턴으로 raw 로드 + rule engine 호출 + content 저장 이슈 #134 에서 fetcher 와 parser 를 분리하면서 도입된 별도 consumer group `issuetracker-parsers`. [Claim Check 패턴](https://learn.microsoft.com/en-us/azure/architecture/patterns/claim-check) 으로 @@ -17,8 +17,8 @@ Kafka 메시지에 raw HTML 을 싣지 않고 DB 에서 로드합니다. | 파일 | 역할 | |------------------------------------------------------------------------|-----------------------------------------------------| -| [parser_worker.go](../../../../internal/parser/worker/parser_worker.go) | `ParserWorker` — TopicFetched consume + parse + content store | -| [cleanup.go](../../../../internal/parser/worker/cleanup.go) | `RawContentCleaner` — cron 으로 오래된 raw_contents 정리 | +| [parser_worker.go](../../../../internal/processor/parser/worker/parser_worker.go) | `ParserWorker` — TopicFetched consume + parse + content store | +| [cleanup.go](../../../../internal/processor/parser/worker/cleanup.go) | `RawContentCleaner` — cron 으로 오래된 raw_contents 정리 |
@@ -54,20 +54,20 @@ Kafka 메시지에 raw HTML 을 싣지 않고 DB 에서 로드합니다. ## RawContentCleaner ParserWorker 가 이상 종료 / rule.Error 잔존 / LLM 재처리 윈도우 만료된 row 가 누적되는 것을 방지하는 -간단한 cron. [`cleanup.go`](../../../../internal/parser/worker/cleanup.go) 가 일정 주기마다 +간단한 cron. [`cleanup.go`](../../../../internal/processor/parser/worker/cleanup.go) 가 일정 주기마다 `RawContentService.PurgeOlderThan(...)` 호출.
## 의존 -- [`internal/parser/rule`](rule.md) — `Parser`, `Resolver` -- [`internal/parser/rule/llmgen`](rule.md) — `Generator.Enqueue` (선택) +- [`internal/processor/parser/rule`](rule.md) — `Parser`, `Resolver` +- [`internal/processor/parser/rule/llmgen`](rule.md) — `Generator.Enqueue` (선택) - [`internal/processor/fetcher/domain/general`](../processor/fetcher/domain.md) — `ConvertPageToContent` -- [`internal/locks`](../locks/README.md) — `ProcessingLock` -- [`internal/storage/service`](../storage/service.md) — `RawContentService`, `ContentService` -- [`internal/storage`](../storage/README.md) — `SampleURLRepository` -- [`internal/publisher`](../publisher.md) — chained job 발행 +- [`internal/locks`](../../locks/README.md) — `ProcessingLock` +- [`internal/storage/service`](../../storage/service.md) — `RawContentService`, `ContentService` +- [`internal/storage`](../../storage/README.md) — `SampleURLRepository` +- [`internal/publisher`](../../publisher.md) — chained job 발행 - [`pkg/queue`](../../pkg/queue.md), [`pkg/logger`](../../pkg/logger.md)
diff --git a/docs/architecture/internal/processor/parser/rule.md b/docs/architecture/internal/processor/parser/rule.md index f4413fe7..415245b8 100644 --- a/docs/architecture/internal/processor/parser/rule.md +++ b/docs/architecture/internal/processor/parser/rule.md @@ -1,14 +1,14 @@ -# internal/parser — Rule Engine 상세 +# internal/processor/parser — Rule Engine 상세 상위 [README.md](README.md) 의 layer 1 (rule engine) 상세 문서. Kafka worker 부분은 README 참조. -소스: [`internal/parser/parser.go`](../../../../internal/parser/parser.go) (도메인 중립 인터페이스) + [`internal/parser/rule/`](../../../../internal/parser/rule/) (`parsing_rules` 테이블 기반 단일 engine, 이슈 #100) +소스: [`internal/processor/parser/parser.go`](../../../../internal/processor/parser/parser.go) (도메인 중립 인터페이스) + [`internal/processor/parser/rule/`](../../../../internal/processor/parser/rule/) (`parsing_rules` 테이블 기반 단일 engine, 이슈 #100) 사이트별 hardcode 파서 (NaverParser/CNNParser/…) 를 폐기하고 DB 기반 rule 한 개의 parser engine 으로 모든 사이트를 처리.
-## 1. 도메인 중립 인터페이스 ([parser.go](../../../../internal/parser/parser.go)) +## 1. 도메인 중립 인터페이스 ([parser.go](../../../../internal/processor/parser/parser.go)) ```go type ContentParser interface { @@ -33,14 +33,14 @@ type Page struct {
-## 2. Rule Engine ([rule/](../../../../internal/parser/rule/)) +## 2. Rule Engine ([rule/](../../../../internal/processor/parser/rule/)) | 파일 | 역할 | |----------------------------------------------------------------------------------------|-----------------------------------------------------------------| -| [parser.go](../../../../internal/parser/rule/parser.go) | `Parser` 구조체 — `ParsePage` / `ParseLinks` 를 SelectorMap 으로 실행 | -| [resolver.go](../../../../internal/parser/rule/resolver.go) | `Resolver` — host → `[]*ParsingRuleRecord` 캐시 (TTL 5min/30s, regex compile cache) | -| [discovery.go](../../../../internal/parser/rule/discovery.go) | `PageLinkDiscovery` — 전체 페이지 `` 스캔 모드 (이슈 #139) | -| [errors.go](../../../../internal/parser/rule/errors.go) | `Error` + `ErrCode` (`ErrNoRule`, `ErrEmptySelector`, `ErrParseFailure`) | +| [parser.go](../../../../internal/processor/parser/rule/parser.go) | `Parser` 구조체 — `ParsePage` / `ParseLinks` 를 SelectorMap 으로 실행 | +| [resolver.go](../../../../internal/processor/parser/rule/resolver.go) | `Resolver` — host → `[]*ParsingRuleRecord` 캐시 (TTL 5min/30s, regex compile cache) | +| [discovery.go](../../../../internal/processor/parser/rule/discovery.go) | `PageLinkDiscovery` — 전체 페이지 `` 스캔 모드 (이슈 #139) | +| [errors.go](../../../../internal/processor/parser/rule/errors.go) | `Error` + `ErrCode` (`ErrNoRule`, `ErrEmptySelector`, `ErrParseFailure`) | **처리 흐름**: ``` @@ -54,46 +54,46 @@ ParsePage(ctx, raw)
-## 3. LLM Selector Generator ([rule/llmgen/](../../../../internal/parser/rule/llmgen/)) +## 3. LLM Selector Generator ([rule/llmgen/](../../../../internal/processor/parser/rule/llmgen/)) 이슈 #149. `ErrNoRule` 발생 시 **비동기**로 LLM 에게 selector 를 생성시키고 `enabled=false` 로 DB 에 INSERT (운영자 검토 후 활성). | 파일 | 역할 | |--------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| [generator.go](../../../../internal/parser/rule/llmgen/generator.go) | `Generator` — Enqueue / 처리 goroutine / Stop | -| [prompt.go](../../../../internal/parser/rule/llmgen/prompt.go) | LLM 프롬프트 템플릿 (HTML 샘플 + 출력 스키마) | -| [dedup.go](../../../../internal/parser/rule/llmgen/dedup.go) | in-flight set — 동일 host 중복 enqueue 방지 | +| [generator.go](../../../../internal/processor/parser/rule/llmgen/generator.go) | `Generator` — Enqueue / 처리 goroutine / Stop | +| [prompt.go](../../../../internal/processor/parser/rule/llmgen/prompt.go) | LLM 프롬프트 템플릿 (HTML 샘플 + 출력 스키마) | +| [dedup.go](../../../../internal/processor/parser/rule/llmgen/dedup.go) | in-flight set — 동일 host 중복 enqueue 방지 | **동작**: ParserWorker 가 `ErrNoRule` 받으면 `llmGen.Enqueue(ctx, host, targetType, raw)` 호출 — `targetType` (`page`/`list`) 별로 별도 in-flight set 으로 dedup 됩니다. Generator 는 별도 goroutine 에서 LLM 호출 → 결과 검증 (selector 가 실제 매칭되는지 확인) → INSERT (enabled=false) → -[`Resolver.Invalidate(host, targetType)`](../../../../internal/parser/rule/resolver.go) 로 +[`Resolver.Invalidate(host, targetType)`](../../../../internal/processor/parser/rule/resolver.go) 로 해당 (host, targetType) 캐시 항목 무효화 (전체 invalidation 은 `Resolver.InvalidateAll()`).
-## 4. Path Pattern Inference ([rule/pathinfer/](../../../../internal/parser/rule/pathinfer/)) +## 4. Path Pattern Inference ([rule/pathinfer/](../../../../internal/processor/parser/rule/pathinfer/)) | 파일 | 역할 | |--------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| [pathinfer.go](../../../../internal/parser/rule/pathinfer/pathinfer.go) | `InferHeuristic` — 알고리즘 only (공통 prefix / 숫자 ID 패턴) | -| [llm.go](../../../../internal/parser/rule/pathinfer/llm.go) | `InferLLM` — heuristic 실패 시 LLM 에 위임 | +| [pathinfer.go](../../../../internal/processor/parser/rule/pathinfer/pathinfer.go) | `InferHeuristic` — 알고리즘 only (공통 prefix / 숫자 ID 패턴) | +| [llm.go](../../../../internal/processor/parser/rule/pathinfer/llm.go) | `InferLLM` — heuristic 실패 시 LLM 에 위임 | -[refiner](../../../../internal/parser/rule/refiner/) 가 본 함수들을 호출. +[refiner](../../../../internal/processor/parser/rule/refiner/) 가 본 함수들을 호출.
-## 5. Refiner ([rule/refiner/](../../../../internal/parser/rule/refiner/)) +## 5. Refiner ([rule/refiner/](../../../../internal/processor/parser/rule/refiner/)) 이슈 #173 단계 4-2. `llm-auto` source 의 **catch-all** rule 의 `path_pattern` 을 `sample_urls` 누적 데이터로 정밀화. | 파일 | 역할 | |-----------------------------------------------------------------------------------------------------|------------------------------------------------------| -| [refiner.go](../../../../internal/parser/rule/refiner/refiner.go) | `Refiner` — Start / Stop / RunOnce — interval polling goroutine | -| [llm_adapter.go](../../../../internal/parser/rule/refiner/llm_adapter.go) | `pkg/llm.Provider` → `LLMClient` 인터페이스 어댑터 | -| [metrics.go](../../../../internal/parser/rule/refiner/metrics.go) | `refiner_attempts` Prometheus counter (PR #191) | +| [refiner.go](../../../../internal/processor/parser/rule/refiner/refiner.go) | `Refiner` — Start / Stop / RunOnce — interval polling goroutine | +| [llm_adapter.go](../../../../internal/processor/parser/rule/refiner/llm_adapter.go) | `pkg/llm.Provider` → `LLMClient` 인터페이스 어댑터 | +| [metrics.go](../../../../internal/processor/parser/rule/refiner/metrics.go) | `refiner_attempts` Prometheus counter (PR #191) | **동작**: ``` @@ -126,8 +126,8 @@ rule/discovery/ ──→ pkg/links (link extract) ## 호출 측 -- [`internal/parser/worker.ParserWorker`](README.md) — `rule.Parser` 와 `Resolver` 를 인스턴스로 보유, 각 메시지마다 `ParsePage` / `ParseLinks` 호출 -- [`internal/parser/worker.ParserWorker`](README.md) — `ErrNoRule` 발생 시 `llmGen.Enqueue` 호출 +- [`internal/processor/parser/worker.ParserWorker`](README.md) — `rule.Parser` 와 `Resolver` 를 인스턴스로 보유, 각 메시지마다 `ParsePage` / `ParseLinks` 호출 +- [`internal/processor/parser/worker.ParserWorker`](README.md) — `ErrNoRule` 발생 시 `llmGen.Enqueue` 호출 - [`cmd/issuetracker.buildRefiner`](../../cmd/issuetracker.md) — Refiner 인스턴스 wire
diff --git a/docs/architecture/internal/publisher.md b/docs/architecture/internal/publisher.md index ab6822c8..35df40f7 100644 --- a/docs/architecture/internal/publisher.md +++ b/docs/architecture/internal/publisher.md @@ -53,7 +53,7 @@ for each job in batch: ## 호출 측 -- [`internal/parser/worker.ParserWorker`](parser/README.md) — TargetTypeList 처리 시 발견 링크 발행 +- [`internal/processor/parser/worker.ParserWorker`](processor/parser/README.md) — TargetTypeList 처리 시 발견 링크 발행 - [`internal/scheduler.Scheduler`](scheduler.md) 는 **미사용** — Scheduler 는 직접 `Emitter.Emit` 호출
diff --git a/docs/architecture/internal/storage/service.md b/docs/architecture/internal/storage/service.md index b4bb1078..e41fdcb5 100644 --- a/docs/architecture/internal/storage/service.md +++ b/docs/architecture/internal/storage/service.md @@ -69,8 +69,8 @@ ParserWorker (이슈 #134) 가 파싱 완료된 raw row 를 즉시 `Delete` — | 호출자 | 사용 메소드 | |----------------------------------------------------|--------------------------------------------------| | [processor/fetcher/domain/general.ChainHandler](../processor/fetcher/domain.md) | `RawContentService.Store` (Claim Check) | -| [parser/worker.ParserWorker](../parser/README.md) | `RawContentService.GetByID` / `Delete`
`ContentService.Store` | -| [parser/worker.RawContentCleaner](../parser/README.md) | `RawContentService.PurgeOlderThan` | +| [parser/worker.ParserWorker](../processor/parser/README.md) | `RawContentService.GetByID` / `Delete`
`ContentService.Store` | +| [parser/worker.RawContentCleaner](../processor/parser/README.md) | `RawContentService.PurgeOlderThan` | | [processor/validate.Worker](../processor/validate.md) | `ContentService.GetByID` / `Delete` / `UpdateValidationStatus` |
diff --git a/docs/architecture/pkg/config.md b/docs/architecture/pkg/config.md index e227f009..86afcc21 100644 --- a/docs/architecture/pkg/config.md +++ b/docs/architecture/pkg/config.md @@ -59,7 +59,7 @@ | `LLM_*`, `GEMINI_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY` | [pkg/llm](llm.md) | | `VALIDATE_*` | [validate.md](../internal/processor/validate.md) | | `SCHEDULER_*` | [scheduler.md](../internal/scheduler.md) | -| `REFINEMENT_*` | [refiner](../internal/parser/rule.md) | +| `REFINEMENT_*` | [refiner](../internal/processor/parser/rule.md) | | `METRICS_ADDR` | [pkg/metrics](metrics.md) | | `LOG_*` | [pkg/logger](logger.md) | | `CLASSIFIER_*` | [classifier](../internal/classifier/README.md) | diff --git a/docs/architecture/pkg/links.md b/docs/architecture/pkg/links.md index 585ab587..f00789ed 100644 --- a/docs/architecture/pkg/links.md +++ b/docs/architecture/pkg/links.md @@ -65,5 +65,5 @@ func (e *HTMLLinkExtractor) Extract(raw *RawContent) ([]Link, error) { ## 호출 측 - [`internal/publisher`](../internal/publisher.md) — Publisher.SetNormalizer -- [`internal/parser/rule/discovery`](../internal/parser/rule.md) — full-page link discovery +- [`internal/processor/parser/rule/discovery`](../internal/processor/parser/rule.md) — full-page link discovery - [`internal/processor/fetcher/core/extractor`](../internal/processor/fetcher/core.md) — boundary wrapper diff --git a/docs/architecture/pkg/llm.md b/docs/architecture/pkg/llm.md index 43cb3f60..a21f6eba 100644 --- a/docs/architecture/pkg/llm.md +++ b/docs/architecture/pkg/llm.md @@ -115,8 +115,8 @@ p := chain.NewWithPolicy(pol, []llm.Provider{...}, chain.WithPolicyLogger(log)) ## 호출 측 - [`cmd/issuetracker.buildLLMProvider`](../cmd/issuetracker.md) — chain 구성 -- [`internal/parser/rule/llmgen`](../internal/parser/rule.md) — selector 자동 생성 -- [`internal/parser/rule/refiner`](../internal/parser/rule.md) — path_pattern 정밀화 +- [`internal/processor/parser/rule/llmgen`](../internal/processor/parser/rule.md) — selector 자동 생성 +- [`internal/processor/parser/rule/refiner`](../internal/processor/parser/rule.md) — path_pattern 정밀화
diff --git a/docs/architecture/pkg/metrics.md b/docs/architecture/pkg/metrics.md index 79ed5b2d..0fd80824 100644 --- a/docs/architecture/pkg/metrics.md +++ b/docs/architecture/pkg/metrics.md @@ -27,10 +27,10 @@ defer stop() | 메트릭 | 종류 | 위치 | 의미 | |------------------------------------|-----------|---------------------------------------------------------------------|-------------------------------------| | Go runtime / process | Collector | NewRegistry 자동 | GC, goroutine, mem, fd 등 | -| `refiner_attempts` | Counter | [refiner/metrics.go](../../../internal/parser/rule/refiner/metrics.go) | path_pattern 정밀화 시도 횟수 (PR #191) | +| `refiner_attempts` | Counter | [refiner/metrics.go](../../../internal/processor/parser/rule/refiner/metrics.go) | path_pattern 정밀화 시도 횟수 (PR #191) | | (그 외) Circuit Breaker 상태 등 | (개별 패키지가 자체 등록) | - | | -신규 메트릭 추가 시 해당 패키지 (예: `internal/parser/rule/refiner`) 가 +신규 메트릭 추가 시 해당 패키지 (예: `internal/processor/parser/rule/refiner`) 가 `prometheus.NewCounterVec` 등으로 만들고 `metricsRegistry.MustRegister` 로 등록 — registry 인스턴스를 [`cmd/issuetracker`](../cmd/issuetracker.md) 가 wiring 시점에 주입. diff --git a/docs/architecture/pkg/queue.md b/docs/architecture/pkg/queue.md index 779d2d9f..930e3da3 100644 --- a/docs/architecture/pkg/queue.md +++ b/docs/architecture/pkg/queue.md @@ -118,7 +118,7 @@ seed publish 차단. ## 호출 측 -거의 모든 단계 worker — [`processor/fetcher/worker`](../internal/processor/fetcher/worker.md), [`parser/worker`](../internal/parser/README.md), +거의 모든 단계 worker — [`processor/fetcher/worker`](../internal/processor/fetcher/worker.md), [`parser/worker`](../internal/processor/parser/README.md), [`processor/validate`](../internal/processor/validate.md), [`scheduler`](../internal/scheduler.md), [`publisher`](../internal/publisher.md).
From 95abf684dde0042fe97274f9ab5f42b16cb1d810 Mon Sep 17 00:00:00 2001 From: juhy0987 Date: Fri, 1 May 2026 21:48:51 +0900 Subject: [PATCH 3/3] =?UTF-8?q?[FIX]:=20=ED=94=BC=EB=93=9C=EB=B0=B1=20?= =?UTF-8?q?=EB=B0=98=EC=98=81,=20=EC=9D=B4=EB=8F=99=EB=90=9C=20docs=20?= =?UTF-8?q?=EC=9D=98=20=EA=B9=A8=EC=A7=84=20=EC=83=81=EB=8C=80=20path=20+?= =?UTF-8?q?=20=EC=9D=BC=EA=B4=80=EC=84=B1=20=ED=91=9C=EA=B8=B0=20=EC=A0=95?= =?UTF-8?q?=EC=A0=95=20(PR=20#205=20gemini=20+=20CodeRabbit)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit (broken paths — Major): - docs/architecture/internal/processor/parser/{README.md,rule.md} 의 ../../../../internal/X / ../../../../cmd/X 는 깊이 4 (이전 위치 기준) — 이동 후 깊이 5 필요 - 모든 ../../../../internal/ → ../../../../../internal/ 일괄 치환 - 모든 ../../../../cmd/ → ../../../../../cmd/ 일괄 치환 - README.md:78 / rule.md:131 의 ../../cmd/ → ../../../cmd/ 추가 보정 CodeRabbit (Minor): - README.md:66 의 ../processor/fetcher/domain.md → ../fetcher/domain.md (sibling 단순 참조) gemini (Medium — link text 일관성): - docs/architecture/README.md:152 [parser/worker] → [processor/parser/worker] - docs/architecture/internal/README.md:17 [parser/README.md] → [processor/parser/README.md] - docs/architecture/pkg/queue.md:121 [`parser/worker`] → [`processor/parser/worker`] Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/architecture/README.md | 2 +- .../internal/processor/parser/README.md | 16 +++---- .../internal/processor/parser/rule.md | 44 +++++++++---------- docs/architecture/pkg/queue.md | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 6d3737f9..0518f2a9 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -149,7 +149,7 @@ docs/architecture/ | `issuetracker.crawl.normal` | [scheduler](../../internal/scheduler/) / publisher | `issuetracker-crawler-workers` (동일) | | `issuetracker.crawl.low` | [scheduler](../../internal/scheduler/) / publisher | `issuetracker-crawler-workers` (동일) | | `issuetracker.fetched` | [processor/fetcher/worker](../../internal/processor/fetcher/worker/) | `issuetracker-parsers` (`GroupParsers`) | -| `issuetracker.normalized` | [parser/worker](../../internal/processor/parser/worker/) | `issuetracker-validators` (`GroupValidators`) | +| `issuetracker.normalized` | [processor/parser/worker](../../internal/processor/parser/worker/) | `issuetracker-validators` (`GroupValidators`) | | `issuetracker.validated` | [processor/validate](../../internal/processor/validate/) | (downstream — TBD) | | `issuetracker.dlq` | 모든 stage 실패 분기 | (운영 모니터링) | diff --git a/docs/architecture/internal/processor/parser/README.md b/docs/architecture/internal/processor/parser/README.md index 38a4f787..fe833992 100644 --- a/docs/architecture/internal/processor/parser/README.md +++ b/docs/architecture/internal/processor/parser/README.md @@ -1,11 +1,11 @@ # internal/processor/parser — Domain-Agnostic Parser + Claim Check Worker -소스: [`internal/processor/parser/`](../../../../internal/processor/parser/) +소스: [`internal/processor/parser/`](../../../../../internal/processor/parser/) 본 패키지는 두 layer 로 구성됩니다 (이슈 #196 통합): 1. **Rule engine** ([rule.md](rule.md)) — `parser.go` (도메인 중립 인터페이스) + `rule/` (DB-driven engine, llmgen, pathinfer, refiner) -2. **Kafka worker** ([worker/](../../../../internal/processor/parser/worker/)) — Claim Check 패턴으로 raw 로드 + rule engine 호출 + content 저장 +2. **Kafka worker** ([worker/](../../../../../internal/processor/parser/worker/)) — Claim Check 패턴으로 raw 로드 + rule engine 호출 + content 저장 이슈 #134 에서 fetcher 와 parser 를 분리하면서 도입된 별도 consumer group `issuetracker-parsers`. [Claim Check 패턴](https://learn.microsoft.com/en-us/azure/architecture/patterns/claim-check) 으로 @@ -17,8 +17,8 @@ Kafka 메시지에 raw HTML 을 싣지 않고 DB 에서 로드합니다. | 파일 | 역할 | |------------------------------------------------------------------------|-----------------------------------------------------| -| [parser_worker.go](../../../../internal/processor/parser/worker/parser_worker.go) | `ParserWorker` — TopicFetched consume + parse + content store | -| [cleanup.go](../../../../internal/processor/parser/worker/cleanup.go) | `RawContentCleaner` — cron 으로 오래된 raw_contents 정리 | +| [parser_worker.go](../../../../../internal/processor/parser/worker/parser_worker.go) | `ParserWorker` — TopicFetched consume + parse + content store | +| [cleanup.go](../../../../../internal/processor/parser/worker/cleanup.go) | `RawContentCleaner` — cron 으로 오래된 raw_contents 정리 |
@@ -54,7 +54,7 @@ Kafka 메시지에 raw HTML 을 싣지 않고 DB 에서 로드합니다. ## RawContentCleaner ParserWorker 가 이상 종료 / rule.Error 잔존 / LLM 재처리 윈도우 만료된 row 가 누적되는 것을 방지하는 -간단한 cron. [`cleanup.go`](../../../../internal/processor/parser/worker/cleanup.go) 가 일정 주기마다 +간단한 cron. [`cleanup.go`](../../../../../internal/processor/parser/worker/cleanup.go) 가 일정 주기마다 `RawContentService.PurgeOlderThan(...)` 호출.
@@ -63,7 +63,7 @@ ParserWorker 가 이상 종료 / rule.Error 잔존 / LLM 재처리 윈도우 만 - [`internal/processor/parser/rule`](rule.md) — `Parser`, `Resolver` - [`internal/processor/parser/rule/llmgen`](rule.md) — `Generator.Enqueue` (선택) -- [`internal/processor/fetcher/domain/general`](../processor/fetcher/domain.md) — `ConvertPageToContent` +- [`internal/processor/fetcher/domain/general`](../fetcher/domain.md) — `ConvertPageToContent` - [`internal/locks`](../../locks/README.md) — `ProcessingLock` - [`internal/storage/service`](../../storage/service.md) — `RawContentService`, `ContentService` - [`internal/storage`](../../storage/README.md) — `SampleURLRepository` @@ -74,8 +74,8 @@ ParserWorker 가 이상 종료 / rule.Error 잔존 / LLM 재처리 윈도우 만 ## Wiring 위치 -[`cmd/issuetracker/main.go`](../../../../cmd/issuetracker/main.go) — 단계 10 (parser worker), 단계 12 -(cleanup cron). 자세한 wiring 은 [cmd/issuetracker.md](../../cmd/issuetracker.md) 참조. +[`cmd/issuetracker/main.go`](../../../../../cmd/issuetracker/main.go) — 단계 10 (parser worker), 단계 12 +(cleanup cron). 자세한 wiring 은 [cmd/issuetracker.md](../../../cmd/issuetracker.md) 참조.
diff --git a/docs/architecture/internal/processor/parser/rule.md b/docs/architecture/internal/processor/parser/rule.md index 415245b8..81ef3762 100644 --- a/docs/architecture/internal/processor/parser/rule.md +++ b/docs/architecture/internal/processor/parser/rule.md @@ -2,13 +2,13 @@ 상위 [README.md](README.md) 의 layer 1 (rule engine) 상세 문서. Kafka worker 부분은 README 참조. -소스: [`internal/processor/parser/parser.go`](../../../../internal/processor/parser/parser.go) (도메인 중립 인터페이스) + [`internal/processor/parser/rule/`](../../../../internal/processor/parser/rule/) (`parsing_rules` 테이블 기반 단일 engine, 이슈 #100) +소스: [`internal/processor/parser/parser.go`](../../../../../internal/processor/parser/parser.go) (도메인 중립 인터페이스) + [`internal/processor/parser/rule/`](../../../../../internal/processor/parser/rule/) (`parsing_rules` 테이블 기반 단일 engine, 이슈 #100) 사이트별 hardcode 파서 (NaverParser/CNNParser/…) 를 폐기하고 DB 기반 rule 한 개의 parser engine 으로 모든 사이트를 처리.
-## 1. 도메인 중립 인터페이스 ([parser.go](../../../../internal/processor/parser/parser.go)) +## 1. 도메인 중립 인터페이스 ([parser.go](../../../../../internal/processor/parser/parser.go)) ```go type ContentParser interface { @@ -29,18 +29,18 @@ type Page struct { ``` 호출자는 도메인 (news / blog / community) 별로 `Page → 자기 모델` 변환 책임을 가집니다 (예: -[`domain/general/convert.go`](../../../../internal/processor/fetcher/domain/general/convert.go) 가 `Page → Content`). +[`domain/general/convert.go`](../../../../../internal/processor/fetcher/domain/general/convert.go) 가 `Page → Content`).
-## 2. Rule Engine ([rule/](../../../../internal/processor/parser/rule/)) +## 2. Rule Engine ([rule/](../../../../../internal/processor/parser/rule/)) | 파일 | 역할 | |----------------------------------------------------------------------------------------|-----------------------------------------------------------------| -| [parser.go](../../../../internal/processor/parser/rule/parser.go) | `Parser` 구조체 — `ParsePage` / `ParseLinks` 를 SelectorMap 으로 실행 | -| [resolver.go](../../../../internal/processor/parser/rule/resolver.go) | `Resolver` — host → `[]*ParsingRuleRecord` 캐시 (TTL 5min/30s, regex compile cache) | -| [discovery.go](../../../../internal/processor/parser/rule/discovery.go) | `PageLinkDiscovery` — 전체 페이지 `
` 스캔 모드 (이슈 #139) | -| [errors.go](../../../../internal/processor/parser/rule/errors.go) | `Error` + `ErrCode` (`ErrNoRule`, `ErrEmptySelector`, `ErrParseFailure`) | +| [parser.go](../../../../../internal/processor/parser/rule/parser.go) | `Parser` 구조체 — `ParsePage` / `ParseLinks` 를 SelectorMap 으로 실행 | +| [resolver.go](../../../../../internal/processor/parser/rule/resolver.go) | `Resolver` — host → `[]*ParsingRuleRecord` 캐시 (TTL 5min/30s, regex compile cache) | +| [discovery.go](../../../../../internal/processor/parser/rule/discovery.go) | `PageLinkDiscovery` — 전체 페이지 `` 스캔 모드 (이슈 #139) | +| [errors.go](../../../../../internal/processor/parser/rule/errors.go) | `Error` + `ErrCode` (`ErrNoRule`, `ErrEmptySelector`, `ErrParseFailure`) | **처리 흐름**: ``` @@ -54,46 +54,46 @@ ParsePage(ctx, raw)
-## 3. LLM Selector Generator ([rule/llmgen/](../../../../internal/processor/parser/rule/llmgen/)) +## 3. LLM Selector Generator ([rule/llmgen/](../../../../../internal/processor/parser/rule/llmgen/)) 이슈 #149. `ErrNoRule` 발생 시 **비동기**로 LLM 에게 selector 를 생성시키고 `enabled=false` 로 DB 에 INSERT (운영자 검토 후 활성). | 파일 | 역할 | |--------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| [generator.go](../../../../internal/processor/parser/rule/llmgen/generator.go) | `Generator` — Enqueue / 처리 goroutine / Stop | -| [prompt.go](../../../../internal/processor/parser/rule/llmgen/prompt.go) | LLM 프롬프트 템플릿 (HTML 샘플 + 출력 스키마) | -| [dedup.go](../../../../internal/processor/parser/rule/llmgen/dedup.go) | in-flight set — 동일 host 중복 enqueue 방지 | +| [generator.go](../../../../../internal/processor/parser/rule/llmgen/generator.go) | `Generator` — Enqueue / 처리 goroutine / Stop | +| [prompt.go](../../../../../internal/processor/parser/rule/llmgen/prompt.go) | LLM 프롬프트 템플릿 (HTML 샘플 + 출력 스키마) | +| [dedup.go](../../../../../internal/processor/parser/rule/llmgen/dedup.go) | in-flight set — 동일 host 중복 enqueue 방지 | **동작**: ParserWorker 가 `ErrNoRule` 받으면 `llmGen.Enqueue(ctx, host, targetType, raw)` 호출 — `targetType` (`page`/`list`) 별로 별도 in-flight set 으로 dedup 됩니다. Generator 는 별도 goroutine 에서 LLM 호출 → 결과 검증 (selector 가 실제 매칭되는지 확인) → INSERT (enabled=false) → -[`Resolver.Invalidate(host, targetType)`](../../../../internal/processor/parser/rule/resolver.go) 로 +[`Resolver.Invalidate(host, targetType)`](../../../../../internal/processor/parser/rule/resolver.go) 로 해당 (host, targetType) 캐시 항목 무효화 (전체 invalidation 은 `Resolver.InvalidateAll()`).
-## 4. Path Pattern Inference ([rule/pathinfer/](../../../../internal/processor/parser/rule/pathinfer/)) +## 4. Path Pattern Inference ([rule/pathinfer/](../../../../../internal/processor/parser/rule/pathinfer/)) | 파일 | 역할 | |--------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| [pathinfer.go](../../../../internal/processor/parser/rule/pathinfer/pathinfer.go) | `InferHeuristic` — 알고리즘 only (공통 prefix / 숫자 ID 패턴) | -| [llm.go](../../../../internal/processor/parser/rule/pathinfer/llm.go) | `InferLLM` — heuristic 실패 시 LLM 에 위임 | +| [pathinfer.go](../../../../../internal/processor/parser/rule/pathinfer/pathinfer.go) | `InferHeuristic` — 알고리즘 only (공통 prefix / 숫자 ID 패턴) | +| [llm.go](../../../../../internal/processor/parser/rule/pathinfer/llm.go) | `InferLLM` — heuristic 실패 시 LLM 에 위임 | -[refiner](../../../../internal/processor/parser/rule/refiner/) 가 본 함수들을 호출. +[refiner](../../../../../internal/processor/parser/rule/refiner/) 가 본 함수들을 호출.
-## 5. Refiner ([rule/refiner/](../../../../internal/processor/parser/rule/refiner/)) +## 5. Refiner ([rule/refiner/](../../../../../internal/processor/parser/rule/refiner/)) 이슈 #173 단계 4-2. `llm-auto` source 의 **catch-all** rule 의 `path_pattern` 을 `sample_urls` 누적 데이터로 정밀화. | 파일 | 역할 | |-----------------------------------------------------------------------------------------------------|------------------------------------------------------| -| [refiner.go](../../../../internal/processor/parser/rule/refiner/refiner.go) | `Refiner` — Start / Stop / RunOnce — interval polling goroutine | -| [llm_adapter.go](../../../../internal/processor/parser/rule/refiner/llm_adapter.go) | `pkg/llm.Provider` → `LLMClient` 인터페이스 어댑터 | -| [metrics.go](../../../../internal/processor/parser/rule/refiner/metrics.go) | `refiner_attempts` Prometheus counter (PR #191) | +| [refiner.go](../../../../../internal/processor/parser/rule/refiner/refiner.go) | `Refiner` — Start / Stop / RunOnce — interval polling goroutine | +| [llm_adapter.go](../../../../../internal/processor/parser/rule/refiner/llm_adapter.go) | `pkg/llm.Provider` → `LLMClient` 인터페이스 어댑터 | +| [metrics.go](../../../../../internal/processor/parser/rule/refiner/metrics.go) | `refiner_attempts` Prometheus counter (PR #191) | **동작**: ``` @@ -128,7 +128,7 @@ rule/discovery/ ──→ pkg/links (link extract) - [`internal/processor/parser/worker.ParserWorker`](README.md) — `rule.Parser` 와 `Resolver` 를 인스턴스로 보유, 각 메시지마다 `ParsePage` / `ParseLinks` 호출 - [`internal/processor/parser/worker.ParserWorker`](README.md) — `ErrNoRule` 발생 시 `llmGen.Enqueue` 호출 -- [`cmd/issuetracker.buildRefiner`](../../cmd/issuetracker.md) — Refiner 인스턴스 wire +- [`cmd/issuetracker.buildRefiner`](../../../cmd/issuetracker.md) — Refiner 인스턴스 wire
diff --git a/docs/architecture/pkg/queue.md b/docs/architecture/pkg/queue.md index 930e3da3..66a16dd6 100644 --- a/docs/architecture/pkg/queue.md +++ b/docs/architecture/pkg/queue.md @@ -118,7 +118,7 @@ seed publish 차단. ## 호출 측 -거의 모든 단계 worker — [`processor/fetcher/worker`](../internal/processor/fetcher/worker.md), [`parser/worker`](../internal/processor/parser/README.md), +거의 모든 단계 worker — [`processor/fetcher/worker`](../internal/processor/fetcher/worker.md), [`processor/parser/worker`](../internal/processor/parser/README.md), [`processor/validate`](../internal/processor/validate.md), [`scheduler`](../internal/scheduler.md), [`publisher`](../internal/publisher.md).