[REFAC#197] crawler/worker 의 lock 인프라 → internal/locks 분리 (메타 이슈 #195 단계 2)

.claude/rules/01-architecture.md

@@ -125,6 +125,9 @@ issuetracker/
 │   │   │   ├── pathinfer/     # path_pattern 추론 알고리즘 (이슈 #173)
 │   │   │   └── refiner/       # path_pattern 정밀화 polling
 │   │   └── worker/            # Claim Check 기반 ParserWorker (Kafka consumer)
+│   ├── locks/                 # ✅ 단계 무관 distributed lock — fetcher/parser/validator 공유 (이슈 #197)
+│   │   ├── ingestion_lock.go  # IngestionLock (Publisher 가 Kafka enqueue 직전 사용)
+│   │   └── processing_lock.go # ProcessingLock + ProcessingKey(stage, url)
 │   ├── processor/             # Processing pipeline (planned)
 │   │   ├── normalize/         # Data normalization
 │   │   ├── enrich/            # Data enrichment

cmd/issuetracker/main.go

@@ -15,6 +15,7 @@ import (
 	"issuetracker/internal/crawler/domain/general/sources/us"
 	"issuetracker/internal/crawler/handler"
 	crawlerWorker "issuetracker/internal/crawler/worker"
+	"issuetracker/internal/locks"
 	"issuetracker/internal/parser/rule"
 	"issuetracker/internal/parser/rule/llmgen"
 	"issuetracker/internal/parser/rule/refiner"
@@ -143,8 +144,8 @@ func main() {
 	// 단계 구분은 ProcessingKey(stage, url) 의 stage prefix 로 처리.
 	// worker/manager 가 nil 을 NoopProcessingLock 로 fallback 처리하는 설계와 일관되게,
 	// Redis 초기화 실패 시에도 크롤링이 중단되지 않도록 graceful degrade 합니다.
-	var procLock crawlerWorker.ProcessingLock
-	var ingestionLock crawlerWorker.IngestionLock
+	var procLock locks.ProcessingLock
+	var ingestionLock locks.IngestionLock
 	var retryScheduler crawlerWorker.RetryScheduler
 	var retrySchedulerStop func()
 	redisCfg, err := config.LoadRedis()
@@ -160,8 +161,8 @@ func main() {
 				"host": redisCfg.Host,
 				"port": redisCfg.Port,
 			}).Info("redis connected for processing lock and ingestion lock")
-			procLock = crawlerWorker.NewRedisProcessingLock(redisClient, crawlerWorker.DefaultProcessingLockTTL)
-			ingestionLock = crawlerWorker.NewRedisIngestionLock(redisClient, redisCfg.IngestionLockTTL)
+			procLock = locks.NewRedisProcessingLock(redisClient, locks.DefaultProcessingLockTTL)
+			ingestionLock = locks.NewRedisIngestionLock(redisClient, redisCfg.IngestionLockTTL)
 
 			// Delayed retry queue (이슈 #82): retry 를 Redis ZSET 에 보관하고 별도
 			// goroutine 이 ScheduledAt 도달 시 Kafka 에 발행 — worker 슬롯 점유 회피.

cmd/processor/main.go

@@ -7,7 +7,7 @@ import (
 	"syscall"
 	"time"
 
-	crawlerWorker "issuetracker/internal/crawler/worker"
+	"issuetracker/internal/locks"
 	"issuetracker/internal/processor/validate"
 	pgstore "issuetracker/internal/storage/postgres"
 	"issuetracker/internal/storage/service"
@@ -93,7 +93,7 @@ func main() {
 	// validator 결과 (passed/rejected) 는 contentSvc.UpdateValidationStatus 로 contents 에 기록 (이슈 #135 / #161).
 	// processor 단독 실행은 dev/test 시나리오 — Redis wiring 없이 NoopProcessingLock 사용.
 	// 다중 인스턴스 운영은 cmd/issuetracker 통합 바이너리에서 Redis 기반 ProcessingLock 공유.
-	worker := validate.NewWorker(consumer, producer, contentSvc, crawlerWorker.NoopProcessingLock{}, validateWorkerCount, validateCfg)
+	worker := validate.NewWorker(consumer, producer, contentSvc, locks.NoopProcessingLock{}, validateWorkerCount, validateCfg)
 	worker.Start(ctx)
 
 	log.WithFields(map[string]interface{}{

docs/architecture/README.md

@@ -42,7 +42,9 @@ docs/architecture/
 │   │   ├── handler.md                     ← Registry (crawler_name → Handler)
 │   │   ├── implementation.md              ← chromedp / goquery
 │   │   ├── rate_limiter.md
-│   │   └── worker.md                      ← PoolManager + ProcessingLock + RetryScheduler
+│   │   └── worker.md                      ← PoolManager + RetryScheduler + CircuitBreaker
+│   ├── locks/                             ← 단계 무관 distributed lock (이슈 #197)
+│   │   └── README.md                      ← ProcessingLock + IngestionLock (Redis SETNX)
 │   ├── parser/
 │   │   ├── README.md                      ← Domain-Agnostic Parser + Claim Check Worker
 │   │   └── rule.md                        ← rule.Parser (DB-driven) + llmgen + pathinfer + refiner
@@ -163,7 +165,7 @@ docs/architecture/
 |----------------------------|------------------------------------------------------------|-----------------------------------------------|
 | Kafka                      | [pkg/queue/](../../pkg/queue/)                              | 모든 stage 간 메시지 버스                      |
 | PostgreSQL                 | [internal/storage/postgres/](../../internal/storage/postgres/) | contents / content_bodies / content_meta / raw_contents / parsing_rules / sample_urls / schema_migrations |
-| Redis                      | [pkg/redis/](../../pkg/redis/), [internal/crawler/worker/](../../internal/crawler/worker/) | ProcessingLock(SETNX) / IngestionLock / RetryQueue(ZSET) |
+| Redis                      | [pkg/redis/](../../pkg/redis/), [internal/locks/](../../internal/locks/), [internal/crawler/worker/](../../internal/crawler/worker/) | ProcessingLock + IngestionLock (locks) / RetryQueue ZSET (crawler/worker) |
 | LLM (Gemini/OpenAI/Claude) | [pkg/llm/](../../pkg/llm/)                                  | parser rule 자동 생성 / path_pattern refinement |
 | Chrome (CDP)               | [internal/crawler/implementation/chromedp/](../../internal/crawler/implementation/chromedp/) | 동적 페이지 헤드리스 렌더                      |
 | ELArchive Classifier       | [internal/classifier/](../../internal/classifier/) + [proto/classifier/](../../proto/classifier/) | 카테고리 분류 (gRPC primary, HTTP fallback)    |

docs/architecture/cmd/README.md

@@ -40,7 +40,7 @@ make pg-migrate-down    # 마이그레이션 롤백 (운영자 전용)
 
 ```
 cmd/issuetracker  ──→ internal/* (전부) + pkg/* (전부)
-cmd/processor     ──→ internal/{processor/validate, storage/*, crawler/worker(NoopProcessingLock)} + pkg/*
+cmd/processor     ──→ internal/{processor/validate, storage/*, locks(NoopProcessingLock)} + pkg/*
 cmd/migrate       ──→ internal/storage/postgres + migrations
 cmd/migrate-down  ──→ internal/storage/postgres + migrations
 ```

docs/architecture/cmd/issuetracker.md

@@ -38,7 +38,8 @@
 
 각 단계의 자세한 책임은 해당 패키지 문서 참조:
 
-- [internal/crawler/worker.md](../internal/crawler/worker.md) — PoolManager, ProcessingLock, RetryScheduler
+- [internal/crawler/worker.md](../internal/crawler/worker.md) — PoolManager, RetryScheduler
+- [internal/locks/README.md](../internal/locks/README.md) — ProcessingLock, IngestionLock
 - [internal/parser/rule.md](../internal/parser/rule.md) — rule.Parser, llmgen, refiner
 - [internal/parser/README.md](../internal/parser/README.md) — ParserWorker (Claim Check)
 - [internal/processor/validate.md](../internal/processor/validate.md) — Validator

docs/architecture/cmd/processor.md

@@ -34,7 +34,7 @@
 ## 의존 패키지
 
 - [`internal/processor/validate`](../../../internal/processor/validate/) — Worker / Validator
-- [`internal/crawler/worker`](../../../internal/crawler/worker/) — `NoopProcessingLock`
+- [`internal/locks`](../../../internal/locks/) — `NoopProcessingLock`
 - [`internal/storage/postgres`](../../../internal/storage/postgres/) + [`internal/storage/service`](../../../internal/storage/service/)
 - [`pkg/config`](../../../pkg/config/), [`pkg/queue`](../../../pkg/queue/), [`pkg/logger`](../../../pkg/logger/), [`pkg/metrics`](../../../pkg/metrics/)
 

docs/architecture/internal/crawler/worker.md

@@ -1,23 +1,23 @@
-# internal/crawler/worker — Pool Manager + Distributed Coordination
+# internal/crawler/worker — Pool Manager + Retry/CircuitBreaker
 
 소스: [`internal/crawler/worker/`](../../../../internal/crawler/worker/)
 
 크롤러 단계의 핵심 오케스트레이터. **3-tier priority Kafka consumer pool** 을 운영하며, **Redis 기반
-ProcessingLock / IngestionLock / RetryScheduler** 로 다중 인스턴스 환경의 중복 처리/재시도를 안전하게
-처리합니다. 추가로 **per-source CircuitBreaker** 로 실패 폭주를 차단합니다.
+RetryScheduler** 로 지연 재시도를 처리하고 **per-source CircuitBreaker** 로 실패 폭주를 차단합니다.
+
+> 단계 무관 distributed lock (ProcessingLock / IngestionLock) 은 [`internal/locks`](../locks/README.md)
+> 로 분리됨 (이슈 #197). fetcher worker 는 `locks.ProcessingLock(stage="fetcher", url)` 형태로 사용.
 
 <br>
 
 ## 핵심 타입
 
-| 타입                          | 위치                                                                   | 책임                                                            |
-|------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------|
-| `PoolManager`                 | [manager.go](../../../../internal/crawler/worker/manager.go)           | 3개 priority pool 의 lifecycle 관리                              |
-| `KafkaConsumerPool`           | [pool.go](../../../../internal/crawler/worker/pool.go)                 | 단일 priority 의 worker goroutine pool + retry 라우팅           |
-| `ProcessingLock` (interface)  | [processing_lock.go](../../../../internal/crawler/worker/processing_lock.go) | URL 동시 처리 방지 — Redis SETNX / Noop 두 구현                |
-| `IngestionLock` (interface)   | [ingestion_lock.go](../../../../internal/crawler/worker/ingestion_lock.go)   | URL pipeline 진입 marker — Publisher 가 사용                  |
-| `RetryScheduler` (interface)  | [retry_scheduler.go](../../../../internal/crawler/worker/retry_scheduler.go) | 지연 retry — Redis ZSET 또는 즉시 republish                  |
-| `CircuitBreakerRegistry`      | [circuit_breaker.go](../../../../internal/crawler/worker/circuit_breaker.go) | per-source 실패율 트래킹 + 차단                              |
+| 타입                           | 위치                                                                   | 책임                                                            |
+|-------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------|
+| `PoolManager`                  | [manager.go](../../../../internal/crawler/worker/manager.go)           | 3개 priority pool 의 lifecycle 관리                              |
+| `KafkaConsumerPool`            | [pool.go](../../../../internal/crawler/worker/pool.go)                 | 단일 priority 의 worker goroutine pool + retry 라우팅           |
+| `RetryScheduler` (interface)   | [retry_scheduler.go](../../../../internal/crawler/worker/retry_scheduler.go) | 지연 retry — Redis ZSET 또는 즉시 republish                  |
+| `CircuitBreakerRegistry`       | [circuit_breaker.go](../../../../internal/crawler/worker/circuit_breaker.go) | per-source 실패율 트래킹 + 차단                              |
 | `PriorityResolver` (interface) | [resolver.go](../../../../internal/crawler/worker/resolver.go)         | retry/escalation 시 새 priority 결정 전략                       |
 
 <br>
@@ -54,45 +54,6 @@ ProcessingLock / IngestionLock / RetryScheduler** 로 다중 인스턴스 환경
 
 <br>
 
-## ProcessingLock (Redis SETNX 또는 Noop)
-
-이슈 #178. 동일 URL 이 여러 worker / 인스턴스에서 단계별 (fetcher / parser / validator) 중복 처리되는 것을
-차단. 인터페이스는 prebuilt key 만 받으며, **stage 분기는 별도 helper `ProcessingKey(stage, url)` 가
-담당** — 호출자가 정규화된 URL 을 hashed key 로 변환해 전달.
-
-```go
-type ProcessingLock interface {
-    Acquire(ctx context.Context, key string) (acquired bool, err error)
-    Release(ctx context.Context, key string) error
-}
-
-// helper — (stage, normalized_url) → Redis key
-func ProcessingKey(stage, url string) string
-```
-
-stage 상수는 패키지 외부에서도 일관 사용 가능 (예: parser worker 가 `worker.ProcessingKey(StageParser, url)`).
-
-구현:
-- [`NewRedisProcessingLock`](../../../../internal/crawler/worker/processing_lock.go) — Redis `SET NX PX ttl`
-- [`NoopProcessingLock`](../../../../internal/crawler/worker/processing_lock.go) — 항상 acquired=true (단일 프로세스 / dev)
-
-<br>
-
-## IngestionLock (Publisher 와 짝)
-
-[`internal/publisher`](../publisher.md) 가 Kafka enqueue **직전**에 URL 을 marking — 이미 marked URL 은
-Publisher 가 무시. ProcessingLock 은 처리 단계 dedup, IngestionLock 은 진입 dedup 으로 책임 분리.
-
-```go
-type IngestionLock interface {
-    Acquire(ctx context.Context, url string) (acquired bool, err error)
-}
-```
-
-기본 TTL 은 `redisCfg.IngestionLockTTL` (ENV 로 조정).
-
-<br>
-
 ## RetryScheduler (Redis ZSET 또는 즉시)
 
 이슈 #82. retry 대상 job 을 **worker slot 점유 없이** 미래 시점에 재발행.

docs/architecture/internal/locks/README.md

@@ -0,0 +1,71 @@
+## internal/locks — Stage-Agnostic Distributed Coordination
+
+소스: [`internal/locks/`](../../../../internal/locks/)
+
+크롤링 파이프라인 단계 (fetcher / parser / validator) 간 공통으로 사용하는 **Redis 기반 distributed lock**
+인터페이스. 단일 패키지로 분리되어 어떤 stage 도 동일 lock 인스턴스를 공유 — 단계 prefix 만으로 stage 구분.
+
+## 핵심 타입
+
+| 타입                              | 위치                                                                | 책임                                                          |
+|----------------------------------|---------------------------------------------------------------------|---------------------------------------------------------------|
+| `IngestionLock` (interface)       | [ingestion_lock.go](../../../../internal/locks/ingestion_lock.go)   | URL pipeline 진입 marker — Publisher 가 사용 (이슈 #178, #126) |
+| `RedisIngestionLock`              | [ingestion_lock.go](../../../../internal/locks/ingestion_lock.go)   | Redis SETNX 기반 구현                                          |
+| `NoopIngestionLock`               | [ingestion_lock.go](../../../../internal/locks/ingestion_lock.go)   | 항상 acquired=true (단일 프로세스 / dev)                        |
+| `ProcessingLock` (interface)      | [processing_lock.go](../../../../internal/locks/processing_lock.go) | URL 동시 처리 방지 — fetcher/parser/validator 공유 (이슈 #178)  |
+| `RedisProcessingLock`             | [processing_lock.go](../../../../internal/locks/processing_lock.go) | Redis SETNX 기반 구현                                          |
+| `NoopProcessingLock`              | [processing_lock.go](../../../../internal/locks/processing_lock.go) | 항상 acquired=true (단일 프로세스 / dev)                        |
+| `ProcessingKey(stage, url)`       | [processing_lock.go](../../../../internal/locks/processing_lock.go) | (stage, normalized_url) → Redis key 변환 헬퍼                  |
+| `Stage{Fetcher,Parser,Validator}` | [processing_lock.go](../../../../internal/locks/processing_lock.go) | stage 상수                                                     |
+
+## ProcessingLock 패턴
+
+```go
+type ProcessingLock interface {
+    Acquire(ctx context.Context, key string) (acquired bool, err error)
+    Release(ctx context.Context, key string) error
+}
+
+key := locks.ProcessingKey(locks.StageFetcher, normalizedURL)
+acquired, err := lock.Acquire(ctx, key)
+if !acquired {
+    // 다른 worker 가 처리 중 — skip + commit
+}
+defer lock.Release(ctx, key)
+```
+
+stage 분기는 호출자 책임 — `locks` 패키지 자체는 단순 key/value SETNX 만 노출.
+
+## IngestionLock 패턴
+
+```go
+type IngestionLock interface {
+    Acquire(ctx context.Context, url string) (acquired bool, err error)
+    Invalidate(ctx context.Context, url string) error
+}
+
+// Publisher 가 enqueue 직전 호출
+acquired, err := lock.Acquire(ctx, normalizedURL)
+if !acquired {
+    // 이미 pipeline 진입 marker 있음 — 발행 skip
+}
+```
+
+기본 TTL 은 `redisCfg.IngestionLockTTL` (`INGESTION_LOCK_TTL` 환경변수).
+
+## 호출 측
+
+- [`internal/publisher`](../publisher.md) — `IngestionLock` (Kafka enqueue 직전 dedup)
+- [`internal/crawler/worker`](../crawler/worker.md) — fetcher worker pool 의 `ProcessingLock(StageFetcher)`
+- [`internal/parser/worker`](../parser/README.md) — parser worker 의 `ProcessingLock(StageParser)`
+- [`internal/processor/validate`](../processor/validate.md) — validator 의 `ProcessingLock(StageValidator)`
+
+## 외부 시스템
+
+- **Redis**: SETNX (`SET NX PX ttl`) — Acquire / Release / Invalidate
+
+## 관련 이슈
+
+- 이슈 #126 — URL dedup 도입 (Ingestion Lock 의 전신)
+- 이슈 #178 — Ingestion Lock + Processing Lock 분리, 단계별 단일 책임화
+- 이슈 #197 — `crawler/worker` 에서 lock 인프라를 `internal/locks` 로 분리 (단계 무관 패키지화)

docs/architecture/internal/parser/README.md

@@ -64,7 +64,7 @@ ParserWorker 가 이상 종료 / rule.Error 잔존 / LLM 재처리 윈도우 만
 - [`internal/parser/rule`](rule.md) — `Parser`, `Resolver`
 - [`internal/parser/rule/llmgen`](rule.md) — `Generator.Enqueue` (선택)
 - [`internal/crawler/domain/general`](../crawler/domain.md) — `ConvertPageToContent`
-- [`internal/crawler/worker`](../crawler/worker.md) — `ProcessingLock`
+- [`internal/locks`](../locks/README.md) — `ProcessingLock`
 - [`internal/storage/service`](../storage/service.md) — `RawContentService`, `ContentService`
 - [`internal/storage`](../storage/README.md) — `SampleURLRepository`
 - [`internal/publisher`](../publisher.md) — chained job 발행

docs/architecture/internal/processor/validate.md

@@ -60,7 +60,7 @@ graceful shutdown 시 in-flight 메시지를 `drainTimeout` 동안 finalize.
 ## 의존
 
 - [`internal/crawler/core`](../crawler/core.md)
-- [`internal/crawler/worker`](../crawler/worker.md) — `ProcessingLock` (issuetracker 통합 모드) / `NoopProcessingLock` (processor 단독 모드)
+- [`internal/locks`](../locks/README.md) — `ProcessingLock` (issuetracker 통합 모드) / `NoopProcessingLock` (processor 단독 모드)
 - [`internal/storage/service`](../storage/service.md) — `ContentService`
 - [`internal/storage`](../storage/README.md) — `ValidationStatus`
 - [`pkg/queue`](../../pkg/queue.md), [`pkg/config`](../../pkg/config.md), [`pkg/logger`](../../pkg/logger.md)

docs/architecture/internal/publisher.md

@@ -10,7 +10,7 @@
 ## 책임
 
 - URL 정규화 ([`pkg/links.Normalizer`](../pkg/links.md))
-- IngestionLock atomic dedup ([`crawler/worker.IngestionLock`](crawler/worker.md))
+- IngestionLock atomic dedup ([`locks.IngestionLock`](locks/README.md))
 - URL Guard 검사 ([`pkg/urlguard.Gate`](../pkg/urlguard.md))
 - Priority 결정 ([`crawler/worker.PriorityResolver`](crawler/worker.md))
 - 토픽 라우팅 (Priority → TopicCrawlHigh/Normal/Low)
@@ -61,7 +61,8 @@ for each job in batch:
 ## 의존
 
 - [`internal/crawler/core`](crawler/core.md) — `CrawlJob`
-- [`internal/crawler/worker`](crawler/worker.md) — `PriorityResolver`, `IngestionLock`
+- [`internal/crawler/worker`](crawler/worker.md) — `PriorityResolver`
+- [`internal/locks`](locks/README.md) — `IngestionLock`
 - [`pkg/queue`](../pkg/queue.md), [`pkg/links`](../pkg/links.md), [`pkg/urlguard`](../pkg/urlguard.md), [`pkg/logger`](../pkg/logger.md)
 
 <br>