diff --git a/docs/figures/neuron/phase11/adj_dist_compare.png b/docs/figures/neuron/phase11/adj_dist_compare.png new file mode 100644 index 0000000..930d5bf Binary files /dev/null and b/docs/figures/neuron/phase11/adj_dist_compare.png differ diff --git a/docs/figures/neuron/phase11/loss_curves.png b/docs/figures/neuron/phase11/loss_curves.png new file mode 100644 index 0000000..5998efd Binary files /dev/null and b/docs/figures/neuron/phase11/loss_curves.png differ diff --git a/notebooks/02-function-level/10-phase11-scale-corrected-init.ipynb b/notebooks/02-function-level/10-phase11-scale-corrected-init.ipynb new file mode 100644 index 0000000..526ead2 --- /dev/null +++ b/notebooks/02-function-level/10-phase11-scale-corrected-init.ipynb @@ -0,0 +1,365 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# 10-phase11-scale-corrected-init\n", + "\n", + "**neuron Phase 11** — Phase 10 의 결정적 발견 (\"sweet spot magnitude 는 자유도의 의미적 위치에\n", + "의존\") 의 빠른 검증. ChannelGraphLinear 의 adj 에 **scale-corrected init** 적용.\n", + "\n", + "핵심 가설:\n", + "1. **uniform_around_one ≈ channel_full** — 1.0 근처 noise 도 magnitude 균형 유지 + adj 학습 활성\n", + "2. **uniform_around_one > uniform_small** — magnitude rule 직접 입증 (small=10% scale vs around_one=100% scale)\n", + "3. **plain ↔ channel_full ↔ uniform_around_one 모두 ≈** — graph 구조의 free 성 + scale-corrected init 동등성\n", + "\n", + "설계: 4-way × 2 seed = 8 run, max_steps=1500.\n", + "- arch ∈ {plain, channel_full, channel_uniform_small, channel_uniform_around_one}\n", + "- seed ∈ {42, 123}\n", + "\n", + "데이터: TinyShakespeare (char-LM)\n", + "시드: [42, 123]\n", + "작성일: 2026-05-26\n", + "연관: Issue [#63](https://github.com/EinSofINTEREST/GraphLM/issues/63) / Phase 10 baseline PR [#62](https://github.com/EinSofINTEREST/GraphLM/pull/62)" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. 환경" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "import sys\n", + "\n", + "import torch\n", + "\n", + "import graphlm\n", + "from graphlm.data.tinyshakespeare import (\n", + " CharTokenizer,\n", + " TinyShakespeareDataset,\n", + " load_tinyshakespeare_text,\n", + ")\n", + "from graphlm.neuron.graph_channel_demo import train_channel_graph_mlp\n", + "from graphlm.utils import repo_root\n", + "\n", + "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "logging.basicConfig(\n", + " level=logging.WARNING, format=\"%(asctime)s %(levelname)s %(message)s\", datefmt=\"%H:%M:%S\"\n", + ")\n", + "print(\"python :\", sys.version.split()[0])\n", + "print(\"graphlm :\", graphlm.__version__)\n", + "print(\"torch :\", torch.__version__)\n", + "print(\"device :\", DEVICE)" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## 2. 실험 설정" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "ROOT = repo_root()\n", + "DATA_PATH = ROOT / \"data\" / \"tinyshakespeare.txt\"\n", + "OUT_DIR = ROOT / \"runs\" / \"notebook-neuron-phase11\"\n", + "OUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "SEEDS = [42, 123]\n", + "ARCHS = [\"plain\", \"channel_full\", \"channel_uniform_small\", \"channel_uniform_around_one\"]\n", + "EMB_DIM = 64\n", + "HIDDEN_DIM = 256\n", + "N_GRAM = 4\n", + "BATCH_SIZE = 32\n", + "LR = 3e-4\n", + "MAX_STEPS = 1500\n", + "\n", + "# Phase 10 baseline (PR #62)\n", + "PHASE10_PLAIN_MEAN = 2.1487\n", + "PHASE10_CHANNEL_FULL_MEAN = 2.1339\n", + "PHASE10_CHANNEL_UNIFORM_SMALL_MEAN = 2.3268 # +0.18 열위 — magnitude scale 함정\n", + "\n", + "print(f\"SEEDS = {SEEDS}\")\n", + "print(f\"ARCHS = {ARCHS}\")\n", + "print(f\"HIDDEN_DIM = {HIDDEN_DIM}\")\n", + "print(f\"MAX_STEPS = {MAX_STEPS}\")\n", + "print(f\"전체 run = {len(SEEDS) * len(ARCHS)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 3. 데이터 로드" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "text = load_tinyshakespeare_text(DATA_PATH)\n", + "tokenizer = CharTokenizer(text)\n", + "dataset = TinyShakespeareDataset(text, tokenizer)\n", + "V = tokenizer.vocab_size\n", + "print(f\"vocab_size : {V}\")" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## 4. sweep 학습" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "runs = {}\n", + "for seed in SEEDS:\n", + " for arch in ARCHS:\n", + " key = (seed, arch)\n", + " print(f\"--- seed={seed}, arch={arch} ---\")\n", + " runs[key] = train_channel_graph_mlp(\n", + " dataset=dataset,\n", + " vocab_size=V,\n", + " seed=seed,\n", + " arch=arch,\n", + " emb_dim=EMB_DIM,\n", + " hidden_dim=HIDDEN_DIM,\n", + " n_gram=N_GRAM,\n", + " batch_size=BATCH_SIZE,\n", + " lr=LR,\n", + " max_steps=MAX_STEPS,\n", + " device=DEVICE,\n", + " )\n", + " print(f\" done: final_loss={runs[key]['final_loss']:.4f}\")\n", + " if str(DEVICE).startswith(\"cuda\"):\n", + " torch.cuda.empty_cache()" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "## 5. 결과 표 + Phase 10 baseline 비교" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "import statistics\n", + "\n", + "print(f\"{'arch':>32} {'seed':>5} {'final_loss':>11}\")\n", + "print(\"-\" * 60)\n", + "for arch in ARCHS:\n", + " for seed in SEEDS:\n", + " r = runs[(seed, arch)]\n", + " print(f\"{arch:>32} {seed:>5} {r['final_loss']:>11.4f}\")\n", + "\n", + "print()\n", + "print(\"=== arch 별 mean ===\")\n", + "agg = {}\n", + "for arch in ARCHS:\n", + " fls = [runs[(s, arch)][\"final_loss\"] for s in SEEDS]\n", + " agg[arch] = dict(mean=statistics.mean(fls), range=max(fls) - min(fls))\n", + " print(f\" {arch:>32}: mean={agg[arch]['mean']:.4f}, range={agg[arch]['range']:.4f}\")\n", + "\n", + "print()\n", + "print(\"=== Phase 10 baseline (PR #62) 비교 ===\")\n", + "print(f\" Phase 10 plain : {PHASE10_PLAIN_MEAN:.4f}\")\n", + "print(f\" Phase 10 channel_full : {PHASE10_CHANNEL_FULL_MEAN:.4f}\")\n", + "print(\n", + " f\" Phase 10 channel_uniform_small : {PHASE10_CHANNEL_UNIFORM_SMALL_MEAN:.4f} (+0.18 열위 — magnitude 함정)\"\n", + ")\n", + "print()\n", + "print(f\" Phase 11 plain (재현) : {agg['plain']['mean']:.4f}\")\n", + "print(f\" Phase 11 channel_full (재현) : {agg['channel_full']['mean']:.4f}\")\n", + "print(f\" Phase 11 channel_uniform_small (재현) : {agg['channel_uniform_small']['mean']:.4f}\")\n", + "print(\n", + " f\" Phase 11 channel_uniform_around_one : {agg['channel_uniform_around_one']['mean']:.4f} ← 핵심 검증\"\n", + ")\n", + "\n", + "print()\n", + "print(\"=== 자동 verdict ===\")\n", + "plain = agg[\"plain\"][\"mean\"]\n", + "full = agg[\"channel_full\"][\"mean\"]\n", + "around_one = agg[\"channel_uniform_around_one\"][\"mean\"]\n", + "small = agg[\"channel_uniform_small\"][\"mean\"]\n", + "range_max = max(agg[a][\"range\"] for a in ARCHS)\n", + "print(f\" range_max = {range_max:.4f}\")\n", + "print(f\" around_one vs plain : {around_one - plain:+.4f}\")\n", + "print(f\" around_one vs channel_full : {around_one - full:+.4f}\")\n", + "print(f\" around_one vs uniform_small : {around_one - small:+.4f} (음수면 magnitude rule 입증)\")\n", + "if around_one - small < -0.05:\n", + " print(\" ✅ magnitude rule 입증 — around_one 이 uniform_small 보다 명확히 우위\")\n", + "if abs(around_one - full) < range_max:\n", + " print(\" ✅ around_one ≈ channel_full — scale balance 작동\")" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "## 6. 학습된 adj distribution 비교 — 4 arch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "# channel_* 3 arch 의 fc1 adj 분포 비교\n", + "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n", + "arch_names = [\"channel_full\", \"channel_uniform_small\", \"channel_uniform_around_one\"]\n", + "colors = [\"#1f77b4\", \"#ff7f0e\", \"#2ca02c\"]\n", + "\n", + "for i, arch in enumerate(arch_names):\n", + " r = runs[(SEEDS[0], arch)]\n", + " if r[\"final_adj\"] is None:\n", + " continue\n", + " adj = r[\"final_adj\"][\"fc1\"].numpy().flatten()\n", + " axes[i].hist(adj, bins=80, alpha=0.7, color=colors[i])\n", + " axes[i].set_xlabel(\"adj value\")\n", + " axes[i].set_ylabel(\"count\")\n", + " axes[i].set_title(f\"{arch}\\nmean={adj.mean():.3f}, std={adj.std():.3f}\")\n", + " axes[i].axvline(0, color=\"red\", linestyle=\"--\", lw=0.8, alpha=0.5)\n", + " axes[i].axvline(adj.mean(), color=\"green\", linestyle=\":\", lw=1)\n", + " axes[i].grid(alpha=0.3)\n", + "\n", + "fig.suptitle(f\"Phase 11 — channel adj 분포 비교 (seed={SEEDS[0]}, fc1)\", fontsize=11)\n", + "fig.tight_layout()\n", + "fig.savefig(OUT_DIR / \"adj_dist_compare.png\", dpi=120)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "## 7. loss curve 비교 — 4 arch mean ± σ" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "window = 30\n", + "colors = {\n", + " \"plain\": \"#1f77b4\",\n", + " \"channel_full\": \"#2ca02c\",\n", + " \"channel_uniform_small\": \"#ff7f0e\",\n", + " \"channel_uniform_around_one\": \"#d62728\",\n", + "}\n", + "\n", + "fig, ax = plt.subplots(figsize=(13, 5))\n", + "for arch in ARCHS:\n", + " seed_curves = []\n", + " for seed in SEEDS:\n", + " losses = runs[(seed, arch)][\"losses\"]\n", + " smoothed = np.convolve(losses, np.ones(window) / window, mode=\"valid\")\n", + " seed_curves.append(smoothed)\n", + " arr = np.array(seed_curves)\n", + " steps = np.arange(window - 1, window - 1 + arr.shape[1])\n", + " mean = arr.mean(axis=0)\n", + " std = arr.std(axis=0, ddof=1)\n", + " ax.plot(steps, mean, color=colors[arch], lw=1.5, label=arch)\n", + " ax.fill_between(steps, mean - std, mean + std, color=colors[arch], alpha=0.15)\n", + "ax.axhline(\n", + " PHASE10_CHANNEL_FULL_MEAN,\n", + " color=\"gray\",\n", + " linestyle=\":\",\n", + " lw=1,\n", + " alpha=0.7,\n", + " label=f\"Phase 10 channel_full ({PHASE10_CHANNEL_FULL_MEAN})\",\n", + ")\n", + "ax.set_xlabel(\"step\")\n", + "ax.set_ylabel(f\"loss (smoothed window={window})\")\n", + "ax.set_title(\"Phase 11 — 4 arch loss curve (mean ± σ across 2 seeds)\")\n", + "ax.legend(loc=\"upper right\", fontsize=9)\n", + "ax.grid(alpha=0.3)\n", + "fig.tight_layout()\n", + "fig.savefig(OUT_DIR / \"loss_curves.png\", dpi=120)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "## 결과 요약 / Phase 12 권장\n", + "\n", + "확인 포인트:\n", + "- §5 around_one vs uniform_small (마이너스 시 magnitude rule 입증)\n", + "- §5 around_one vs channel_full (range 보다 작은 차이 시 scale balance 작동)\n", + "- §6 around_one 의 adj 분포 — 1.0 근처 spread 가 학습 후 어떻게 변화?\n", + "- §7 loss curve — around_one 이 uniform_small 보다 명확히 낮은 위치?\n", + "\n", + "**판정 시나리오**:\n", + "- **A. around_one ≈ channel_full + uniform_small 명확 열위 유지** ⭐ — magnitude rule 확정, Phase 12 (hybrid) 진입\n", + "- **B. around_one > channel_full** — noise 가 학습 가속 (implicit pruning 활성), surprise positive\n", + "- **C. around_one ≈ uniform_small** — magnitude 만으로 부족, 다른 요인 (예: weight 와의 동시 학습 dynamics)\n", + "\n", + "**참고**:\n", + "- Phase 10 결과: https://www.notion.so/36ce8b70b7aa81ff82a6edd3e2d03770\n", + "- 아키텍처 구성 계획: https://www.notion.so/36ce8b70b7aa818cbf1fe71687b449b8" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "GraphLM (uv .venv)", + "language": "python", + "name": "graphlm-uv" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/graphlm/neuron/graph_channel.py b/src/graphlm/neuron/graph_channel.py index 2153017..85fbde2 100644 --- a/src/graphlm/neuron/graph_channel.py +++ b/src/graphlm/neuron/graph_channel.py @@ -16,13 +16,15 @@ - node = 1 채널 (graph 의 vertex) - edge = (in, out) 쌍의 connection — adj 가 routing strength, W 가 transformation - adj=full (모두 1) → effective = W → standard Linear forward 동치 (function preserving) -- adj=uniform_small → Phase 2 sweet spot 패턴 channel-level edge 에 적용 (0-init 회피) +- adj=uniform_around_one (Phase 11+ 권장) → 1.0 근처 noise — scale 균형 + adj 학습 활성 +- adj=uniform_small → **anti-pattern** — Phase 2 sweet spot (0.10) 를 weight multiplier 위치에 + 잘못 적용한 사례 (Phase 10 의 +0.18 열위 발견, magnitude rule 위반) -**0-init 금지 규칙 적용** (rationale: Phase 9 결과 PR #60 + -[feedback_no_zero_init.md](https://www.notion.so/36ce8b70b7aa8100b0acf756686d2e9f) Notion 정리): -- Phase 1 dead block / Phase 7 amplitude vanishing / Phase 9 block-diagonal 의 3차 반복 발견 -- 본 모듈은 ``adj_init="zero"`` 옵션 명시적 거부 (ValueError) -- default 권장 = ``"full"`` (function preserving) 또는 ``"uniform_small"`` (sweet spot 패턴) +**0-init 금지 + magnitude rule** (rationale: Phase 9 PR #60 + Phase 10 PR #62): +- 0-init 금지: Phase 1 dead block / Phase 7 amplitude vanishing / Phase 9 block-diagonal 의 + 3차 반복 발견. 본 모듈은 ``adj_init="zero"`` 명시적 거부 (ValueError) +- magnitude rule (Phase 10 추가): residual gate ≈ 0.10, **weight multiplier ≈ 1.0** +- default 권장 = ``"full"`` (function preserving) 또는 ``"uniform_around_one"`` (scale-corrected) Phase 9 (GroupGraphLinear) 와의 차이: - Phase 9: group_size 단위 block 으로 묶어 (n_groups_out, n_groups_in) adjacency @@ -38,7 +40,7 @@ import torch.nn.functional as F from torch import Tensor, nn -AdjInit = Literal["full", "uniform_small"] +AdjInit = Literal["full", "uniform_small", "uniform_around_one"] class ChannelGraphLinear(nn.Module): @@ -47,10 +49,13 @@ class ChannelGraphLinear(nn.Module): Args: in_features: 입력 채널 수 (graph 의 in-nodes). out_features: 출력 채널 수 (graph 의 out-nodes). - adj_init: ``"full"`` (모두 1, function preserving) 또는 ``"uniform_small"`` - (uniform[0.05, 0.15], Phase 2 sweet spot 패턴). ``"zero"`` 등 0-init 옵션은 - 거부 (rationale: Phase 9 결과 PR #60 — block-diagonal sparse 시작이 +0.14 loss - 열위, vanishing gradient 함정). + adj_init: 선택지 (memory: feedback_no_zero_init.md 의 0-init 금지 + magnitude rule): + - ``"full"`` — 모두 1 (function preserving, standard Linear 와 forward 동치) + - ``"uniform_around_one"`` — uniform[0.95, 1.05] (1.0 근처 small noise, + weight multiplier 의 적정 magnitude + 0-init 회피 + adj 학습 활성, Phase 11+ 권장) + - ``"uniform_small"`` — uniform[0.05, 0.15] (Phase 2 residual-gate sweet spot 패턴, + **anti-pattern** — weight multiplier 위치엔 magnitude 가 작아 +0.18 열위, Phase 10 실측) + - ❌ ``"zero"`` — 거부 (Phase 1/7/9 vanishing 함정 3차 재현, rationale: PR #60) bias: bias 사용 여부 (standard Linear 동일). Forward: @@ -79,18 +84,24 @@ def __init__( bound = 1.0 / math.sqrt(in_features) nn.init.uniform_(self.weight, -bound, bound) - # adj init — 0-init 거부 (feedback_no_zero_init.md) + # adj init — 0-init 거부 + magnitude rule (memory: feedback_no_zero_init.md) if adj_init == "full": adj = torch.ones(out_features, in_features) elif adj_init == "uniform_small": - # Phase 2 sweet spot (0.10) ± δ — gradient flow 확보 + edge importance 학습 시작 + # Phase 2 residual-gate sweet spot (0.10) ± δ — *주의*: weight multiplier 위치에는 + # magnitude rule 위반 (effective_w 가 10% scale 로 줄어 +0.18 열위, Phase 10 발견). + # 유지 이유: ablation 비교용 / 명시적 anti-pattern 데모. adj = torch.empty(out_features, in_features).uniform_(0.05, 0.15) + elif adj_init == "uniform_around_one": + # Phase 11 scale-corrected: weight multiplier 의 적정 magnitude ≈ 1.0 + small noise + # (memory: feedback_no_zero_init.md 의 magnitude rule). 0-init 회피 + scale 균형. + adj = torch.empty(out_features, in_features).uniform_(0.95, 1.05) elif adj_init in {"zero", "zeros"}: raise ValueError( f"adj_init={adj_init!r} 는 금지됨 — 0-init 은 vanishing gradient 함정 " "(Phase 1 dead block / Phase 7 amplitude vanishing / Phase 9 block-diagonal " - "에서 3차 재현). rationale: Phase 9 PR #60. " - "'full' (function preserving) 또는 'uniform_small' (sweet spot 패턴) 사용 권장." + "에서 3차 재현). rationale: Phase 9 PR #60, magnitude rule: Phase 10 PR #62. " + "'full' (function preserving) 또는 'uniform_around_one' (scale-corrected) 사용 권장." ) else: raise ValueError(f"unknown adj_init: {adj_init!r}") diff --git a/src/graphlm/neuron/graph_channel_demo.py b/src/graphlm/neuron/graph_channel_demo.py index e2d2452..0095a3b 100644 --- a/src/graphlm/neuron/graph_channel_demo.py +++ b/src/graphlm/neuron/graph_channel_demo.py @@ -1,13 +1,15 @@ -"""Phase 10 — ChannelGraphLinear demo MLP-LM + 학습 헬퍼. +"""Phase 10/11 — ChannelGraphLinear demo MLP-LM + 학습 헬퍼. -노트북 분리 규약 준수. Phase 10 노트북 09-phase10-channel-graph-foundations.ipynb 는 여기서 +노트북 분리 규약 준수. Phase 10 / Phase 11 노트북 (09-phase10-..., 10-phase11-...) 모두 여기서 import. Phase 9 의 ``graph_group_demo`` 의 channel-level 대응. -3 가지 architecture 를 같은 학습 루프로 비교: +4 가지 architecture 를 같은 학습 루프로 비교: - ``"plain"`` — 표준 nn.Linear (baseline) - ``"channel_full"`` — ChannelGraphLinear with adj_init="full" (function preserving 시작) - ``"channel_uniform_small"`` — ChannelGraphLinear with adj_init="uniform_small" - (Phase 2 sweet spot 패턴 channel-level edge 에 적용) + (Phase 2 residual-gate sweet spot 패턴, **anti-pattern** — Phase 10 magnitude rule 위반) +- ``"channel_uniform_around_one"`` — ChannelGraphLinear with adj_init="uniform_around_one" + (Phase 11+ 권장 — 1.0 근처 noise, scale 균형 + 0-init 회피) """ from __future__ import annotations @@ -23,7 +25,7 @@ from graphlm.neuron.graph_channel import ChannelGraphLinear from graphlm.utils import set_seed -Arch = Literal["plain", "channel_full", "channel_uniform_small"] +Arch = Literal["plain", "channel_full", "channel_uniform_small", "channel_uniform_around_one"] class ChannelGraphMLPLM(nn.Module): @@ -61,6 +63,8 @@ def _make_linear(in_f: int, out_f: int, arch: Arch) -> nn.Module: return ChannelGraphLinear(in_f, out_f, adj_init="full") if arch == "channel_uniform_small": return ChannelGraphLinear(in_f, out_f, adj_init="uniform_small") + if arch == "channel_uniform_around_one": + return ChannelGraphLinear(in_f, out_f, adj_init="uniform_around_one") raise ValueError(f"unknown arch: {arch}") diff --git a/tests/neuron/test_graph_channel.py b/tests/neuron/test_graph_channel.py index 95d0117..75ea187 100644 --- a/tests/neuron/test_graph_channel.py +++ b/tests/neuron/test_graph_channel.py @@ -30,6 +30,37 @@ def test_uniform_small_adj_init_range(): assert lin.adj.std() > 0.01 +def test_uniform_around_one_adj_init_range(): + """adj_init='uniform_around_one' → uniform[0.95, 1.05] (Phase 11 scale-corrected).""" + torch.manual_seed(0) + lin = ChannelGraphLinear(8, 16, adj_init="uniform_around_one") + assert (lin.adj >= 0.95).all() and (lin.adj <= 1.05).all() + # mean 이 ≈1.0 근처 (scale 균형 — weight multiplier 의 적정 magnitude) + assert abs(lin.adj.mean().item() - 1.0) < 0.05 + # 분포 spread (degenerate 거부) + assert lin.adj.std() > 0.005 + + +def test_uniform_around_one_function_preservation_approximate(): + """uniform_around_one 은 정확히 동치는 아니지만 (noise ±0.05), magnitude balance 는 standard Linear 수준. + + 구체: |effective_w| 의 mean 이 |W| mean 과 ±10% 이내 (vs uniform_small 은 ~10x 작음). + """ + torch.manual_seed(0) + cg = ChannelGraphLinear(64, 128, adj_init="uniform_around_one") + effective = cg.adj * cg.weight + # |effective_w| mean 이 |W| mean 의 ~1.0 배 ± 10% + ratio = effective.abs().mean().item() / cg.weight.abs().mean().item() + assert 0.9 < ratio < 1.1 + + # 대조: uniform_small 은 ratio ~0.10 (10% 수준) + torch.manual_seed(0) + cg_small = ChannelGraphLinear(64, 128, adj_init="uniform_small") + effective_small = cg_small.adj * cg_small.weight + ratio_small = effective_small.abs().mean().item() / cg_small.weight.abs().mean().item() + assert ratio_small < 0.2 # ~0.10 으로 훨씬 작음 + + @pytest.mark.parametrize("bad_init", ["zero", "zeros"]) def test_zero_init_rejected(bad_init): """0-init 옵션 명시적 거부 — Phase 1/7/9 의 vanishing gradient 함정 회피."""