"""
词频统计：从最新结构化任务读取产品与 CSV，两步连跑。

  1. 随机 25 条 content → LLM 归纳「产品专有名词」与「Amazon/产品专属停用词」
  2. NLTK 全量 content 分词 + stem/lemma 并族 → output/word_freq.csv
     （专有名词按完整短语统计；每条评论每个词族最多计 1 次）

用法::

  ./310py/bin/python 词频.py
  ./310py/bin/python 词频.py --skip-llm   # 复用 output/voc_terms.json，仅跑第 2 步
"""
from __future__ import annotations

import argparse
import csv
import json
import logging
import random
import re
import sqlite3
import sys
from collections import Counter, defaultdict
from pathlib import Path
from typing import Dict, Iterable, List, Sequence, Set, Tuple

from voc_llm import CHAT_MODEL, chat_extra_body, create_chat_client, require_chat_api_key

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    stream=sys.stderr,
)
logger = logging.getLogger("voc_wordfreq")

PROJECT_ROOT = Path(__file__).resolve().parent
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
OUTPUT_DIR = PROJECT_ROOT / "output"
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"

MODEL_NAME = CHAT_MODEL

SAMPLE_SIZE = 38
SAMPLE_SEED = 42

# NLTK 英文停用词扩展（与匹配规则参考一致）
_EXTRA_STOP_WORDS = frozenset(
    {
        "www", "http", "https", "com", "amazon", "asin", "sku",
        "oz", "lb", "lbs", "inch", "inches", "ft", "mm", "cm", "ml", "kg", "pcs", "pc",
    }
)

# 产品标题拆词中可视为「不重要」、应参与停用的功能词
_PRODUCT_NAME_FILLER = frozenset(
    {
        "a", "an", "the", "and", "or", "but", "for", "nor", "so", "yet",
        "at", "by", "in", "of", "on", "to", "up", "as", "is", "it", "be",
        "with", "from", "into", "via", "per", "vs", "vs.",
    }
)


def _ensure_nltk():
    """加载 NLTK 分词 / 词性 / 词形还原依赖（首次自动下载数据包）。"""
    try:
        import nltk  # noqa: F401
    except ImportError as e:
        raise RuntimeError(
            "未安装 nltk，请执行: uv pip install --python 310py/bin/python nltk"
        ) from e
    import nltk
    from nltk.corpus import wordnet as wn
    from nltk.stem import PorterStemmer, WordNetLemmatizer
    from nltk.tag import pos_tag
    from nltk.tokenize import word_tokenize

    for resource, pkg in (
        ("tokenizers/punkt", "punkt"),
        ("tokenizers/punkt_tab", "punkt_tab"),
        ("corpora/wordnet", "wordnet"),
        ("corpora/omw-1.4", "omw-1.4"),
        ("taggers/averaged_perceptron_tagger", "averaged_perceptron_tagger"),
        ("taggers/averaged_perceptron_tagger_eng", "averaged_perceptron_tagger_eng"),
        ("corpora/stopwords", "stopwords"),
    ):
        try:
            nltk.data.find(resource)
        except LookupError:
            logger.info("下载 NLTK 数据包: %s", pkg)
            nltk.download(pkg, quiet=True)

    return word_tokenize, pos_tag, WordNetLemmatizer(), PorterStemmer(), wn


def _penn_to_wn_pos(tag: str, wn) -> str:
    if tag.startswith("J"):
        return wn.ADJ
    if tag.startswith("V"):
        return wn.VERB
    if tag.startswith("N"):
        return wn.NOUN
    if tag.startswith("R"):
        return wn.ADV
    return wn.NOUN


def load_nltk_stop_words() -> Set[str]:
    _ensure_nltk()
    import nltk
    from nltk.corpus import stopwords

    try:
        words = set(stopwords.words("english"))
    except LookupError:
        nltk.download("stopwords", quiet=True)
        words = set(stopwords.words("english"))
    words |= _EXTRA_STOP_WORDS
    return words


class WordVariantEngine:
    """NLTK 分词 + stem / POS-lemma 词形变体（不含同义词并族）。"""

    def __init__(self) -> None:
        word_tokenize, pos_tag_fn, lemmatizer, stemmer, wn = _ensure_nltk()
        self._word_tokenize = word_tokenize
        self._pos_tag = pos_tag_fn
        self._lemmatizer = lemmatizer
        self._stemmer = stemmer
        self._wn = wn

    def variants_for_word(self, word: str, pos: str | None = None) -> Set[str]:
        w = (word or "").lower().strip()
        if not w:
            return set()
        out: Set[str] = {w, self._stemmer.stem(w)}
        if pos is not None:
            wn_pos = _penn_to_wn_pos(pos, self._wn)
            out.add(self._lemmatizer.lemmatize(w, pos=wn_pos))
        for p in (self._wn.NOUN, self._wn.VERB, self._wn.ADJ, self._wn.ADV):
            out.add(self._lemmatizer.lemmatize(w, pos=p))
        return {x for x in out if x}

    def variant_pool(self, word: str, pos: str | None = None) -> Set[str]:
        return self.variants_for_word(word, pos=pos)

    def pos_tag_tokens(self, tokens: Sequence[str]) -> List[Tuple[str, str]]:
        if not tokens:
            return []
        try:
            return self._pos_tag(list(tokens))
        except Exception:
            return [(t, "NN") for t in tokens]


class _UnionFind:
    def __init__(self) -> None:
        self._parent: Dict[str, str] = {}

    def add(self, x: str) -> None:
        if x not in self._parent:
            self._parent[x] = x

    def find(self, x: str) -> str:
        self.add(x)
        while self._parent[x] != x:
            self._parent[x] = self._parent[self._parent[x]]
            x = self._parent[x]
        return x

    def union(self, a: str, b: str) -> None:
        ra, rb = self.find(a), self.find(b)
        if ra != rb:
            self._parent[rb] = ra

    def groups(self) -> Dict[str, List[str]]:
        out: Dict[str, List[str]] = defaultdict(list)
        for x in self._parent:
            out[self.find(x)].append(x)
        return dict(out)


def _strip_think(text: str) -> str:
    if not text:
        return text
    text = re.sub(
        r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
    )
    text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
    return text.strip()


def _latest_job(conn: sqlite3.Connection) -> Tuple[int, str, str, str]:
    row = conn.execute(
        """
        SELECT id, industry, product_name, source_file
        FROM analysis_jobs
        ORDER BY id DESC
        LIMIT 1
        """
    ).fetchone()
    if not row:
        raise RuntimeError(f"{STRUCTURED_DB} 中无 analysis_jobs 记录，请先跑结构化")
    return int(row[0]), str(row[1]), str(row[2]), str(row[3])


def _resolve_source_path(source_file: str) -> Path:
    p = Path(source_file)
    if p.is_file():
        return p.resolve()
    candidate = PROJECT_ROOT / source_file
    if candidate.is_file():
        return candidate.resolve()
    raise FileNotFoundError(f"找不到评论 CSV: {source_file}")


def _load_contents(csv_path: Path) -> List[Tuple[int, str]]:
    rows: List[Tuple[int, str]] = []
    with csv_path.open(encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f)
        if not reader.fieldnames or "content" not in reader.fieldnames:
            raise ValueError(f"{csv_path} 缺少 content 列")
        for i, row in enumerate(reader, start=1):
            text = (row.get("content") or "").strip()
            if text:
                rows.append((i, text))
    if not rows:
        raise ValueError(f"{csv_path} 无有效 content")
    return rows


def _sample_reviews(
    rows: Sequence[Tuple[int, str]], *, n: int, seed: int
) -> List[Tuple[int, str]]:
    rng = random.Random(seed)
    k = min(n, len(rows))
    return rng.sample(list(rows), k)


def _build_terms_prompt(
    industry: str, product_name: str, samples: Sequence[Tuple[int, str]]
) -> Tuple[str, str]:
    blocks = []
    for row_id, text in samples:
        blocks.append(f"[R{row_id}]\n{text}")
    joined = "\n\n---\n\n".join(blocks)
    system = (
        "你是亚马逊美国站英文评论与电商文本分析专家。"
        "根据样本评论，列出两类词表，供后续英文分词与词频统计使用。"
    )
    user = f"""行业：{industry}
产品：{product_name}

以下为该产品 {len(samples)} 条随机评论（仅正文 content）：

{joined}

请输出两部分（可用 Markdown 标题、编号列表或逗号分隔，不必是 JSON）：

## 产品专有名词（product_terms）
列出该产品评论中可能出现的**全部**专有表达，尽量合理扩展，包括但不限于：
成分/品类、宠物品种与病种、剂型包装、用法场景、Amazon 物流售后相关但**属于本产品语境**的词等。
多词短语请保留完整形式（如 turkey tail mushroom powder）；后续词频**只统计完整短语**，不会拆成单词分别计数。
**不要**把下面「专属停用词」里的词放进本列表。

## 专属停用词（stopwords）
列出 Amazon 美国站电商通用词，以及与本产品强相关、词频高但**分析价值低**的词（如泛化评价词、过泛产品描述词等）。
多词短语可保留。
注意：不要把产品名「{product_name}」**整句**列为停用词；也不要把其中的**实词/品类成分**（如 turkey、tail、mushroom）列为停用词。
产品名拆词里**无分析价值的虚词/功能词**（如 for、the、a、of、and）可以且应列入停用词。

每条一行或逗号分隔即可。"""
    return system, user


def _call_llm(system: str, user: str, api_key: str) -> str:
    _ = api_key
    # 术语提取只需短 JSON/列表，200k max_tokens 会导致 API 长时间生成或挂起
    client = create_chat_client(timeout=600.0)
    extra_body = chat_extra_body(MODEL_NAME)
    resp = client.chat.completions.create(
        model=MODEL_NAME,
        messages=[
            {"role": "system", "content": system},
            {"role": "user", "content": user},
        ],
        temperature=0.6,
        max_tokens=90000,
        extra_body=extra_body,
    )
    msg = resp.choices[0].message
    text = msg.content or getattr(msg, "reasoning_content", None) or ""
    if not text.strip():
        raise RuntimeError("LLM 返回为空")
    return _strip_think(text)


def _normalize_term(s: str) -> str:
    s = re.sub(r"\s+", " ", (s or "").strip().lower())
    return s.strip(".,;:\"'""''`-–—()[]{}")


def _split_list_items(line: str) -> List[str]:
    line = re.sub(r"^[\s\d\.\)\-•\*]+", "", line.strip())
    if not line or line.startswith("#"):
        return []
    parts = re.split(r"[,，;；、|]", line)
    return [_normalize_term(p) for p in parts if _normalize_term(p)]


def _try_parse_json_terms(text: str) -> Tuple[List[str], List[str]] | None:
    text = text.replace("```json", "").replace("```", "").strip()
    start, end = text.find("{"), text.rfind("}")
    if start == -1 or end <= start:
        return None
    try:
        obj = json.loads(text[start : end + 1])
    except json.JSONDecodeError:
        return None
    if not isinstance(obj, dict):
        return None

    def pick(keys: Sequence[str]) -> List[str]:
        for k in keys:
            v = obj.get(k)
            if isinstance(v, list):
                return [_normalize_term(str(x)) for x in v if _normalize_term(str(x))]
            if isinstance(v, str):
                return _split_list_items(v)
        return []

    terms = pick(("product_terms", "专有名词", "terms", "keywords"))
    stops = pick(("stopwords", "stop_words", "停用词", "custom_stopwords"))
    if terms or stops:
        return terms, stops
    return None


def _parse_section_lines(text: str, mode: str) -> List[str]:
    """mode: product | stop"""
    lines = text.splitlines()
    collecting = False
    items: List[str] = []
    product_hdr = re.compile(
        r"(专有名词|product[_\s-]*terms?|product\s+terms)", re.I
    )
    stop_hdr = re.compile(
        r"(专属停用|停用词|stop[_\s-]*words?|custom\s+stop)", re.I
    )
    other_hdr = re.compile(r"^#{1,3}\s+")

    for line in lines:
        raw = line.strip()
        if not raw:
            if collecting and items:
                collecting = False
            continue
        if mode == "product" and product_hdr.search(raw):
            collecting = True
            inline = re.split(r"[:：]", raw, maxsplit=1)
            if len(inline) > 1:
                items.extend(_split_list_items(inline[1]))
            continue
        if mode == "stop" and stop_hdr.search(raw):
            collecting = True
            inline = re.split(r"[:：]", raw, maxsplit=1)
            if len(inline) > 1:
                items.extend(_split_list_items(inline[1]))
            continue
        if collecting and (other_hdr.match(raw) or (
            mode == "product" and stop_hdr.search(raw)
        ) or (mode == "stop" and product_hdr.search(raw))):
            collecting = False
            continue
        if collecting:
            items.extend(_split_list_items(raw))
    return items


def _parse_llm_terms(text: str) -> Tuple[List[str], List[str]]:
    parsed = _try_parse_json_terms(text)
    if parsed:
        return parsed
    product = _parse_section_lines(text, "product")
    stop = _parse_section_lines(text, "stop")
    if not product and not stop:
        # 兜底：按行提取非空短语
        for line in text.splitlines():
            items = _split_list_items(line)
            if items and len(items) <= 8:
                product.extend(items)
    return _dedupe_terms(product), _dedupe_terms(stop)


def _dedupe_terms(items: Iterable[str]) -> List[str]:
    seen: Set[str] = set()
    out: List[str] = []
    for x in items:
        if x and x not in seen:
            seen.add(x)
            out.append(x)
    return out


def _product_name_tokens(product_name: str) -> Set[str]:
    return {
        t
        for t in re.findall(r"[a-z0-9]+", product_name.lower())
        if t
    }


def _product_name_filler_tokens(product_name: str, nltk_stops: Set[str]) -> Set[str]:
    """产品名拆词中的虚词/功能词 → 应停用。"""
    tokens = _product_name_tokens(product_name)
    return {t for t in tokens if t in nltk_stops or t in _PRODUCT_NAME_FILLER}


def _product_name_core_tokens(product_name: str) -> Set[str]:
    """产品名中有分析价值的实词 → 不停用。"""
    fillers = _product_name_filler_tokens(product_name, load_nltk_stop_words())
    return _product_name_tokens(product_name) - fillers


def _is_pure_number(s: str) -> bool:
    return bool(s) and s.isdigit()


def _finalize_term_lists(
    product_terms: List[str],
    stopwords: List[str],
    product_name: str,
) -> Tuple[List[str], Set[str]]:
    stop_set = load_nltk_stop_words()
    stop_set.update(_dedupe_terms(stopwords))
    stop_set.update(_product_name_filler_tokens(product_name, stop_set))
    for tok in _product_name_core_tokens(product_name):
        stop_set.discard(tok)
    pn_lower = _normalize_term(product_name)
    if pn_lower:
        stop_set.discard(pn_lower)

    cleaned_terms: List[str] = []
    for t in _dedupe_terms(product_terms):
        if t in stop_set:
            continue
        cleaned_terms.append(t)
    cleaned_terms.sort(key=lambda x: (-len(x.split()), -len(x)))
    return cleaned_terms, stop_set


def _phrase_pattern(phrase: str) -> re.Pattern[str]:
    parts = [re.escape(p) for p in phrase.split()]
    body = r"\s+".join(parts)
    return re.compile(rf"(?<!\w){body}(?!\w)", re.IGNORECASE)


def _merge_spans(spans: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
    if not spans:
        return []
    ordered = sorted(spans)
    merged: List[Tuple[int, int]] = [ordered[0]]
    for start, end in ordered[1:]:
        prev_s, prev_e = merged[-1]
        if start <= prev_e:
            merged[-1] = (prev_s, max(prev_e, end))
        else:
            merged.append((start, end))
    return merged


def _overlaps_span(char_start: int, char_end: int, spans: Sequence[Tuple[int, int]]) -> bool:
    for s, e in spans:
        if char_start < e and char_end > s:
            return True
    return False


def _collect_protected_spans(lower: str, product_terms: List[str], stop_set: Set[str]) -> List[Tuple[int, int]]:
    """返回专有名词短语匹配区间（屏蔽拆词统计，短语单独计数）。"""
    spans: List[Tuple[int, int]] = []
    for phrase in product_terms:
        if not phrase or phrase in stop_set or _is_pure_number(phrase):
            continue
        for m in _phrase_pattern(phrase).finditer(lower):
            spans.append((m.start(), m.end()))
    return _merge_spans(spans)


def _phrases_hit_in_review(lower: str, product_terms: List[str], stop_set: Set[str]) -> List[str]:
    """本条评论命中的专有名词（每条评论每短语最多计 1 次）。"""
    hit: List[str] = []
    for phrase in product_terms:
        if not phrase or phrase in stop_set or _is_pure_number(phrase):
            continue
        if _phrase_pattern(phrase).search(lower):
            hit.append(phrase)
    return hit


_TOKEN_RE = re.compile(r"[a-z0-9']+")


def _tokenize_tagged_with_spans(text: str, engine: WordVariantEngine) -> List[Tuple[str, str, int, int]]:
    """带字符区间的分词 + 词性标注。"""
    lower = text.lower()
    raw: List[Tuple[str, int, int]] = []
    for m in _TOKEN_RE.finditer(lower):
        tok = m.group()
        if tok:
            raw.append((tok, m.start(), m.end()))
    if not raw:
        return []
    words = [t for t, _, _ in raw]
    tagged = engine.pos_tag_tokens(words)
    return [(tagged[i][0], tagged[i][1], raw[i][1], raw[i][2]) for i in range(len(raw))]


def _is_valid_token(tok: str, stop_set: Set[str]) -> bool:
    if not tok or len(tok) < 1 or tok in stop_set or _is_pure_number(tok):
        return False
    return bool(re.search(r"[a-z]", tok))


def _register_review_tokens_in_uf(
    text: str,
    protected: Sequence[Tuple[int, int]],
    engine: WordVariantEngine,
    uf: _UnionFind,
    variant_index: Dict[str, Set[str]],
    stop_set: Set[str],
) -> None:
    """Pass 1：将本条评论 token 注册进 Union-Find（stem/lemma 并族）。"""
    seen: Set[str] = set()
    for tok, tag, start, end in _tokenize_tagged_with_spans(text, engine):
        if _overlaps_span(start, end, protected):
            continue
        if not _is_valid_token(tok, stop_set):
            continue
        if tok in seen:
            continue
        seen.add(tok)
        uf.add(tok)
        variants = engine.variant_pool(tok, pos=tag)
        related: Set[str] = set()
        for v in variants:
            related |= variant_index[v]
        for other in related:
            uf.union(tok, other)
        for v in variants:
            variant_index[v].add(tok)


def _family_labels(uf: _UnionFind, surface_doc_freq: Counter[str]) -> Dict[str, str]:
    """词族代表形：族内 surface 文档频次最高者，并列取最短。"""
    labels: Dict[str, str] = {}
    for root, members in uf.groups().items():
        best = sorted(
            members,
            key=lambda m: (-surface_doc_freq.get(m, 0), len(m), m),
        )[0]
        labels[root] = best
    return labels


def _build_word_freq(
    rows: Sequence[Tuple[int, str]],
    product_terms: List[str],
    stop_set: Set[str],
) -> Counter[str]:
    """
    NLTK 分词 + stem/lemma Union-Find 并族。
    每条评论：每个词族最多 +1；专有名词短语命中也最多 +1/短语。
    输出 word 为词族代表形（方案 1）。
    """
    engine = WordVariantEngine()
    uf = _UnionFind()
    variant_index: Dict[str, Set[str]] = defaultdict(set)

    # Pass 1：全库注册词形变体并族
    for _, text in rows:
        if not text.strip():
            continue
        protected = _collect_protected_spans(text.lower(), product_terms, stop_set)
        _register_review_tokens_in_uf(text, protected, engine, uf, variant_index, stop_set)

    family_counter: Counter[str] = Counter()
    phrase_counter: Counter[str] = Counter()
    surface_doc_freq: Counter[str] = Counter()

    # Pass 2：按评论计数（每词族 / 每短语最多 1 次）
    for _, text in rows:
        if not text.strip():
            continue
        lower = text.lower()
        protected = _collect_protected_spans(lower, product_terms, stop_set)

        for phrase in _phrases_hit_in_review(lower, product_terms, stop_set):
            phrase_counter[phrase] += 1

        families_seen: Set[str] = set()
        seen_tok: Set[str] = set()
        for tok, tag, start, end in _tokenize_tagged_with_spans(text, engine):
            if _overlaps_span(start, end, protected):
                continue
            if not _is_valid_token(tok, stop_set):
                continue
            if tok in seen_tok:
                continue
            seen_tok.add(tok)
            surface_doc_freq[tok] += 1
            root = uf.find(tok)
            if root not in families_seen:
                families_seen.add(root)
                family_counter[root] += 1

    labels = _family_labels(uf, surface_doc_freq)
    merged: Counter[str] = Counter()
    for root, count in family_counter.items():
        merged[labels.get(root, root)] += count
    for phrase, count in phrase_counter.items():
        merged[phrase] += count

    return Counter({k: v for k, v in merged.items() if v > 0 and not _is_pure_number(k)})


def _save_word_freq(counter: Counter[str], path: Path) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8", newline="") as f:
        w = csv.writer(f)
        w.writerow(["word", "count"])
        for word, count in counter.most_common():
            w.writerow([word, count])

def _step1_extract_terms(
    *,
    job_id: int,
    industry: str,
    product_name: str,
    samples: List[Tuple[int, str]],
    api_key: str,
) -> dict:
    system, user = _build_terms_prompt(industry, product_name, samples)
    logger.info("第 1 步：调用 %s 提取专有名词与停用词（样本 %s 条）", MODEL_NAME, len(samples))
    raw = _call_llm(system, user, api_key)
    product_raw, stop_raw = _parse_llm_terms(raw)
    product_terms, _stop_set = _finalize_term_lists(
        product_raw, stop_raw, product_name
    )
    logger.info(
        "解析得到专有名词 %s 条、专属停用词 %s 条",
        len(product_terms),
        len(stop_raw),
    )
    payload = {
        "job_id": job_id,
        "industry": industry,
        "product_name": product_name,
        "sample_size": len(samples),
        "sample_seed": SAMPLE_SEED,
        "sample_row_ids": [r for r, _ in samples],
        "model": MODEL_NAME,
        "product_terms": product_terms,
        "stopwords_custom": _dedupe_terms(stop_raw),
        "llm_raw": raw,
    }
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    TERMS_JSON.write_text(
        json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    logger.info("已保存 %s", TERMS_JSON)
    return payload


def _load_terms_json() -> dict:
    if not TERMS_JSON.is_file():
        raise FileNotFoundError(f"缺少 {TERMS_JSON}，请先运行完整流程或去掉 --skip-llm")
    return json.loads(TERMS_JSON.read_text(encoding="utf-8"))


def run(*, skip_llm: bool = False) -> dict:
    if not STRUCTURED_DB.is_file():
        raise FileNotFoundError(f"缺少 {STRUCTURED_DB}")

    conn = sqlite3.connect(STRUCTURED_DB)
    try:
        job_id, industry, product_name, source_file = _latest_job(conn)
    finally:
        conn.close()

    csv_path = _resolve_source_path(source_file)
    rows = _load_contents(csv_path)
    logger.info(
        "job_id=%s product=%r 评论 %s 条，来源 %s",
        job_id,
        product_name,
        len(rows),
        csv_path.name,
    )

    if skip_llm:
        meta = _load_terms_json()
        product_terms, stop_set = _finalize_term_lists(
            list(meta.get("product_terms") or []),
            list(meta.get("stopwords_custom") or []),
            product_name,
        )
        logger.info("第 1 步跳过，复用 %s", TERMS_JSON)
    else:
        require_chat_api_key()
        api_key = ""
        samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED)
        meta = _step1_extract_terms(
            job_id=job_id,
            industry=industry,
            product_name=product_name,
            samples=samples,
            api_key=api_key,
        )
        product_terms, stop_set = _finalize_term_lists(
            meta["product_terms"],
            meta.get("stopwords_custom") or [],
            product_name,
        )

    logger.info("第 2 步：NLTK 分词 + stem/lemma 并族词频统计")
    counter = _build_word_freq(rows, product_terms, stop_set)
    _save_word_freq(counter, WORD_FREQ_CSV)
    logger.info("已写入 %s", WORD_FREQ_CSV)

    return {
        "job_id": job_id,
        "product_name": product_name,
        "terms_json": str(TERMS_JSON),
        "word_freq_csv": str(WORD_FREQ_CSV),
        "unique_words": len(counter),
        "total_tokens": sum(counter.values()),
    }


def main() -> None:
    parser = argparse.ArgumentParser(description="VOC 词频统计")
    parser.add_argument(
        "--skip-llm",
        action="store_true",
        help="跳过第 1 步，使用 output/voc_terms.json",
    )
    args = parser.parse_args()
    result = run(skip_llm=args.skip_llm)
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()