cb6692c0c0
包含 Persona 锚定聚类、LLM 报告生成、方法论文档及 .gitignore 更新，便于在 Gitee 独立部署。 Co-authored-by: Cursor <cursoragent@cursor.com>
756 lines
25 KiB
Python
756 lines
25 KiB
Python
"""
|
||
词频统计：从最新结构化任务读取产品与 CSV，两步连跑。
|
||
|
||
1. 随机 25 条 content → LLM 归纳「产品专有名词」与「Amazon/产品专属停用词」
|
||
2. NLTK 全量 content 分词 + stem/lemma 并族 → output/word_freq.csv
|
||
（专有名词按完整短语统计；每条评论每个词族最多计 1 次）
|
||
|
||
用法::
|
||
|
||
./310py/bin/python 词频.py
|
||
./310py/bin/python 词频.py --skip-llm # 复用 output/voc_terms.json，仅跑第 2 步
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import logging
|
||
import random
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from pathlib import Path
|
||
from typing import Dict, Iterable, List, Sequence, Set, Tuple
|
||
|
||
from voc_llm import CHAT_MODEL, chat_extra_body, create_chat_client, require_chat_api_key
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||
stream=sys.stderr,
|
||
)
|
||
logger = logging.getLogger("voc_wordfreq")
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||
OUTPUT_DIR = PROJECT_ROOT / "output"
|
||
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
|
||
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"
|
||
|
||
MODEL_NAME = CHAT_MODEL
|
||
|
||
SAMPLE_SIZE = 38
|
||
SAMPLE_SEED = 42
|
||
|
||
# NLTK 英文停用词扩展（与匹配规则参考一致）
|
||
_EXTRA_STOP_WORDS = frozenset(
|
||
{
|
||
"www", "http", "https", "com", "amazon", "asin", "sku",
|
||
"oz", "lb", "lbs", "inch", "inches", "ft", "mm", "cm", "ml", "kg", "pcs", "pc",
|
||
}
|
||
)
|
||
|
||
# 产品标题拆词中可视为「不重要」、应参与停用的功能词
|
||
_PRODUCT_NAME_FILLER = frozenset(
|
||
{
|
||
"a", "an", "the", "and", "or", "but", "for", "nor", "so", "yet",
|
||
"at", "by", "in", "of", "on", "to", "up", "as", "is", "it", "be",
|
||
"with", "from", "into", "via", "per", "vs", "vs.",
|
||
}
|
||
)
|
||
|
||
|
||
def _ensure_nltk():
|
||
"""加载 NLTK 分词 / 词性 / 词形还原依赖（首次自动下载数据包）。"""
|
||
try:
|
||
import nltk # noqa: F401
|
||
except ImportError as e:
|
||
raise RuntimeError(
|
||
"未安装 nltk，请执行: uv pip install --python 310py/bin/python nltk"
|
||
) from e
|
||
import nltk
|
||
from nltk.corpus import wordnet as wn
|
||
from nltk.stem import PorterStemmer, WordNetLemmatizer
|
||
from nltk.tag import pos_tag
|
||
from nltk.tokenize import word_tokenize
|
||
|
||
for resource, pkg in (
|
||
("tokenizers/punkt", "punkt"),
|
||
("tokenizers/punkt_tab", "punkt_tab"),
|
||
("corpora/wordnet", "wordnet"),
|
||
("corpora/omw-1.4", "omw-1.4"),
|
||
("taggers/averaged_perceptron_tagger", "averaged_perceptron_tagger"),
|
||
("taggers/averaged_perceptron_tagger_eng", "averaged_perceptron_tagger_eng"),
|
||
("corpora/stopwords", "stopwords"),
|
||
):
|
||
try:
|
||
nltk.data.find(resource)
|
||
except LookupError:
|
||
logger.info("下载 NLTK 数据包: %s", pkg)
|
||
nltk.download(pkg, quiet=True)
|
||
|
||
return word_tokenize, pos_tag, WordNetLemmatizer(), PorterStemmer(), wn
|
||
|
||
|
||
def _penn_to_wn_pos(tag: str, wn) -> str:
|
||
if tag.startswith("J"):
|
||
return wn.ADJ
|
||
if tag.startswith("V"):
|
||
return wn.VERB
|
||
if tag.startswith("N"):
|
||
return wn.NOUN
|
||
if tag.startswith("R"):
|
||
return wn.ADV
|
||
return wn.NOUN
|
||
|
||
|
||
def load_nltk_stop_words() -> Set[str]:
|
||
_ensure_nltk()
|
||
import nltk
|
||
from nltk.corpus import stopwords
|
||
|
||
try:
|
||
words = set(stopwords.words("english"))
|
||
except LookupError:
|
||
nltk.download("stopwords", quiet=True)
|
||
words = set(stopwords.words("english"))
|
||
words |= _EXTRA_STOP_WORDS
|
||
return words
|
||
|
||
|
||
class WordVariantEngine:
|
||
"""NLTK 分词 + stem / POS-lemma 词形变体（不含同义词并族）。"""
|
||
|
||
def __init__(self) -> None:
|
||
word_tokenize, pos_tag_fn, lemmatizer, stemmer, wn = _ensure_nltk()
|
||
self._word_tokenize = word_tokenize
|
||
self._pos_tag = pos_tag_fn
|
||
self._lemmatizer = lemmatizer
|
||
self._stemmer = stemmer
|
||
self._wn = wn
|
||
|
||
def variants_for_word(self, word: str, pos: str | None = None) -> Set[str]:
|
||
w = (word or "").lower().strip()
|
||
if not w:
|
||
return set()
|
||
out: Set[str] = {w, self._stemmer.stem(w)}
|
||
if pos is not None:
|
||
wn_pos = _penn_to_wn_pos(pos, self._wn)
|
||
out.add(self._lemmatizer.lemmatize(w, pos=wn_pos))
|
||
for p in (self._wn.NOUN, self._wn.VERB, self._wn.ADJ, self._wn.ADV):
|
||
out.add(self._lemmatizer.lemmatize(w, pos=p))
|
||
return {x for x in out if x}
|
||
|
||
def variant_pool(self, word: str, pos: str | None = None) -> Set[str]:
|
||
return self.variants_for_word(word, pos=pos)
|
||
|
||
def pos_tag_tokens(self, tokens: Sequence[str]) -> List[Tuple[str, str]]:
|
||
if not tokens:
|
||
return []
|
||
try:
|
||
return self._pos_tag(list(tokens))
|
||
except Exception:
|
||
return [(t, "NN") for t in tokens]
|
||
|
||
|
||
class _UnionFind:
|
||
def __init__(self) -> None:
|
||
self._parent: Dict[str, str] = {}
|
||
|
||
def add(self, x: str) -> None:
|
||
if x not in self._parent:
|
||
self._parent[x] = x
|
||
|
||
def find(self, x: str) -> str:
|
||
self.add(x)
|
||
while self._parent[x] != x:
|
||
self._parent[x] = self._parent[self._parent[x]]
|
||
x = self._parent[x]
|
||
return x
|
||
|
||
def union(self, a: str, b: str) -> None:
|
||
ra, rb = self.find(a), self.find(b)
|
||
if ra != rb:
|
||
self._parent[rb] = ra
|
||
|
||
def groups(self) -> Dict[str, List[str]]:
|
||
out: Dict[str, List[str]] = defaultdict(list)
|
||
for x in self._parent:
|
||
out[self.find(x)].append(x)
|
||
return dict(out)
|
||
|
||
|
||
def _strip_think(text: str) -> str:
|
||
if not text:
|
||
return text
|
||
text = re.sub(
|
||
r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
|
||
)
|
||
text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
|
||
return text.strip()
|
||
|
||
|
||
def _latest_job(conn: sqlite3.Connection) -> Tuple[int, str, str, str]:
|
||
row = conn.execute(
|
||
"""
|
||
SELECT id, industry, product_name, source_file
|
||
FROM analysis_jobs
|
||
ORDER BY id DESC
|
||
LIMIT 1
|
||
"""
|
||
).fetchone()
|
||
if not row:
|
||
raise RuntimeError(f"{STRUCTURED_DB} 中无 analysis_jobs 记录，请先跑结构化")
|
||
return int(row[0]), str(row[1]), str(row[2]), str(row[3])
|
||
|
||
|
||
def _resolve_source_path(source_file: str) -> Path:
|
||
p = Path(source_file)
|
||
if p.is_file():
|
||
return p.resolve()
|
||
candidate = PROJECT_ROOT / source_file
|
||
if candidate.is_file():
|
||
return candidate.resolve()
|
||
raise FileNotFoundError(f"找不到评论 CSV: {source_file}")
|
||
|
||
|
||
def _load_contents(csv_path: Path) -> List[Tuple[int, str]]:
|
||
rows: List[Tuple[int, str]] = []
|
||
with csv_path.open(encoding="utf-8-sig", newline="") as f:
|
||
reader = csv.DictReader(f)
|
||
if not reader.fieldnames or "content" not in reader.fieldnames:
|
||
raise ValueError(f"{csv_path} 缺少 content 列")
|
||
for i, row in enumerate(reader, start=1):
|
||
text = (row.get("content") or "").strip()
|
||
if text:
|
||
rows.append((i, text))
|
||
if not rows:
|
||
raise ValueError(f"{csv_path} 无有效 content")
|
||
return rows
|
||
|
||
|
||
def _sample_reviews(
|
||
rows: Sequence[Tuple[int, str]], *, n: int, seed: int
|
||
) -> List[Tuple[int, str]]:
|
||
rng = random.Random(seed)
|
||
k = min(n, len(rows))
|
||
return rng.sample(list(rows), k)
|
||
|
||
|
||
def _build_terms_prompt(
|
||
industry: str, product_name: str, samples: Sequence[Tuple[int, str]]
|
||
) -> Tuple[str, str]:
|
||
blocks = []
|
||
for row_id, text in samples:
|
||
blocks.append(f"[R{row_id}]\n{text}")
|
||
joined = "\n\n---\n\n".join(blocks)
|
||
system = (
|
||
"你是亚马逊美国站英文评论与电商文本分析专家。"
|
||
"根据样本评论，列出两类词表，供后续英文分词与词频统计使用。"
|
||
)
|
||
user = f"""行业：{industry}
|
||
产品：{product_name}
|
||
|
||
以下为该产品 {len(samples)} 条随机评论（仅正文 content）：
|
||
|
||
{joined}
|
||
|
||
请输出两部分（可用 Markdown 标题、编号列表或逗号分隔，不必是 JSON）：
|
||
|
||
## 产品专有名词（product_terms）
|
||
列出该产品评论中可能出现的**全部**专有表达，尽量合理扩展，包括但不限于：
|
||
成分/品类、宠物品种与病种、剂型包装、用法场景、Amazon 物流售后相关但**属于本产品语境**的词等。
|
||
多词短语请保留完整形式（如 turkey tail mushroom powder）；后续词频**只统计完整短语**，不会拆成单词分别计数。
|
||
**不要**把下面「专属停用词」里的词放进本列表。
|
||
|
||
## 专属停用词（stopwords）
|
||
列出 Amazon 美国站电商通用词，以及与本产品强相关、词频高但**分析价值低**的词（如泛化评价词、过泛产品描述词等）。
|
||
多词短语可保留。
|
||
注意：不要把产品名「{product_name}」**整句**列为停用词；也不要把其中的**实词/品类成分**（如 turkey、tail、mushroom）列为停用词。
|
||
产品名拆词里**无分析价值的虚词/功能词**（如 for、the、a、of、and）可以且应列入停用词。
|
||
|
||
每条一行或逗号分隔即可。"""
|
||
return system, user
|
||
|
||
|
||
def _call_llm(system: str, user: str, api_key: str) -> str:
|
||
_ = api_key
|
||
# 术语提取只需短 JSON/列表，200k max_tokens 会导致 API 长时间生成或挂起
|
||
client = create_chat_client(timeout=600.0)
|
||
extra_body = chat_extra_body(MODEL_NAME)
|
||
resp = client.chat.completions.create(
|
||
model=MODEL_NAME,
|
||
messages=[
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
temperature=0.6,
|
||
max_tokens=90000,
|
||
extra_body=extra_body,
|
||
)
|
||
msg = resp.choices[0].message
|
||
text = msg.content or getattr(msg, "reasoning_content", None) or ""
|
||
if not text.strip():
|
||
raise RuntimeError("LLM 返回为空")
|
||
return _strip_think(text)
|
||
|
||
|
||
def _normalize_term(s: str) -> str:
|
||
s = re.sub(r"\s+", " ", (s or "").strip().lower())
|
||
return s.strip(".,;:\"'""''`-–—()[]{}")
|
||
|
||
|
||
def _split_list_items(line: str) -> List[str]:
|
||
line = re.sub(r"^[\s\d\.\)\-•\*]+", "", line.strip())
|
||
if not line or line.startswith("#"):
|
||
return []
|
||
parts = re.split(r"[,，;；、|]", line)
|
||
return [_normalize_term(p) for p in parts if _normalize_term(p)]
|
||
|
||
|
||
def _try_parse_json_terms(text: str) -> Tuple[List[str], List[str]] | None:
|
||
text = text.replace("```json", "").replace("```", "").strip()
|
||
start, end = text.find("{"), text.rfind("}")
|
||
if start == -1 or end <= start:
|
||
return None
|
||
try:
|
||
obj = json.loads(text[start : end + 1])
|
||
except json.JSONDecodeError:
|
||
return None
|
||
if not isinstance(obj, dict):
|
||
return None
|
||
|
||
def pick(keys: Sequence[str]) -> List[str]:
|
||
for k in keys:
|
||
v = obj.get(k)
|
||
if isinstance(v, list):
|
||
return [_normalize_term(str(x)) for x in v if _normalize_term(str(x))]
|
||
if isinstance(v, str):
|
||
return _split_list_items(v)
|
||
return []
|
||
|
||
terms = pick(("product_terms", "专有名词", "terms", "keywords"))
|
||
stops = pick(("stopwords", "stop_words", "停用词", "custom_stopwords"))
|
||
if terms or stops:
|
||
return terms, stops
|
||
return None
|
||
|
||
|
||
def _parse_section_lines(text: str, mode: str) -> List[str]:
|
||
"""mode: product | stop"""
|
||
lines = text.splitlines()
|
||
collecting = False
|
||
items: List[str] = []
|
||
product_hdr = re.compile(
|
||
r"(专有名词|product[_\s-]*terms?|product\s+terms)", re.I
|
||
)
|
||
stop_hdr = re.compile(
|
||
r"(专属停用|停用词|stop[_\s-]*words?|custom\s+stop)", re.I
|
||
)
|
||
other_hdr = re.compile(r"^#{1,3}\s+")
|
||
|
||
for line in lines:
|
||
raw = line.strip()
|
||
if not raw:
|
||
if collecting and items:
|
||
collecting = False
|
||
continue
|
||
if mode == "product" and product_hdr.search(raw):
|
||
collecting = True
|
||
inline = re.split(r"[:：]", raw, maxsplit=1)
|
||
if len(inline) > 1:
|
||
items.extend(_split_list_items(inline[1]))
|
||
continue
|
||
if mode == "stop" and stop_hdr.search(raw):
|
||
collecting = True
|
||
inline = re.split(r"[:：]", raw, maxsplit=1)
|
||
if len(inline) > 1:
|
||
items.extend(_split_list_items(inline[1]))
|
||
continue
|
||
if collecting and (other_hdr.match(raw) or (
|
||
mode == "product" and stop_hdr.search(raw)
|
||
) or (mode == "stop" and product_hdr.search(raw))):
|
||
collecting = False
|
||
continue
|
||
if collecting:
|
||
items.extend(_split_list_items(raw))
|
||
return items
|
||
|
||
|
||
def _parse_llm_terms(text: str) -> Tuple[List[str], List[str]]:
|
||
parsed = _try_parse_json_terms(text)
|
||
if parsed:
|
||
return parsed
|
||
product = _parse_section_lines(text, "product")
|
||
stop = _parse_section_lines(text, "stop")
|
||
if not product and not stop:
|
||
# 兜底：按行提取非空短语
|
||
for line in text.splitlines():
|
||
items = _split_list_items(line)
|
||
if items and len(items) <= 8:
|
||
product.extend(items)
|
||
return _dedupe_terms(product), _dedupe_terms(stop)
|
||
|
||
|
||
def _dedupe_terms(items: Iterable[str]) -> List[str]:
|
||
seen: Set[str] = set()
|
||
out: List[str] = []
|
||
for x in items:
|
||
if x and x not in seen:
|
||
seen.add(x)
|
||
out.append(x)
|
||
return out
|
||
|
||
|
||
def _product_name_tokens(product_name: str) -> Set[str]:
|
||
return {
|
||
t
|
||
for t in re.findall(r"[a-z0-9]+", product_name.lower())
|
||
if t
|
||
}
|
||
|
||
|
||
def _product_name_filler_tokens(product_name: str, nltk_stops: Set[str]) -> Set[str]:
|
||
"""产品名拆词中的虚词/功能词 → 应停用。"""
|
||
tokens = _product_name_tokens(product_name)
|
||
return {t for t in tokens if t in nltk_stops or t in _PRODUCT_NAME_FILLER}
|
||
|
||
|
||
def _product_name_core_tokens(product_name: str) -> Set[str]:
|
||
"""产品名中有分析价值的实词 → 不停用。"""
|
||
fillers = _product_name_filler_tokens(product_name, load_nltk_stop_words())
|
||
return _product_name_tokens(product_name) - fillers
|
||
|
||
|
||
def _is_pure_number(s: str) -> bool:
|
||
return bool(s) and s.isdigit()
|
||
|
||
|
||
def _finalize_term_lists(
|
||
product_terms: List[str],
|
||
stopwords: List[str],
|
||
product_name: str,
|
||
) -> Tuple[List[str], Set[str]]:
|
||
stop_set = load_nltk_stop_words()
|
||
stop_set.update(_dedupe_terms(stopwords))
|
||
stop_set.update(_product_name_filler_tokens(product_name, stop_set))
|
||
for tok in _product_name_core_tokens(product_name):
|
||
stop_set.discard(tok)
|
||
pn_lower = _normalize_term(product_name)
|
||
if pn_lower:
|
||
stop_set.discard(pn_lower)
|
||
|
||
cleaned_terms: List[str] = []
|
||
for t in _dedupe_terms(product_terms):
|
||
if t in stop_set:
|
||
continue
|
||
cleaned_terms.append(t)
|
||
cleaned_terms.sort(key=lambda x: (-len(x.split()), -len(x)))
|
||
return cleaned_terms, stop_set
|
||
|
||
|
||
def _phrase_pattern(phrase: str) -> re.Pattern[str]:
|
||
parts = [re.escape(p) for p in phrase.split()]
|
||
body = r"\s+".join(parts)
|
||
return re.compile(rf"(?<!\w){body}(?!\w)", re.IGNORECASE)
|
||
|
||
|
||
def _merge_spans(spans: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
|
||
if not spans:
|
||
return []
|
||
ordered = sorted(spans)
|
||
merged: List[Tuple[int, int]] = [ordered[0]]
|
||
for start, end in ordered[1:]:
|
||
prev_s, prev_e = merged[-1]
|
||
if start <= prev_e:
|
||
merged[-1] = (prev_s, max(prev_e, end))
|
||
else:
|
||
merged.append((start, end))
|
||
return merged
|
||
|
||
|
||
def _overlaps_span(char_start: int, char_end: int, spans: Sequence[Tuple[int, int]]) -> bool:
|
||
for s, e in spans:
|
||
if char_start < e and char_end > s:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _collect_protected_spans(lower: str, product_terms: List[str], stop_set: Set[str]) -> List[Tuple[int, int]]:
|
||
"""返回专有名词短语匹配区间（屏蔽拆词统计，短语单独计数）。"""
|
||
spans: List[Tuple[int, int]] = []
|
||
for phrase in product_terms:
|
||
if not phrase or phrase in stop_set or _is_pure_number(phrase):
|
||
continue
|
||
for m in _phrase_pattern(phrase).finditer(lower):
|
||
spans.append((m.start(), m.end()))
|
||
return _merge_spans(spans)
|
||
|
||
|
||
def _phrases_hit_in_review(lower: str, product_terms: List[str], stop_set: Set[str]) -> List[str]:
|
||
"""本条评论命中的专有名词（每条评论每短语最多计 1 次）。"""
|
||
hit: List[str] = []
|
||
for phrase in product_terms:
|
||
if not phrase or phrase in stop_set or _is_pure_number(phrase):
|
||
continue
|
||
if _phrase_pattern(phrase).search(lower):
|
||
hit.append(phrase)
|
||
return hit
|
||
|
||
|
||
_TOKEN_RE = re.compile(r"[a-z0-9']+")
|
||
|
||
|
||
def _tokenize_tagged_with_spans(text: str, engine: WordVariantEngine) -> List[Tuple[str, str, int, int]]:
|
||
"""带字符区间的分词 + 词性标注。"""
|
||
lower = text.lower()
|
||
raw: List[Tuple[str, int, int]] = []
|
||
for m in _TOKEN_RE.finditer(lower):
|
||
tok = m.group()
|
||
if tok:
|
||
raw.append((tok, m.start(), m.end()))
|
||
if not raw:
|
||
return []
|
||
words = [t for t, _, _ in raw]
|
||
tagged = engine.pos_tag_tokens(words)
|
||
return [(tagged[i][0], tagged[i][1], raw[i][1], raw[i][2]) for i in range(len(raw))]
|
||
|
||
|
||
def _is_valid_token(tok: str, stop_set: Set[str]) -> bool:
|
||
if not tok or len(tok) < 1 or tok in stop_set or _is_pure_number(tok):
|
||
return False
|
||
return bool(re.search(r"[a-z]", tok))
|
||
|
||
|
||
def _register_review_tokens_in_uf(
|
||
text: str,
|
||
protected: Sequence[Tuple[int, int]],
|
||
engine: WordVariantEngine,
|
||
uf: _UnionFind,
|
||
variant_index: Dict[str, Set[str]],
|
||
stop_set: Set[str],
|
||
) -> None:
|
||
"""Pass 1：将本条评论 token 注册进 Union-Find（stem/lemma 并族）。"""
|
||
seen: Set[str] = set()
|
||
for tok, tag, start, end in _tokenize_tagged_with_spans(text, engine):
|
||
if _overlaps_span(start, end, protected):
|
||
continue
|
||
if not _is_valid_token(tok, stop_set):
|
||
continue
|
||
if tok in seen:
|
||
continue
|
||
seen.add(tok)
|
||
uf.add(tok)
|
||
variants = engine.variant_pool(tok, pos=tag)
|
||
related: Set[str] = set()
|
||
for v in variants:
|
||
related |= variant_index[v]
|
||
for other in related:
|
||
uf.union(tok, other)
|
||
for v in variants:
|
||
variant_index[v].add(tok)
|
||
|
||
|
||
def _family_labels(uf: _UnionFind, surface_doc_freq: Counter[str]) -> Dict[str, str]:
|
||
"""词族代表形：族内 surface 文档频次最高者，并列取最短。"""
|
||
labels: Dict[str, str] = {}
|
||
for root, members in uf.groups().items():
|
||
best = sorted(
|
||
members,
|
||
key=lambda m: (-surface_doc_freq.get(m, 0), len(m), m),
|
||
)[0]
|
||
labels[root] = best
|
||
return labels
|
||
|
||
|
||
def _build_word_freq(
|
||
rows: Sequence[Tuple[int, str]],
|
||
product_terms: List[str],
|
||
stop_set: Set[str],
|
||
) -> Counter[str]:
|
||
"""
|
||
NLTK 分词 + stem/lemma Union-Find 并族。
|
||
每条评论：每个词族最多 +1；专有名词短语命中也最多 +1/短语。
|
||
输出 word 为词族代表形（方案 1）。
|
||
"""
|
||
engine = WordVariantEngine()
|
||
uf = _UnionFind()
|
||
variant_index: Dict[str, Set[str]] = defaultdict(set)
|
||
|
||
# Pass 1：全库注册词形变体并族
|
||
for _, text in rows:
|
||
if not text.strip():
|
||
continue
|
||
protected = _collect_protected_spans(text.lower(), product_terms, stop_set)
|
||
_register_review_tokens_in_uf(text, protected, engine, uf, variant_index, stop_set)
|
||
|
||
family_counter: Counter[str] = Counter()
|
||
phrase_counter: Counter[str] = Counter()
|
||
surface_doc_freq: Counter[str] = Counter()
|
||
|
||
# Pass 2：按评论计数（每词族 / 每短语最多 1 次）
|
||
for _, text in rows:
|
||
if not text.strip():
|
||
continue
|
||
lower = text.lower()
|
||
protected = _collect_protected_spans(lower, product_terms, stop_set)
|
||
|
||
for phrase in _phrases_hit_in_review(lower, product_terms, stop_set):
|
||
phrase_counter[phrase] += 1
|
||
|
||
families_seen: Set[str] = set()
|
||
seen_tok: Set[str] = set()
|
||
for tok, tag, start, end in _tokenize_tagged_with_spans(text, engine):
|
||
if _overlaps_span(start, end, protected):
|
||
continue
|
||
if not _is_valid_token(tok, stop_set):
|
||
continue
|
||
if tok in seen_tok:
|
||
continue
|
||
seen_tok.add(tok)
|
||
surface_doc_freq[tok] += 1
|
||
root = uf.find(tok)
|
||
if root not in families_seen:
|
||
families_seen.add(root)
|
||
family_counter[root] += 1
|
||
|
||
labels = _family_labels(uf, surface_doc_freq)
|
||
merged: Counter[str] = Counter()
|
||
for root, count in family_counter.items():
|
||
merged[labels.get(root, root)] += count
|
||
for phrase, count in phrase_counter.items():
|
||
merged[phrase] += count
|
||
|
||
return Counter({k: v for k, v in merged.items() if v > 0 and not _is_pure_number(k)})
|
||
|
||
|
||
def _save_word_freq(counter: Counter[str], path: Path) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8", newline="") as f:
|
||
w = csv.writer(f)
|
||
w.writerow(["word", "count"])
|
||
for word, count in counter.most_common():
|
||
w.writerow([word, count])
|
||
|
||
def _step1_extract_terms(
|
||
*,
|
||
job_id: int,
|
||
industry: str,
|
||
product_name: str,
|
||
samples: List[Tuple[int, str]],
|
||
api_key: str,
|
||
) -> dict:
|
||
system, user = _build_terms_prompt(industry, product_name, samples)
|
||
logger.info("第 1 步：调用 %s 提取专有名词与停用词（样本 %s 条）", MODEL_NAME, len(samples))
|
||
raw = _call_llm(system, user, api_key)
|
||
product_raw, stop_raw = _parse_llm_terms(raw)
|
||
product_terms, _stop_set = _finalize_term_lists(
|
||
product_raw, stop_raw, product_name
|
||
)
|
||
logger.info(
|
||
"解析得到专有名词 %s 条、专属停用词 %s 条",
|
||
len(product_terms),
|
||
len(stop_raw),
|
||
)
|
||
payload = {
|
||
"job_id": job_id,
|
||
"industry": industry,
|
||
"product_name": product_name,
|
||
"sample_size": len(samples),
|
||
"sample_seed": SAMPLE_SEED,
|
||
"sample_row_ids": [r for r, _ in samples],
|
||
"model": MODEL_NAME,
|
||
"product_terms": product_terms,
|
||
"stopwords_custom": _dedupe_terms(stop_raw),
|
||
"llm_raw": raw,
|
||
}
|
||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||
TERMS_JSON.write_text(
|
||
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||
)
|
||
logger.info("已保存 %s", TERMS_JSON)
|
||
return payload
|
||
|
||
|
||
def _load_terms_json() -> dict:
|
||
if not TERMS_JSON.is_file():
|
||
raise FileNotFoundError(f"缺少 {TERMS_JSON}，请先运行完整流程或去掉 --skip-llm")
|
||
return json.loads(TERMS_JSON.read_text(encoding="utf-8"))
|
||
|
||
|
||
def run(*, skip_llm: bool = False) -> dict:
|
||
if not STRUCTURED_DB.is_file():
|
||
raise FileNotFoundError(f"缺少 {STRUCTURED_DB}")
|
||
|
||
conn = sqlite3.connect(STRUCTURED_DB)
|
||
try:
|
||
job_id, industry, product_name, source_file = _latest_job(conn)
|
||
finally:
|
||
conn.close()
|
||
|
||
csv_path = _resolve_source_path(source_file)
|
||
rows = _load_contents(csv_path)
|
||
logger.info(
|
||
"job_id=%s product=%r 评论 %s 条，来源 %s",
|
||
job_id,
|
||
product_name,
|
||
len(rows),
|
||
csv_path.name,
|
||
)
|
||
|
||
if skip_llm:
|
||
meta = _load_terms_json()
|
||
product_terms, stop_set = _finalize_term_lists(
|
||
list(meta.get("product_terms") or []),
|
||
list(meta.get("stopwords_custom") or []),
|
||
product_name,
|
||
)
|
||
logger.info("第 1 步跳过，复用 %s", TERMS_JSON)
|
||
else:
|
||
require_chat_api_key()
|
||
api_key = ""
|
||
samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED)
|
||
meta = _step1_extract_terms(
|
||
job_id=job_id,
|
||
industry=industry,
|
||
product_name=product_name,
|
||
samples=samples,
|
||
api_key=api_key,
|
||
)
|
||
product_terms, stop_set = _finalize_term_lists(
|
||
meta["product_terms"],
|
||
meta.get("stopwords_custom") or [],
|
||
product_name,
|
||
)
|
||
|
||
logger.info("第 2 步：NLTK 分词 + stem/lemma 并族词频统计")
|
||
counter = _build_word_freq(rows, product_terms, stop_set)
|
||
_save_word_freq(counter, WORD_FREQ_CSV)
|
||
logger.info("已写入 %s", WORD_FREQ_CSV)
|
||
|
||
return {
|
||
"job_id": job_id,
|
||
"product_name": product_name,
|
||
"terms_json": str(TERMS_JSON),
|
||
"word_freq_csv": str(WORD_FREQ_CSV),
|
||
"unique_words": len(counter),
|
||
"total_tokens": sum(counter.values()),
|
||
}
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="VOC 词频统计")
|
||
parser.add_argument(
|
||
"--skip-llm",
|
||
action="store_true",
|
||
help="跳过第 1 步，使用 output/voc_terms.json",
|
||
)
|
||
args = parser.parse_args()
|
||
result = run(skip_llm=args.skip_llm)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|