From 2441c9119cc5e6a917af68df4f2dafdd32008856 Mon Sep 17 00:00:00 2001 From: OnesvmWhoops Date: Wed, 3 Jun 2026 15:40:13 +0800 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20VOC=20LLM=20=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E5=8C=96=E5=88=86=E6=9E=90=E6=B5=81=E6=B0=B4=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 包含七步编排入口、结构化/向量化/聚类/词频/报告模块与 prompts 配置；忽略原始 CSV 与本地密钥。 Co-authored-by: Cursor --- .gitignore | 33 + content清洗.py | 153 ++ main_voc分析.md | 297 ++ main_voc分析.py | 393 +++ main_voc分析_jieba.py | 342 +++ prompts/README.md | 33 + prompts/__init__.py | 45 + prompts/config.yaml | 26 + prompts/extraction/batch_output_format.md | 7 + prompts/extraction/examples.yaml | 69 + prompts/extraction/examples_footer_batch.md | 1 + prompts/extraction/examples_section.md | 3 + prompts/extraction/field_rules.md | 18 + prompts/extraction/filter_irrelevant.md | 3 + .../extraction/format_constraints_batch.md | 3 + .../extraction/format_constraints_single.md | 4 + prompts/extraction/system_intro.md | 4 + prompts/extraction/user_batch.md | 3 + prompts/extraction/user_single_tail.md | 3 + prompts/loader.py | 320 +++ prompts/report/analysis_requirements.md | 8 + prompts/report/correction.md | 15 + prompts/report/json_markers.md | 24 + prompts/report/output_format.md | 8 + prompts/report/system.md | 4 + prompts/schema.yaml | 31 + prompts/smoke.py | 139 + prompts/smoke/reviews.yaml | 12 + prompts/word_freq/analysis_system.md | 1 + prompts/word_freq/analysis_user.md | 9 + prompts/word_freq/assign_system.md | 4 + prompts/word_freq/assign_user.md | 12 + prompts/word_freq/category_definitions.md | 10 + pyproject.toml | 10 + requirements.txt | 8 + voc_report.py | 2415 +++++++++++++++++ 合并评论数据.py | 103 + 向量化.py | 534 ++++ 结构化_Prompt.py | 59 + 结构化_server.py | 1082 ++++++++ 聚类.py | 1226 +++++++++ 词频.py | 608 +++++ 词频_jieba.py | 182 ++ 43 files changed, 8264 insertions(+) create mode 100644 .gitignore create mode 100644 content清洗.py create mode 100644 main_voc分析.md create mode 100644 main_voc分析.py create mode 100644 main_voc分析_jieba.py create mode 100644 prompts/README.md create mode 100644 prompts/__init__.py create mode 100644 prompts/config.yaml create mode 100644 prompts/extraction/batch_output_format.md create mode 100644 prompts/extraction/examples.yaml create mode 100644 prompts/extraction/examples_footer_batch.md create mode 100644 prompts/extraction/examples_section.md create mode 100644 prompts/extraction/field_rules.md create mode 100644 prompts/extraction/filter_irrelevant.md create mode 100644 prompts/extraction/format_constraints_batch.md create mode 100644 prompts/extraction/format_constraints_single.md create mode 100644 prompts/extraction/system_intro.md create mode 100644 prompts/extraction/user_batch.md create mode 100644 prompts/extraction/user_single_tail.md create mode 100644 prompts/loader.py create mode 100644 prompts/report/analysis_requirements.md create mode 100644 prompts/report/correction.md create mode 100644 prompts/report/json_markers.md create mode 100644 prompts/report/output_format.md create mode 100644 prompts/report/system.md create mode 100644 prompts/schema.yaml create mode 100644 prompts/smoke.py create mode 100644 prompts/smoke/reviews.yaml create mode 100644 prompts/word_freq/analysis_system.md create mode 100644 prompts/word_freq/analysis_user.md create mode 100644 prompts/word_freq/assign_system.md create mode 100644 prompts/word_freq/assign_user.md create mode 100644 prompts/word_freq/category_definitions.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 voc_report.py create mode 100644 合并评论数据.py create mode 100644 向量化.py create mode 100644 结构化_Prompt.py create mode 100644 结构化_server.py create mode 100644 聚类.py create mode 100644 词频.py create mode 100644 词频_jieba.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9dbbd16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# macOS +.DS_Store + +# Python +310py/ +.venv/ +venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.mypy_cache/ + +# 密钥（勿提交） +.dashscope_key +.env +.env.* + +# 本地数据库 +voc_structured.sqlite +voc_embeddings.sqlite +voc_clustering.sqlite + +# 流水线输出（含 word_freq.csv、报告 HTML 等） +output/ + +# 原始 / 中间 CSV（不纳入版本库） +*.csv + +# 常见 --input-dir 原始数据目录（目录名因产品而异，按需追加） +reviews_export/ +*-voc/ +cat deterrent indoor / diff --git a/content清洗.py b/content清洗.py new file mode 100644 index 0000000..4c5f79e --- /dev/null +++ b/content清洗.py @@ -0,0 +1,153 @@ +""" +清洗合并后的评论 CSV（merged_reviews.csv 等同结构文件）。 +保留元数据列，对 content 列做文本清洗与过滤。 +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +import pandas as pd + +_PROJECT_ROOT = Path(__file__).resolve().parent +DEFAULT_INPUT_PATH = _PROJECT_ROOT / "merged_reviews.csv" +DEFAULT_OUTPUT_PATH = _PROJECT_ROOT / "merged_reviews_cleaned.csv" + +CONTENT_COLUMN = "content" +MIN_WORD_COUNT = 4 + +# 与 content 重复的正文列，输出时一律删除，避免一行存两遍长文本 +REDUNDANT_CONTENT_COLUMNS = ( + "content_raw", + "content_cleaned", + "Cleaned_Content", + "cleaned_content", +) + +# 亚马逊「视频/图片无法加载」占位文案，后常跟大量空行 +_MEDIA_LOAD_NOISE = re.compile( + r"The\s+media\s+could\s+not\s+be\s+loaded\.?\s*", + flags=re.IGNORECASE, +) + + +def strip_empty_lines(text) -> str: + """去掉仅含空白/空白的行，并将剩余行合并为单行文本。""" + if pd.isna(text): + return "" + text = str(text).replace("\r\n", "\n").replace("\r", "\n") + lines = [line.strip() for line in text.split("\n")] + lines = [line for line in lines if line] + return " ".join(lines) + + +def clean_amazon_review(text) -> str: + if pd.isna(text): + return "" + text = str(text).strip() + + if text.startswith('"') and text.endswith('"'): + text = text[1:-1] + + text = strip_empty_lines(text) + text = _MEDIA_LOAD_NOISE.sub("", text) + + text = re.sub(r"", " ", text, flags=re.IGNORECASE) + text = text.replace(" ", " ").replace("&", "&") + + noise_prefixes = [ + r"Why did you pick this product vs others\?:", + r"Quality:", + r"Update:", + ] + for prefix in noise_prefixes: + text = re.sub(prefix, "", text, flags=re.IGNORECASE).strip() + + text = re.sub(r"\s+", " ", text).strip() + return text + + +def drop_redundant_content_columns(df: pd.DataFrame) -> pd.DataFrame: + """删除与 content 重复存储的正文列（如 content_raw），避免 CSV 一行两份长文本。""" + drop_cols = [ + c for c in REDUNDANT_CONTENT_COLUMNS if c in df.columns and c != CONTENT_COLUMN + ] + if drop_cols: + df = df.drop(columns=drop_cols) + return df + + +def process_reviews( + csv_file_path: str | Path, + *, + content_column: str = CONTENT_COLUMN, +) -> pd.DataFrame: + """ + 读取合并后的评论 CSV，清洗 content 列并过滤无效/重复行。 + 输出保留原表头及元数据（_id, asin, rating, title 等）。 + """ + csv_file_path = Path(csv_file_path) + df = pd.read_csv(csv_file_path, dtype=str, keep_default_na=False) + + if content_column not in df.columns: + raise ValueError( + f"缺少评论正文列 {content_column!r}，当前列: {list(df.columns)}" + ) + + print(f"1. 原始数据量: {len(df)}") + + df = drop_redundant_content_columns(df) + df[content_column] = df[content_column].apply(clean_amazon_review) + + df = df[df[content_column] != ""] + + df = df[ + df[content_column].str.contains( + r"[a-zA-Z0-9áéíóúñÁÉÍÓÚÑ]", regex=True, na=False + ) + ] + + word_counts = df[content_column].apply(lambda x: len(str(x).split())) + df = df[word_counts >= MIN_WORD_COUNT] + + df = df.drop_duplicates(subset=[content_column], keep="first") + df = drop_redundant_content_columns(df) + + print(f"2. 清洗及过滤后数据量: {len(df)}") + return df + + +def save_cleaned_reviews(df: pd.DataFrame, output_path: str | Path) -> Path: + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path, index=False, encoding="utf-8-sig") + print(f"3. 已写入: {output_path}") + return output_path + + +def main() -> None: + parser = argparse.ArgumentParser(description="清洗合并后的评论 CSV") + parser.add_argument( + "-i", + "--input", + type=Path, + default=DEFAULT_INPUT_PATH, + help=f"输入 CSV（默认: {DEFAULT_INPUT_PATH}）", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + default=DEFAULT_OUTPUT_PATH, + help=f"输出 CSV（默认: {DEFAULT_OUTPUT_PATH}）", + ) + args = parser.parse_args() + + df = process_reviews(args.input) + save_cleaned_reviews(df, args.output) + + +if __name__ == "__main__": + main() diff --git a/main_voc分析.md b/main_voc分析.md new file mode 100644 index 0000000..014e121 --- /dev/null +++ b/main_voc分析.md @@ -0,0 +1,297 @@ +# main_voc分析.py 说明文档 + +VOC（Voice of Customer）分析**全流程编排入口**：合并原始 CSV → 清洗 → LLM 结构化 → 向量化 →（聚类 + 词频）→ 生成 HTML 报告。 + +> 本文档与 `main_voc分析.py` 同目录放置，风格对齐 `prompts/README.md`（表格 + 可执行示例 + 路径约定）。 + +--- + +## 1. 流程总览 + +```mermaid +flowchart LR + S1[步骤1 合并CSV] --> S2[步骤2 清洗] + S2 --> S3[步骤3 结构化 LLM] + S3 --> S4[步骤4 向量化 API] + S4 --> S5[步骤5 聚类 UMAP+HDBSCAN] + S4 --> S6[步骤6 词频 spaCy+LLM] + S5 --> S7[步骤7 报告 HTML] + S6 --> S7 +``` + +| 步骤 | 模块 | 是否调用 DashScope API | +|------|------|------------------------| +| 1 | `合并评论数据.py` | 否 | +| 2 | `content清洗.py` | 否 | +| 3 | `结构化_server.py` | **是**（Chat 结构化） | +| 4 | `向量化.py` | **是**（Embedding） | +| 5–6 | `聚类.py` ∥ `词频.py` | **是**（聚类调参评估 + 词频术语提取） | +| 7 | `voc_report.py` | **是**（报告撰写、词频分类、翻译等） | + +步骤 5 与 6 在 `from_step ≤ 5` 且两步均需执行时，由 `ThreadPoolExecutor(max_workers=2)` **并行**运行。 + +--- + +## 2. 命令行输入 / 输出 + +### 2.1 输入（CLI 参数） + +| 参数 | 必填条件 | 默认值 | 说明 | +|------|----------|--------|------| +| `--input-dir` | 是 | — | 原始 VOC评论数据文件 CSV 目录（目录内的所有`*.csv` 表头须一致） | +| `--industry` | 是 | `Pet Supplies` | 行业名，写入结构化任务 | +| `--product` | 是 | — | 产品名 | +| `--merged-csv` | 否 | `merged_reviews.csv` | 步骤 1 输出路径 | +| `--cleaned-csv` | 否 | `merged_reviews_cleaned.csv` | 步骤 2 输出路径 | +| `--from-step N` | 否 | `1` | 从第 N 步执行到结束（N=1…7） | +| `--only-step N` | 否 | — | 仅执行第 N 步（需已有前置产物） | +| `--keep-db` | 否 | 默认**会**清理库 | 保留已有三个 `.sqlite`，不覆盖删除 | +| `--clean-intermediates` | 否 | 关闭 | 步骤 7 成功后删除中间 csv/sqlite，仅保留报告目录 | +| `--skip-wordfreq-llm` | 否 | 关闭 | 词频复用已有 `output/voc_terms.json`，跳过 LLM 术语提取 | +| `--filter-small-clusters` | 否 | 关闭 | 报告仅纳入簇内去重评论占比 ≥ 10% 的簇 | +| `--save-llm-raw` | 否 | 关闭 | 将报告 LLM 完整原文写入 `{product}_voc_report.html` 同目录 `report_llm_raw.txt` | + +### 2.2 输出 + +**标准输出（stdout）**：流程结束后打印 JSON 字典（`ensure_ascii=False`），键随实际执行步骤变化，常见字段： + +| 键 | 含义 | +|----|------| +| `merged_csv` | 合并后 CSV 路径 | +| `cleaned_csv` | 清洗后 CSV 路径 | +| `structured_job_id` | 结构化任务 ID | +| `structured_db` | `voc_structured.sqlite` | +| `embed` | 向量化模块返回信息（含 `job_id` 等） | +| `clustering` | 聚类模块返回信息 | +| `wordfreq` | 词频模块返回信息 | +| `report_html` | 报告 HTML 绝对/相对路径 | +| `report` | `generate_report` 返回值 | + +**磁盘产物（项目根为 `PROJECT_ROOT`，即本文件所在目录）**： + +| 路径 | 产生步骤 | 说明 | +|------|----------|------| +| `merged_reviews.csv` | 1 | 多文件纵向合并 | +| `merged_reviews_cleaned.csv` | 2 | 清洗、去重后的评论 | +| `voc_structured.sqlite` | 3 | 结构化任务与逐条提取结果 | +| `voc_embeddings.sqlite` | 4 | 各实体短语 256 维向量 | +| `voc_clustering.sqlite` | 5 | 多阶段 UMAP+HDBSCAN 簇标签 | +| `output/voc_terms.json` | 6 | LLM 归纳的产品专有名词 / 停用词 | +| `output/word_freq.csv` | 6 | 全量词频表（`word`, `count`） | +| `output/{product}/{product}_voc_report.html` | 7 | 最终 VOC 分析报告（产品名经路径安全化） | + +`{product}` 目录名由 `_safe_product_dir_name` 生成：去除 `<>:"/\|?*` 等非法字符。 + +### 2.3 常用命令 + +```bash +cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化" + +# 全流程（默认写入 sqlite 前会清理旧库；加 --keep-db 则保留） +python3 main_voc分析.py --input-dir "某目录" --product "产品名" + +# 从步骤 4 续跑（product 可省略） +python3 main_voc分析.py --from-step 4 --keep-db + +# 仅重跑词频 LLM 之前的 spaCy 统计 +python3 main_voc分析.py --from-step 6 --skip-wordfreq-llm + +# 仅生成报告 +python3 main_voc分析.py --only-step 7 +``` + +### 2.4 程序式调用 + +```python +from main_voc分析 import run_voc_analysis # 别名 run_pipeline + +result = run_voc_analysis( + input_dir=Path("reviews_export"), + industry="Pet Supplies", + product_name="cat deterrent indoor", + merged_csv=Path("merged_reviews.csv"), + cleaned_csv=Path("merged_reviews_cleaned.csv"), + from_step=1, + clean_databases=True, # 对应 CLI 未加 --keep-db +) +``` + +--- + +## 3. 七步数据流与数学 / 算法 + +### 步骤 1：合并 CSV（`合并评论数据.py`） + +- **输入**：目录内表头一致的 `*.csv`。 +- **方法**：`pandas.concat` 纵向合并；表头不一致则报错。 +- **输出**：单表 `merged_reviews.csv`（`utf-8-sig`）。 + +### 步骤 2：清洗（`content清洗.py`） + +- **输入**：合并 CSV，正文列 `content`（或兼容列名由下游结构化处理）。 +- **方法**（规则型，无机器学习）： + - 去空行、亚马逊媒体占位文案、HTML 实体、常见噪声前缀； + - 保留含字母/数字的行的词数 ≥ 4； + - 按 `content` 去重（`keep="first"`）。 +- **输出**：`merged_reviews_cleaned.csv`。 + +### 步骤 3：结构化（`结构化_server.py` + `结构化_Prompt.py` + `prompts/`） + +- **输入**：清洗 CSV、`industry`、`product_name`。 +- **方法**：DashScope **Chat Completions**（默认 `qwen3.6-flash`），按 token 估算批量调用，从评论中提取 JSON 字段（audience、pain_points、aspect、opinion、category、sentiment 等，以 `prompts/schema.yaml` 为准）。 +- **输出**：`voc_structured.sqlite`（`analysis_jobs`、`comment_extractions`）。 + +### 步骤 4：向量化（`向量化.py`） + +- **输入**：最新或指定 `job_id` 的结构化实体；`embed_text` 由 audience / pain_point / aspect / opinion / aspect_opinion 等展开。 +- **方法**： + - API：`text-embedding-v4`，**256 维**，余弦相似度空间中的稠密向量； + - 存储：`float32` 打包为 BLOB（`struct.pack`）； + - 批大小 ≤ 10/请求，默认 6 线程并行多批。 +- **输出**：`voc_embeddings.sqlite`（`embedding_items`）。 + +### 步骤 5：聚类（`聚类.py`） + +- **输入**：`voc_embeddings.sqlite` 中向量与元数据。 +- **核心数学管线**（对每个聚类 stage 的子集）： + + 1. **UMAP 降维**（`umap-learn`） + - `n_components = min(30, n-2)` + - `metric = cosine` + - `min_dist = 0.1` + - `random_state = 42` + - `n_neighbors` 由自动调参循环递增（初值 10，上限 45） + + 2. **HDBSCAN**（`hdbscan`） + - `min_samples = 1`，`cluster_selection_method = eom` + - 初始 `min_cluster_size = max(2, n // 20)`，若簇数 > 20 则增大 `min_cluster_size` 直至 ≤ 20 或无法再增 + - 标签 `-1` 表示离群点（各 stage 是否参与后续见模块内注释） + + 3. **轮廓系数**（`sklearn.metrics.silhouette_score`） + - 在 UMAP 空间、非离群点上计算；用于早停：连续 8 轮中后 7 轮均低于窗口首值则回退轮廓最高的一轮。 + + 4. **LLM 聚类质量评估**（非传统指标，辅助调参） + - 对各簇抽样短语，统计「跨簇语义相似」比例；> 10% 则继续增大 `n_neighbors`；≤ 10% 且轮廓 > 0.6 则停止。 + +- **特例**：子集样本数 `n < 10` 时不跑 HDBSCAN，每条独立簇 `0..n-1`。 +- **输出**：`voc_clustering.sqlite`（多 stage：如 audience、簇内 pain_point、按情感分桶的 aspect_opinion 等）。 + +### 步骤 6：词频（`词频.py`） + +- **第 1 步（LLM）**：固定种子 `SAMPLE_SEED=42` 随机 **25** 条 `content`，归纳产品专有名词与专属停用词 → `output/voc_terms.json`。 +- **第 2 步（spaCy）**： + - 英文分词与停用词过滤（含 EN stop words + 自定义停用词）； + - 专有名词按完整短语计数（protected spans）； + - 词形归并：`_merge_word_forms` 将复数/时态等 lemmatize 后合并计数； + - 纯数字词剔除。 +- **输出**：`output/word_freq.csv`。 + +### 步骤 7：报告（`voc_report.py` + `prompts/report/`） + +- **输入**：`voc_clustering.sqlite`、`voc_embeddings.sqlite`、`merged_reviews_cleaned.csv`、`word_freq.csv`、可选 `voc_structured.sqlite`。 +- **方法**： + - 从库中聚合各 stage 簇与代表短语； + - 多次 LLM 调用生成分析正文、词频六类归类、短语翻译等； + - 词云字号：`count^0.5` 映射到 `[14, 96]` px； + - 可选 `min_cluster_review_ratio`（默认 0.10）过滤小簇。 +- **输出**：`output/{product}/{product}_voc_report.html`（内嵌词云、词频表、AI 报告、分簇表述）。 + +--- + +## 4. API Key 使用说明 + +`main_voc分析.py` **本身不读取、不持有 API Key**；密钥解析在各子模块内统一实现，优先级一致： + +1. 环境变量 `DASHSCOPE_API_KEY` +2. 环境变量 `DASHSCOPE_API_KEY_FILE` 指向的单行密钥文件 +3. 项目根文件 `.dashscope_key`（单行，无引号） + +可选环境变量：`DASHSCOPE_MODEL`（默认各模块为 `qwen3.6-flash`）。 + +| 模块 | 使用 API Key 的位置 | API 类型 / 用途 | +|------|---------------------|-----------------| +| `结构化_server.py` | `_resolve_dashscope_api_key()` → `OpenAI(...)` | Chat：批量/单条评论结构化 | +| `向量化.py` | `_resolve_api_key()` → `_embed_one_api_batch` | Embeddings：`text-embedding-v4` | +| `聚类.py` | `_resolve_api_key()` → `run_clustering` 内 `OpenAI` | Chat：簇间相似度评估、调 `n_neighbors` | +| `词频.py` | `_resolve_api_key()` → `_step1_extract_terms` / `_call_llm` | Chat：专有名词与停用词提取 | +| `voc_report.py` | `_resolve_api_key()` → `generate_report` 及子函数 | Chat：报告生成、词频分类、翻译等 | +| `prompts/smoke.py` | `--live` 时 | 冒烟测试（非 main 流程） | + +**仓库安全规范**（见 `.gitignore`）： + +- **禁止提交** `.dashscope_key`、`voc_structured.sqlite` 及含真实评论/密钥的敏感导出； +- 密钥仅通过环境变量或本机未跟踪文件提供； +- 文档与代码中勿写入真实 `sk-` 密钥。 + +--- + +## 5. 各模块职责 + +| 文件 | 职责 | +|------|------| +| `main_voc分析.py` | 七步编排、断点续跑、库清理策略、步骤 5/6 并行、CLI | +| `合并评论数据.py` | 目录多 CSV 合并为一张表 | +| `content清洗.py` | 评论正文清洗、过滤、去重 | +| `结构化_server.py` | LLM 结构化入库、job 管理、库清理钩子 | +| `结构化_Prompt.py` | 组装结构化 prompt（被 server 动态加载） | +| `向量化.py` | 结构化实体 → 256 维向量库 | +| `聚类.py` | UMAP + HDBSCAN 多阶段聚类 + LLM 调参 | +| `词频.py` | LLM 术语 + spaCy 全量词频 | +| `voc_report.py` | 聚类/词频/原文 → 单页 HTML 报告 | +| `prompts/` | 可编辑 prompt 与 `schema.yaml`（见 `prompts/README.md`） | +| `main_voc分析_jieba.py` | 可选变体入口：词频走 `词频_jieba.py`（jieba 分词），其余步骤与主流程类似 | + +### main 内主要函数 + +| 函数 | 作用 | +|------|------| +| `run_voc_analysis` / `run_pipeline` | 核心流水线 | +| `_should_run` | 根据 `--from-step` / `--only-step` 判断是否执行某步 | +| `_latest_job_meta` | 从结构化库读最新 `job_id`、`industry`、`product_name` | +| `_resolve_industry_product` | CLI 下 product 自动补全 | +| `_clean_intermediates` | `--clean-intermediates` 时删除中间文件 | +| `_report_html_path` | 计算报告 HTML 路径 | + +--- + +## 6. SQLite 与路径约定 + +| 库文件 | 写入步骤 | 主要内容 | +|--------|----------|----------| +| `voc_structured.sqlite` | 3 | `analysis_jobs`、`comment_extractions` | +| `voc_embeddings.sqlite` | 4 | `embedding_items`（向量 BLOB + 实体元数据） | +| `voc_clustering.sqlite` | 5 | 各 `stage` 簇标签、调参日志、过滤元数据 | + +**默认清理策略**（未加 `--keep-db`）： + +- 步骤 3 运行 `run_analysis(clean_databases=True)` 时清理三个 sqlite（结构化写入逻辑见 `结构化_server.py`）； +- 步骤 4 若未重跑步骤 3，main 会单独删除 `voc_embeddings.sqlite` 与 `voc_clustering.sqlite` 再向量化； +- 步骤 5/6 的 `reset_db` 与 `clean_databases` 联动。 + +**行号溯源**：结构化与向量化的 `source_row` 为 CSV **首条数据行为 1**；对应 `merged_reviews_cleaned.csv` 物理行号 = `source_row + 1`（第 1 行为表头）。 + +--- + +## 7. 依赖与环境 + +```bash +cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化" +python3 -m venv 310py && source 310py/bin/activate # 可选，与 .gitignore 一致 +pip install -r requirements.txt +python -m spacy download en_core_web_sm # 词频步骤需要 +export DASHSCOPE_API_KEY="sk-xxx" # 或配置 .dashscope_key（勿提交） +``` + +`requirements.txt` 中与数学/ NLP 相关的主要包：`numpy`、`umap-learn`、`hdbscan`、`scikit-learn`、`spacy`、`openai`、`pyyaml`。 + +--- + +## 8. 维护说明 + +- 修改 LLM 话术：编辑 `prompts/` 下对应 `.md`，**勿改** `schema.yaml` 中 `report.markers` 四段标记名（见 `prompts/README.md`）。 +- 修改主流程步骤顺序或默认路径：改 `main_voc分析.py` 后请同步更新**本文档**。 +- 验收 prompt 加载：`python3 prompts/smoke.py`（无需 Key）；联调模型：`python3 prompts/smoke.py --live`。 + +--- + +*文档版本：与仓库 `main_voc分析.py` 七步流程一致。* diff --git a/main_voc分析.py b/main_voc分析.py new file mode 100644 index 0000000..cbeedf3 --- /dev/null +++ b/main_voc分析.py @@ -0,0 +1,393 @@ +""" +VOC 全流程：合并 → 清洗 → 结构化 → 向量化 →（聚类 ∥ 词频）→ 分析报告 HTML。 + +用法:: + + # 全流程（--industry 默认 Pet Supplies；写入 sqlite 前默认清理旧库，keep-db不清理） + python3 main_voc分析.py --input-dir 目录 --product "产品名" --keep-db + + # 断点续跑（步骤 4 起可省略 --product，自动读 voc_structured.sqlite） + python3 main_voc分析.py --from-step 5 + python3 main_voc分析.py --from-step 6 --skip-wordfreq-llm # 仅重跑词频 + python3 main_voc分析.py --only-step 7 # 仅生成报告 + + # 保留已有 voc_structured / voc_embeddings / voc_clustering.sqlite，不清理覆盖 + python3 main_voc分析.py --from-step 4 --keep-db + +报告 HTML：output/{product_name}/{product_name}_voc_report.html +""" +from __future__ import annotations + +import argparse +import json +import logging +import re +import sqlite3 +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from content清洗 import process_reviews, save_cleaned_reviews +from 合并评论数据 import merge_csv_directory +from 向量化 import run_embed +from 结构化_server import run_analysis +from 聚类 import run_clustering +from voc_report import DEFAULT_CLUSTER_MIN_REVIEW_RATIO, generate_report +from 词频 import run as run_wordfreq + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_analysis") + +PROJECT_ROOT = Path(__file__).resolve().parent +OUTPUT_DIR = PROJECT_ROOT / "output" + +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite" +CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite" +TERMS_JSON = OUTPUT_DIR / "voc_terms.json" +WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv" + +DEFAULT_MERGED = PROJECT_ROOT / "merged_reviews.csv" +DEFAULT_CLEANED = PROJECT_ROOT / "merged_reviews_cleaned.csv" +DEFAULT_INDUSTRY = "Pet Supplies" + + +def _safe_product_dir_name(product_name: str) -> str: + """用于 output 子目录名；去除路径非法字符。""" + name = (product_name or "product").strip() + name = re.sub(r'[<>:"/\\|?*]', "_", name) + return name or "product" + + +def _product_output_dir(product_name: str) -> Path: + return OUTPUT_DIR / _safe_product_dir_name(product_name) + + +def _report_html_path(product_name: str) -> Path: + slug = _safe_product_dir_name(product_name) + return _product_output_dir(product_name) / f"{slug}_voc_report.html" + + +def _latest_job_meta(structured_db: Path) -> tuple[int, str, str]: + conn = sqlite3.connect(structured_db) + try: + row = conn.execute( + """ + SELECT id, industry, product_name + FROM analysis_jobs ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + if not row: + raise RuntimeError("无结构化任务，请先执行步骤 3") + return int(row[0]), str(row[1]), str(row[2]) + finally: + conn.close() + + +def _should_run(step: int, from_step: int, only_step: int | None) -> bool: + if only_step is not None: + return step == only_step + return step >= from_step + + +def _clean_intermediates( + *, + merged: Path, + cleaned: Path, + structured: Path, + embed: Path, + cluster: Path, + terms: Path, + word_freq: Path, +) -> None: + for p in (merged, cleaned, structured, embed, cluster, terms, word_freq): + if p.is_file(): + p.unlink() + logger.info("已删除中间文件 %s", p) + + +def run_voc_analysis( + *, + input_dir: Path, + industry: str, + product_name: str, + merged_csv: Path, + cleaned_csv: Path, + from_step: int = 1, + only_step: int | None = None, + clean_intermediates: bool = False, + clean_databases: bool = True, + skip_wordfreq_llm: bool = False, + min_cluster_review_ratio: float | None = None, + save_llm_raw: bool = False, +) -> dict: + result: dict = {} + + if _should_run(1, from_step, only_step): + logger.info("步骤 1/7：合并 CSV") + merge_csv_directory(input_dir, merged_csv) + result["merged_csv"] = str(merged_csv) + + if _should_run(2, from_step, only_step): + logger.info("步骤 2/7：清洗评论") + if not merged_csv.is_file(): + raise FileNotFoundError(f"缺少 {merged_csv}，请先执行合并") + df = process_reviews(merged_csv) + save_cleaned_reviews(df, cleaned_csv) + result["cleaned_csv"] = str(cleaned_csv) + + if _should_run(3, from_step, only_step): + logger.info("步骤 3/7：结构化分析") + if not cleaned_csv.is_file(): + raise FileNotFoundError(f"缺少 {cleaned_csv}，请先执行清洗") + ar = run_analysis( + industry=industry, + product_name=product_name, + file_path=str(cleaned_csv), + clean_databases=clean_databases, + ) + result["structured_job_id"] = ar.get("job_id") + result["structured_db"] = str(STRUCTURED_DB) + + job_id: int | None = None + ind, prod = industry, product_name + if STRUCTURED_DB.is_file(): + try: + job_id, ind, prod = _latest_job_meta(STRUCTURED_DB) + except RuntimeError: + pass + + if _should_run(4, from_step, only_step): + if clean_databases and not _should_run(3, from_step, only_step): + logger.info("写入向量化库前清理 voc_embeddings.sqlite / voc_clustering.sqlite") + for p in (EMBED_DB, CLUSTER_DB): + if p.is_file(): + p.unlink() + logger.info("已清理 SQLite: %s", p.name) + logger.info("步骤 4/7：向量化") + er = run_embed( + job_id=job_id, + csv_path=cleaned_csv, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + reset_db=clean_databases, + ) + result["embed"] = er + job_id = int(er.get("job_id", job_id or 0)) + + if job_id is None and STRUCTURED_DB.is_file(): + job_id, ind, prod = _latest_job_meta(STRUCTURED_DB) + + run_cluster = _should_run(5, from_step, only_step) + run_wf = _should_run(6, from_step, only_step) + + if run_cluster and run_wf: + logger.info("步骤 5–6/7：聚类与词频（并行）") + with ThreadPoolExecutor(max_workers=2) as pool: + f_cluster = pool.submit( + run_clustering, + job_id=job_id, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + cluster_db=CLUSTER_DB, + reset_db=clean_databases, + ) + f_wf = pool.submit(run_wordfreq, skip_llm=skip_wordfreq_llm) + result["clustering"] = f_cluster.result() + result["wordfreq"] = f_wf.result() + else: + if run_cluster: + logger.info("步骤 5/7：聚类") + result["clustering"] = run_clustering( + job_id=job_id, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + cluster_db=CLUSTER_DB, + reset_db=clean_databases, + ) + if run_wf: + logger.info("步骤 6/7：词频") + result["wordfreq"] = run_wordfreq(skip_llm=skip_wordfreq_llm) + + if _should_run(7, from_step, only_step): + logger.info("步骤 7/7：生成分析报告") + if not CLUSTER_DB.is_file(): + raise FileNotFoundError(f"缺少 {CLUSTER_DB}") + if not WORD_FREQ_CSV.is_file(): + raise FileNotFoundError(f"缺少 {WORD_FREQ_CSV}") + if not cleaned_csv.is_file(): + raise FileNotFoundError(f"缺少 {cleaned_csv}") + if STRUCTURED_DB.is_file(): + _, ind, prod = _latest_job_meta(STRUCTURED_DB) + report_html = _report_html_path(prod) + result["report_html"] = str(report_html) + result["report"] = generate_report( + product_name=prod, + industry=ind, + cleaned_csv=cleaned_csv, + cluster_db=CLUSTER_DB, + embed_db=EMBED_DB, + word_freq_csv=WORD_FREQ_CSV, + output_html=report_html, + structured_db=STRUCTURED_DB, + min_cluster_review_ratio=min_cluster_review_ratio, + save_llm_raw=save_llm_raw, + ) + + if clean_intermediates and _should_run(7, from_step, only_step): + _clean_intermediates( + merged=merged_csv, + cleaned=cleaned_csv, + structured=STRUCTURED_DB, + embed=EMBED_DB, + cluster=CLUSTER_DB, + terms=TERMS_JSON, + word_freq=WORD_FREQ_CSV, + ) + + return result + + +# 兼容旧名 +run_pipeline = run_voc_analysis + + +def _resolve_industry_product( + *, + industry: str | None, + product_name: str | None, + from_step: int, + only_step: int | None, +) -> tuple[str, str]: + """步骤 4 起可从 structured 库自动读取 product；industry 默认 Pet Supplies。""" + ind = (industry or DEFAULT_INDUSTRY).strip() or DEFAULT_INDUSTRY + if product_name: + return ind, product_name + start_step = only_step if only_step is not None else from_step + if start_step >= 4: + if not STRUCTURED_DB.is_file(): + raise SystemExit( + f"缺少 {STRUCTURED_DB.name}，无法自动读取 --product" + ) + _, db_ind, prod = _latest_job_meta(STRUCTURED_DB) + logger.info("未指定 product，已从 structured 库读取: %r（industry=%r）", prod, db_ind) + return db_ind, prod + raise SystemExit( + f"步骤 1–3 需要 --product；--industry 可省略（默认 {DEFAULT_INDUSTRY}）；" + "从步骤 4 起 product 也可省略（自动读 voc_structured.sqlite）" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 分析全流程") + parser.add_argument( + "--input-dir", + type=Path, + default=None, + help="原始 VOC CSV 目录（步骤 1 合并输入；续跑后续步骤可不传）", + ) + parser.add_argument( + "--industry", + default=DEFAULT_INDUSTRY, + help=f"行业（默认 {DEFAULT_INDUSTRY}）", + ) + parser.add_argument("--product", default=None, help="产品名（步骤 4 起可省略，自动读库）") + parser.add_argument( + "--merged-csv", + type=Path, + default=DEFAULT_MERGED, + help=f"合并输出（默认 {DEFAULT_MERGED.name}）", + ) + parser.add_argument( + "--cleaned-csv", + type=Path, + default=DEFAULT_CLEANED, + help=f"清洗输出（默认 {DEFAULT_CLEANED.name}）", + ) + parser.add_argument( + "--from-step", + type=int, + default=1, + choices=range(1, 8), + metavar="N", + help="从第 N 步开始执行（1–7，默认 1）", + ) + parser.add_argument( + "--only-step", + type=int, + default=None, + choices=range(1, 8), + metavar="N", + help="仅执行第 N 步（需已有前置产物）", + ) + parser.add_argument( + "--keep-db", + action="store_true", + help="保留已有 voc_structured / voc_embeddings / voc_clustering.sqlite，不覆盖清理（默认写入前会清理）", + ) + parser.add_argument( + "--clean-intermediates", + action="store_true", + help="报告生成成功后删除中间 sqlite/csv（仅保留各产品子目录下的报告 HTML）", + ) + parser.add_argument( + "--skip-wordfreq-llm", + action="store_true", + help="词频步骤复用已有 voc_terms.json", + ) + parser.add_argument( + "--filter-small-clusters", + action="store_true", + help=( + "报告仅纳入本 stage 内去重评论占比 ≥ " + f"{DEFAULT_CLUSTER_MIN_REVIEW_RATIO * 100:.0f}%% 的簇；默认不筛选、全部簇参与" + ), + ) + parser.add_argument( + "--save-llm-raw", + action="store_true", + help="步骤 7 将报告 LLM 完整原文写入报告同目录下的 report_llm_raw.txt（默认不保存）", + ) + args = parser.parse_args() + + need_input = args.only_step in (None, 1) and args.from_step <= 1 + if need_input: + if not args.input_dir: + raise SystemExit("步骤 1 需要 --input-dir") + if not args.input_dir.is_dir(): + raise SystemExit(f"输入目录不存在: {args.input_dir}") + input_dir = (args.input_dir or DEFAULT_MERGED.parent).resolve() + industry, product_name = _resolve_industry_product( + industry=args.industry, + product_name=args.product, + from_step=args.from_step, + only_step=args.only_step, + ) + + out = run_voc_analysis( + input_dir=input_dir, + industry=industry, + product_name=product_name, + merged_csv=args.merged_csv.resolve(), + cleaned_csv=args.cleaned_csv.resolve(), + from_step=args.from_step, + only_step=args.only_step, + clean_intermediates=args.clean_intermediates, + clean_databases=not args.keep_db, + skip_wordfreq_llm=args.skip_wordfreq_llm, + min_cluster_review_ratio=( + DEFAULT_CLUSTER_MIN_REVIEW_RATIO + if args.filter_small_clusters + else None + ), + save_llm_raw=args.save_llm_raw, + ) + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/main_voc分析_jieba.py b/main_voc分析_jieba.py new file mode 100644 index 0000000..7085aa2 --- /dev/null +++ b/main_voc分析_jieba.py @@ -0,0 +1,342 @@ +""" +VOC 全流程（jieba 分词版）：与 main_voc分析.py 相同，步骤 6 词频改用 jieba。 + +用法:: + + python3 main_voc分析_jieba.py --input-dir reviews_export --industry "Pet supplements" --product "Turkey tail mushroom for dogs" + + python3 main_voc分析_jieba.py --from-step 6 # 仅重跑 jieba 词频（可省略 --industry/--product） + python3 main_voc分析_jieba.py --only-step 7 # 仅生成报告 +""" +from __future__ import annotations + +import argparse +import json +import logging +import sqlite3 +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from content清洗 import process_reviews, save_cleaned_reviews +from 合并评论数据 import merge_csv_directory +from 向量化 import run_embed +from 结构化_server import run_analysis +from 聚类 import run_clustering +from voc_report import DEFAULT_CLUSTER_MIN_REVIEW_RATIO, generate_report +from 词频_jieba import run as run_wordfreq + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_analysis_jieba") + +PROJECT_ROOT = Path(__file__).resolve().parent +OUTPUT_DIR = PROJECT_ROOT / "output" +REPORT_HTML = OUTPUT_DIR / "voc_report.html" + +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite" +CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite" +TERMS_JSON = OUTPUT_DIR / "voc_terms.json" +WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv" + +DEFAULT_MERGED = PROJECT_ROOT / "merged_reviews.csv" +DEFAULT_CLEANED = PROJECT_ROOT / "merged_reviews_cleaned.csv" + + +def _latest_job_meta(structured_db: Path) -> tuple[int, str, str]: + conn = sqlite3.connect(structured_db) + try: + row = conn.execute( + """ + SELECT id, industry, product_name + FROM analysis_jobs ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + if not row: + raise RuntimeError("无结构化任务，请先执行步骤 3") + return int(row[0]), str(row[1]), str(row[2]) + finally: + conn.close() + + +def _should_run(step: int, from_step: int, only_step: int | None) -> bool: + if only_step is not None: + return step == only_step + return step >= from_step + + +def _clean_intermediates( + *, + merged: Path, + cleaned: Path, + structured: Path, + embed: Path, + cluster: Path, + terms: Path, + word_freq: Path, +) -> None: + for p in (merged, cleaned, structured, embed, cluster, terms, word_freq): + if p.is_file(): + p.unlink() + logger.info("已删除中间文件 %s", p) + + +def run_voc_analysis( + *, + input_dir: Path, + industry: str, + product_name: str, + merged_csv: Path, + cleaned_csv: Path, + from_step: int = 1, + only_step: int | None = None, + clean_intermediates: bool = False, + skip_wordfreq_llm: bool = False, + min_cluster_review_ratio: float | None = None, + save_llm_raw: bool = False, +) -> dict: + result: dict = {"report_html": str(REPORT_HTML), "tokenizer": "jieba"} + + if _should_run(1, from_step, only_step): + logger.info("步骤 1/7：合并 CSV") + merge_csv_directory(input_dir, merged_csv) + result["merged_csv"] = str(merged_csv) + + if _should_run(2, from_step, only_step): + logger.info("步骤 2/7：清洗评论") + if not merged_csv.is_file(): + raise FileNotFoundError(f"缺少 {merged_csv}，请先执行合并") + df = process_reviews(merged_csv) + save_cleaned_reviews(df, cleaned_csv) + result["cleaned_csv"] = str(cleaned_csv) + + if _should_run(3, from_step, only_step): + logger.info("步骤 3/7：结构化分析") + if not cleaned_csv.is_file(): + raise FileNotFoundError(f"缺少 {cleaned_csv}，请先执行清洗") + ar = run_analysis( + industry=industry, + product_name=product_name, + file_path=str(cleaned_csv), + ) + result["structured_job_id"] = ar.get("job_id") + result["structured_db"] = str(STRUCTURED_DB) + + job_id: int | None = None + ind, prod = industry, product_name + if STRUCTURED_DB.is_file(): + try: + job_id, ind, prod = _latest_job_meta(STRUCTURED_DB) + except RuntimeError: + pass + + if _should_run(4, from_step, only_step): + logger.info("步骤 4/7：向量化") + er = run_embed( + job_id=job_id, + csv_path=cleaned_csv, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + ) + result["embed"] = er + job_id = int(er.get("job_id", job_id or 0)) + + if job_id is None and STRUCTURED_DB.is_file(): + job_id, ind, prod = _latest_job_meta(STRUCTURED_DB) + + run_cluster = _should_run(5, from_step, only_step) + run_wf = _should_run(6, from_step, only_step) + + if run_cluster and run_wf: + logger.info("步骤 5–6/7：聚类与词频（jieba，并行）") + with ThreadPoolExecutor(max_workers=2) as pool: + f_cluster = pool.submit( + run_clustering, + job_id=job_id, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + cluster_db=CLUSTER_DB, + ) + f_wf = pool.submit(run_wordfreq, skip_llm=skip_wordfreq_llm) + result["clustering"] = f_cluster.result() + result["wordfreq"] = f_wf.result() + else: + if run_cluster: + logger.info("步骤 5/7：聚类") + result["clustering"] = run_clustering( + job_id=job_id, + structured_db=STRUCTURED_DB, + embed_db=EMBED_DB, + cluster_db=CLUSTER_DB, + ) + if run_wf: + logger.info("步骤 6/7：词频（jieba）") + result["wordfreq"] = run_wordfreq(skip_llm=skip_wordfreq_llm) + + if _should_run(7, from_step, only_step): + logger.info("步骤 7/7：生成分析报告") + if not CLUSTER_DB.is_file(): + raise FileNotFoundError(f"缺少 {CLUSTER_DB}") + if not WORD_FREQ_CSV.is_file(): + raise FileNotFoundError(f"缺少 {WORD_FREQ_CSV}") + if not cleaned_csv.is_file(): + raise FileNotFoundError(f"缺少 {cleaned_csv}") + if STRUCTURED_DB.is_file(): + _, ind, prod = _latest_job_meta(STRUCTURED_DB) + result["report"] = generate_report( + product_name=prod, + industry=ind, + cleaned_csv=cleaned_csv, + cluster_db=CLUSTER_DB, + embed_db=EMBED_DB, + word_freq_csv=WORD_FREQ_CSV, + output_html=REPORT_HTML, + structured_db=STRUCTURED_DB, + min_cluster_review_ratio=min_cluster_review_ratio, + save_llm_raw=save_llm_raw, + ) + + if clean_intermediates and _should_run(7, from_step, only_step): + _clean_intermediates( + merged=merged_csv, + cleaned=cleaned_csv, + structured=STRUCTURED_DB, + embed=EMBED_DB, + cluster=CLUSTER_DB, + terms=TERMS_JSON, + word_freq=WORD_FREQ_CSV, + ) + + return result + + +run_pipeline = run_voc_analysis + + +def _resolve_industry_product( + *, + industry: str | None, + product_name: str | None, + from_step: int, + only_step: int | None, +) -> tuple[str, str]: + """步骤 4 起可从 structured 库自动读取；步骤 1–3 须 CLI 传入。""" + if industry and product_name: + return industry, product_name + start_step = only_step if only_step is not None else from_step + if start_step >= 4: + if not STRUCTURED_DB.is_file(): + raise SystemExit( + f"缺少 {STRUCTURED_DB.name}，无法自动读取 --industry / --product" + ) + _, ind, prod = _latest_job_meta(STRUCTURED_DB) + logger.info("未指定 industry/product，已从 structured 库读取: %r / %r", ind, prod) + return ind, prod + raise SystemExit( + "步骤 1–3 需要 --industry 与 --product；从步骤 4 起可省略（自动读 voc_structured.sqlite）" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 分析全流程（jieba 分词）") + parser.add_argument( + "--input-dir", + type=Path, + default=None, + help="原始 VOC CSV 目录（步骤 1 合并输入；续跑后续步骤可不传）", + ) + parser.add_argument("--industry", default=None, help="行业（步骤 4 起可省略，自动读库）") + parser.add_argument("--product", default=None, help="产品名（步骤 4 起可省略，自动读库）") + parser.add_argument( + "--merged-csv", + type=Path, + default=DEFAULT_MERGED, + help=f"合并输出（默认 {DEFAULT_MERGED.name}）", + ) + parser.add_argument( + "--cleaned-csv", + type=Path, + default=DEFAULT_CLEANED, + help=f"清洗输出（默认 {DEFAULT_CLEANED.name}）", + ) + parser.add_argument( + "--from-step", + type=int, + default=1, + choices=range(1, 8), + metavar="N", + help="从第 N 步开始执行（1–7，默认 1）", + ) + parser.add_argument( + "--only-step", + type=int, + default=None, + choices=range(1, 8), + metavar="N", + help="仅执行第 N 步（需已有前置产物）", + ) + parser.add_argument( + "--clean-intermediates", + action="store_true", + help="报告生成成功后删除中间 sqlite/csv（仅保留 voc_report.html）", + ) + parser.add_argument( + "--skip-wordfreq-llm", + action="store_true", + help="词频步骤复用已有 voc_terms.json", + ) + parser.add_argument( + "--filter-small-clusters", + action="store_true", + help=( + "报告仅纳入本 stage 内去重评论占比 ≥ " + f"{DEFAULT_CLUSTER_MIN_REVIEW_RATIO * 100:.0f}%% 的簇；默认不筛选、全部簇参与" + ), + ) + parser.add_argument( + "--save-llm-raw", + action="store_true", + help="步骤 7 将报告 LLM 完整原文写入 output/report_llm_raw.txt（默认不保存）", + ) + args = parser.parse_args() + + need_input = args.only_step in (None, 1) and args.from_step <= 1 + if need_input: + if not args.input_dir: + raise SystemExit("步骤 1 需要 --input-dir") + if not args.input_dir.is_dir(): + raise SystemExit(f"输入目录不存在: {args.input_dir}") + input_dir = (args.input_dir or DEFAULT_MERGED.parent).resolve() + industry, product_name = _resolve_industry_product( + industry=args.industry, + product_name=args.product, + from_step=args.from_step, + only_step=args.only_step, + ) + + out = run_voc_analysis( + input_dir=input_dir, + industry=industry, + product_name=product_name, + merged_csv=args.merged_csv.resolve(), + cleaned_csv=args.cleaned_csv.resolve(), + from_step=args.from_step, + only_step=args.only_step, + clean_intermediates=args.clean_intermediates, + skip_wordfreq_llm=args.skip_wordfreq_llm, + min_cluster_review_ratio=( + DEFAULT_CLUSTER_MIN_REVIEW_RATIO + if args.filter_small_clusters + else None + ), + save_llm_raw=args.save_llm_raw, + ) + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/prompts/README.md b/prompts/README.md new file mode 100644 index 0000000..4f34ab9 --- /dev/null +++ b/prompts/README.md @@ -0,0 +1,33 @@ +# Prompt 编辑说明（产品 / 运营 / 开发） + +修改后**下次运行脚本自动生效**，无需重启服务。 + +## 可编辑文件 + +| 路径 | 谁改 | 内容 | +|------|------|------| +| `config.yaml` | 产品/运营 | Top N 数值、中文词频六类名称 | +| `extraction/*.md` | 产品/运营 | 结构化提取规则、语气 | +| `extraction/examples.yaml` | 产品/运营 | Few-shot “示例”（output 用 YAML 对象，勿手写 JSON 字符串） | +| `report/*.md` | 产品/运营 | 报告分析要求、输出格式、修正提示 | +| `word_freq/*.md` | 产品/运营 | 词频分类与解读 prompt（含 `category_definitions.md` 各类定义） | +| `schema.yaml` | **仅开发** | JSON 字段名、英文 category、四段输出标记 | + +## 占位符 + +`.md` 文件使用 Python `format` 语法，例如 `{product_name}`、`{industry}`。 +正文里需要字面量花括号时写双花括号：`{{"audience": "unknown"}}`。 + +## 改完怎么验收 + +```bash +cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化" +python3 prompts/smoke.py # 检查文件能否加载（无需 API Key） +python3 prompts/smoke.py --live # 用 3 条样例评论调模型（需 DASHSCOPE_API_KEY） +``` + +## 注意 + +- 不要改 `schema.yaml` 里的 `report.markers` 四段标记名，否则报告解析会失败。 +- `examples.yaml` 里 `sentiment` / `category` 须与 `schema.yaml` 枚举一致。 +- 词频分类若增删类别，需同步 `config.yaml` 的 `word_categories` 与 `voc_report.py` 内 `CATEGORY_COLORS` 配色。 diff --git a/prompts/__init__.py b/prompts/__init__.py new file mode 100644 index 0000000..551f201 --- /dev/null +++ b/prompts/__init__.py @@ -0,0 +1,45 @@ +"""Prompt 包：外置 .md / .yaml 加载与组装。""" + +from prompts.loader import ( + build_batch_extraction_system, + build_batch_extraction_user, + build_category_analysis_prompts, + build_report_analysis_requirements, + build_report_correction_message, + build_report_json_markers, + build_report_output_format, + build_report_system, + build_single_extraction_template, + build_word_assign_prompts, + format_examples, + get_config, + get_schema, + load_extraction_examples, + product_feedback_categories, + report_markers, + sync_voc_report_constants, + validate_prompt_files, + word_categories, +) + +__all__ = [ + "build_batch_extraction_system", + "build_batch_extraction_user", + "build_category_analysis_prompts", + "build_report_analysis_requirements", + "build_report_correction_message", + "build_report_json_markers", + "build_report_output_format", + "build_report_system", + "build_single_extraction_template", + "build_word_assign_prompts", + "format_examples", + "get_config", + "get_schema", + "load_extraction_examples", + "product_feedback_categories", + "report_markers", + "sync_voc_report_constants", + "validate_prompt_files", + "word_categories", +] diff --git a/prompts/config.yaml b/prompts/config.yaml new file mode 100644 index 0000000..7c340e9 --- /dev/null +++ b/prompts/config.yaml @@ -0,0 +1,26 @@ +# 产品/运营可修改：数值与中文词频分类枚举 +# 修改后下次运行脚本自动生效（无需重启） + +word_freq: + wordcloud_top_n: 180 + word_freq_table_n: 180 + word_category_classify_n: 180 + word_freq_page_size: 30 + word_freq_pages: 6 + word_category_assign_batch_size: 40 + word_category_analysis_max_words: 25 + +report: + outlier_label_zh: "未归类" + max_phrases_per_cluster: 15 + +# 词频分类（中文展示名；须与程序词云配色顺序一致） +word_categories: + - "成分/原料" + - "剂型" + - "受众/使用对象" + - "功效/功能" + - "需求/场景" + - "品质/体验" + - "价格/价值" + - "物流/包装" diff --git a/prompts/extraction/batch_output_format.md b/prompts/extraction/batch_output_format.md new file mode 100644 index 0000000..98d5aab --- /dev/null +++ b/prompts/extraction/batch_output_format.md @@ -0,0 +1,7 @@ +## 批量输出格式（必须严格遵守）: +本批共 {n_keys} 条评论，用户消息中每条评论以 [C1]、[C2]… 前缀标识。 +- 只输出一个 JSON 对象；顶层键必须且仅能是：{keys_literal} +- 每个顶层键对应一条同前缀评论，不得遗漏、不得新增其他顶层键 +- 每个键的值是单条结构化对象，仅含 audience、pain_points、product_feedback 三个字段 +- 不要用 JSON 数组作为顶层；不要把多条评论合并进一个对象；不要用 results、data 等包裹层 +- 不要输出 markdown 代码围栏或任何解释文字 diff --git a/prompts/extraction/examples.yaml b/prompts/extraction/examples.yaml new file mode 100644 index 0000000..f31ceb3 --- /dev/null +++ b/prompts/extraction/examples.yaml @@ -0,0 +1,69 @@ +# Few-shot 示例：output 为 YAML 对象，程序自动转为 JSON 字符串 +- instruction: "示例 1 教学：展示正常长评论如何标准提取，如何准确分类产品优点、缺陷以及物流问题。" + review: "Bought this for my 12yo lab who struggles with stairs. It is very soft and helps her get onto the bed easily. But the zipper broke after a week and the shipping box was damaged." + output: + audience: "12yo lab" + pain_points: + - "struggles with stairs" + - "difficulty getting onto bed" + product_feedback: + - aspect: "fabric" + opinion: "very soft" + sentiment: Positive + category: Quality + - aspect: "zipper" + opinion: "broke after a week" + sentiment: Negative + category: Quality + - aspect: "shipping box" + opinion: "damaged" + sentiment: Negative + category: Logistics + +- instruction: "示例 2 教学：防混淆与隐性属性提炼。什么是「产品缺陷（如讲太快）」，它属于 product_feedback，绝不是 pain_points（用户需求）！并提炼出 teaching speed 这个隐性反馈对象。" + review: "The course content is good, the teacher speaks too fast, can't keep up." + output: + audience: "student" + pain_points: [] + product_feedback: + - aspect: "course content" + opinion: "good" + sentiment: Positive + category: Quality + - aspect: "teaching speed" + opinion: "too fast to keep up" + sentiment: Negative + category: Function + +- instruction: "示例 3 教学：深度上下文推理。如何通过动作（抓跳蚤）推理出使用者（pet），如何通过「全家能睡觉」推理出用户需求（失眠），以及如何将「Worth every penny」准确归类为对 Price（价格）的产品反馈。" + review: "Finally found something that stops the midnight flea scratching! The chemical smell is a bit strong initially, but it fades. Worth every penny since our family can finally sleep." + output: + audience: "pet" + pain_points: + - "midnight flea scratching" + - "sleep deprivation" + product_feedback: + - aspect: "chemical smell" + opinion: "strong initially but fades" + sentiment: Neutral + category: Quality + - aspect: "price" + opinion: "worth every penny" + sentiment: Positive + category: Price + +- instruction: "示例 4 教学：保留关键成分与症状等核心具体词汇。绝不能将带有具体成分/病症的词泛化提取。例如遇到「chicken flavor」（鸡肉风味）或「joint pain」（关节疼痛）时，必须保留核心修饰词，绝不能错误缩减提取为「flavor」或「pain」。" + review: "My elderly cat suffers from severe joint pain. But he is very attracted by the chicken flavor of this supplement! It really helps him walk better." + output: + audience: "elderly cat" + pain_points: + - "severe joint pain" + product_feedback: + - aspect: "chicken flavor" + opinion: "very attracted" + sentiment: Positive + category: Quality + - aspect: "mobility improvement" + opinion: "helps walk better" + sentiment: Positive + category: Function diff --git a/prompts/extraction/examples_footer_batch.md b/prompts/extraction/examples_footer_batch.md new file mode 100644 index 0000000..d218502 --- /dev/null +++ b/prompts/extraction/examples_footer_batch.md @@ -0,0 +1 @@ +(以上为单条示例；你对用户消息中的每一条带前缀评论分别做同样的结构化提取。) diff --git a/prompts/extraction/examples_section.md b/prompts/extraction/examples_section.md new file mode 100644 index 0000000..9716ec6 --- /dev/null +++ b/prompts/extraction/examples_section.md @@ -0,0 +1,3 @@ +## 分析示例 (请学习以下示例中的推理逻辑): +在分析{product_name}时，你需要参考以下跨行业示例的逻辑， +{examples_str} diff --git a/prompts/extraction/field_rules.md b/prompts/extraction/field_rules.md new file mode 100644 index 0000000..32e4423 --- /dev/null +++ b/prompts/extraction/field_rules.md @@ -0,0 +1,18 @@ +## 分析要求: + +1. audience (为谁购买): + - 提取出实际的使用者，使用简短的英文名词。若无明确提及，请根据上下文推理；若完全无法推理则输出 'unknown'。 + +2. pain_points (用户需求): + - 仅限提取用户在购买前遇到的外部困扰、疾病、或具体场景（购买前尚未被本产品解决的需求）。 + - **注意保留具体病症/需求核心词**（如 "joint pain" 不能缩减为 "pain"）。 + - **注意防重**：不要把「产品本身的缺陷或优点」当做用户需求提取。若无提及，输出空列表 []。 + +3. product_feedback (产品反馈): + - 将用户对产品的反馈拆解为具体对象与反馈内容，包含以下子字段： + - aspect (对象): 提炼成准确的英文简短名词。**务必保留具体的成分、材质或特定属性词**（例如 "chicken flavor" 不能泛化为 "flavor"）。 + - opinion (反馈内容): 必须是英文简短词组。 + - sentiment (情感): 仅限 {sentiments_literal}。 + - category (类别): 仅限 {categories_literal}（禁止 Value、Cost 等自创词；性价比高/物有所值 归入 Price）。 + - 每条 product_feedback 必须同时包含 aspect、opinion、sentiment、category 四个子字段。 + - 若无提及 product_feedback，输出空列表 [] diff --git a/prompts/extraction/filter_irrelevant.md b/prompts/extraction/filter_irrelevant.md new file mode 100644 index 0000000..b337ee7 --- /dev/null +++ b/prompts/extraction/filter_irrelevant.md @@ -0,0 +1,3 @@ +5. 无关评论过滤： + - 若某条评论明显与{product_name}无关（其他品类、其他 SKU 或完全跑题），该条输出：{{"audience": "unknown", "pain_points": [], "product_feedback": []}}。 + - 不得将无关内容填入 audience、pain_points 或 product_feedback。 diff --git a/prompts/extraction/format_constraints_batch.md b/prompts/extraction/format_constraints_batch.md new file mode 100644 index 0000000..0f37d81 --- /dev/null +++ b/prompts/extraction/format_constraints_batch.md @@ -0,0 +1,3 @@ +4. 单条评论对象内的格式与字段约束： + - 每个评论对象只能包含 `audience`, `pain_points`, `product_feedback` 这 3 个字段。 + - **绝对不要**在 JSON 中输出 `instruction`、`教学说明` 或其他任何多余字段。 diff --git a/prompts/extraction/format_constraints_single.md b/prompts/extraction/format_constraints_single.md new file mode 100644 index 0000000..fa3119d --- /dev/null +++ b/prompts/extraction/format_constraints_single.md @@ -0,0 +1,4 @@ +4. 格式与字段约束： + - **你的 JSON 输出只能包含 `audience`, `pain_points`, `product_feedback` 这 3 个根字段。** + - **绝对不要**在 JSON 中输出 `instruction`、`教学说明` 或其他任何多余字段。 + - 必须以纯 JSON 格式输出结果，不要包含任何 markdown 标记（如 ```json ）或其他解释性文字。 diff --git a/prompts/extraction/system_intro.md b/prompts/extraction/system_intro.md new file mode 100644 index 0000000..3625b4c --- /dev/null +++ b/prompts/extraction/system_intro.md @@ -0,0 +1,4 @@ +你是一个专业的亚马逊电商评论分析专家，当前正在分析【{industry}】领域的【{product_name}】产品。 +请精准、简短地提炼评论中的核心信息，需要你具备深入的上下文推理能力（不仅仅是提取字面词汇，需结合{product_name}的使用语境）。 + +为了避免后续词频统计重复，请严格遵守各维度的定义，**同一概念或词汇绝对不能在不同字段中重复提取**。 diff --git a/prompts/extraction/user_batch.md b/prompts/extraction/user_batch.md new file mode 100644 index 0000000..848c38f --- /dev/null +++ b/prompts/extraction/user_batch.md @@ -0,0 +1,3 @@ +以下为 {n_keys} 段带前缀的英文评论，每段互相独立。请严格按 system 中的「批量输出格式」返回 JSON，顶层键为 {keys_literal}。 + +{tagged_input} diff --git a/prompts/extraction/user_single_tail.md b/prompts/extraction/user_single_tail.md new file mode 100644 index 0000000..0c1220e --- /dev/null +++ b/prompts/extraction/user_single_tail.md @@ -0,0 +1,3 @@ +## 请分析以下评论: +[输入评论]: "{{review_content}}" +[输出 JSON]: diff --git a/prompts/loader.py b/prompts/loader.py new file mode 100644 index 0000000..eeeaf7a --- /dev/null +++ b/prompts/loader.py @@ -0,0 +1,320 @@ +""" +外置 Prompt 加载器：每次调用均从磁盘重读 .md / .yaml（无缓存）。 + +编辑入口： + prompts/config.yaml — 数值、中文词频分类（产品/运营可改） + prompts/schema.yaml — 锁定 JSON 字段与标记（改前需开发确认） + prompts/extraction/ — 结构化提取规则与示例 + prompts/report/ — 报告静态规则 + prompts/word_freq/ — 词频分类/解读 +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import yaml + +PROMPTS_ROOT = Path(__file__).resolve().parent + + +def get_config() -> Dict[str, Any]: + with (PROMPTS_ROOT / "config.yaml").open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def get_schema() -> Dict[str, Any]: + with (PROMPTS_ROOT / "schema.yaml").open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def read_text(relative_path: str) -> str: + return (PROMPTS_ROOT / relative_path).read_text(encoding="utf-8") + + +def render(relative_path: str, **kwargs: Any) -> str: + return read_text(relative_path).format(**kwargs).strip() + + +def word_categories(config: Dict[str, Any] | None = None) -> Tuple[str, ...]: + cfg = config or get_config() + return tuple(cfg.get("word_categories") or ()) + + +def report_markers(schema: Dict[str, Any] | None = None) -> Tuple[str, ...]: + sch = schema or get_schema() + return tuple(sch.get("report", {}).get("markers") or ()) + + +def product_feedback_categories(schema: Dict[str, Any] | None = None) -> frozenset[str]: + sch = schema or get_schema() + cats = sch.get("extraction", {}).get("product_feedback_categories") or [] + return frozenset(str(c) for c in cats) + + +def _categories_literal(schema: Dict[str, Any]) -> str: + cats = schema.get("extraction", {}).get("product_feedback_categories") or [] + return ", ".join(f"'{c}'" for c in cats) + + +def _sentiments_literal(schema: Dict[str, Any]) -> str: + sent = schema.get("extraction", {}).get("sentiments") or [] + return ", ".join(f"'{s}'" for s in sent) + + +def load_extraction_examples() -> List[Dict[str, Any]]: + with (PROMPTS_ROOT / "extraction/examples.yaml").open(encoding="utf-8") as f: + return yaml.safe_load(f) or [] + + +def format_examples(examples: List[Dict[str, Any]] | None = None) -> str: + items = examples if examples is not None else load_extraction_examples() + lines: List[str] = [] + for i, ex in enumerate(items, 1): + output_json = json.dumps(ex["output"], ensure_ascii=False, separators=(",", ":")) + lines.append(f"=== 示例 {i} ===") + lines.append(f"[教学说明]: {ex['instruction']}") + lines.append(f"[输入评论]: \"{ex['review']}\"") + lines.append(f"[输出 JSON]:\n{output_json}\n") + return "\n".join(lines).strip() + + +def _extraction_vars(product_name: str, schema: Dict[str, Any]) -> Dict[str, Any]: + return { + "product_name": product_name, + "categories_literal": _categories_literal(schema), + "sentiments_literal": _sentiments_literal(schema), + "examples_str": format_examples(), + } + + +def build_batch_extraction_system( + industry: str, + product_name: str, + *, + n_keys: int, + keys_literal: str, +) -> str: + schema = get_schema() + common = _extraction_vars(product_name, schema) + parts = [ + render("extraction/system_intro.md", industry=industry, product_name=product_name), + render( + "extraction/batch_output_format.md", + n_keys=n_keys, + keys_literal=keys_literal, + ), + render("extraction/field_rules.md", **common), + render("extraction/format_constraints_batch.md"), + render("extraction/filter_irrelevant.md", product_name=product_name), + render( + "extraction/examples_section.md", + product_name=product_name, + examples_str=common["examples_str"], + ), + render("extraction/examples_footer_batch.md"), + ] + return "\n\n".join(p for p in parts if p.strip()) + + +def build_batch_extraction_user( + *, + n_keys: int, + keys_literal: str, + tagged_input: str, +) -> str: + return render( + "extraction/user_batch.md", + n_keys=n_keys, + keys_literal=keys_literal, + tagged_input=tagged_input, + ) + + +def build_single_extraction_template(industry: str, product_name: str) -> str: + schema = get_schema() + common = _extraction_vars(product_name, schema) + parts = [ + render("extraction/system_intro.md", industry=industry, product_name=product_name), + render("extraction/field_rules.md", **common), + render("extraction/format_constraints_single.md"), + render("extraction/filter_irrelevant.md", product_name=product_name), + render( + "extraction/examples_section.md", + product_name=product_name, + examples_str=common["examples_str"], + ), + render("extraction/user_single_tail.md"), + ] + return "\n\n".join(p for p in parts if p.strip()) + + +def _report_marker_kwargs( + config: Dict[str, Any], + schema: Dict[str, Any], +) -> Dict[str, Any]: + markers = list(report_markers(schema)) + wf = config.get("word_freq", {}) + rp = config.get("report", {}) + return { + "marker_word_zh": markers[0], + "marker_word_category": markers[1], + "marker_cluster_names": markers[2], + "marker_report_html": markers[3], + "wordcloud_top_n": wf.get("wordcloud_top_n", 180), + "word_category_classify_n": wf.get("word_category_classify_n", 180), + "outlier_label_zh": rp.get("outlier_label_zh", "未归类"), + } + + +def build_report_system() -> str: + return render("report/system.md") + + +def build_report_analysis_requirements( + *, + product_name: str, + stage_1_audience: str, +) -> str: + return render( + "report/analysis_requirements.md", + product_name=product_name, + stage_1_audience=stage_1_audience, + ) + + +def build_report_output_format() -> str: + return render("report/output_format.md", **_report_marker_kwargs(get_config(), get_schema())) + + +def _word_category_definitions() -> str: + return read_text("word_freq/category_definitions.md").strip() + + +def build_report_json_markers(*, cats_literal: str) -> str: + return render( + "report/json_markers.md", + cats_literal=cats_literal, + category_definitions=_word_category_definitions(), + **_report_marker_kwargs(get_config(), get_schema()), + ) + + +def build_report_correction_message(*, err_block: str, raw_body: str, cats_literal: str) -> str: + return render( + "report/correction.md", + err_block=err_block, + raw_body=raw_body, + cats_literal=cats_literal, + **_report_marker_kwargs(get_config(), get_schema()), + ) + + +def build_word_assign_prompts( + *, + industry: str, + product_name: str, + word_lines: str, + cats_literal: str, +) -> Tuple[str, str]: + system = render( + "word_freq/assign_system.md", + cats_literal=cats_literal, + category_definitions=_word_category_definitions(), + ) + user = render( + "word_freq/assign_user.md", + industry=industry, + product_name=product_name, + word_lines=word_lines, + cats_literal=cats_literal, + ) + return system, user + + +def build_category_analysis_prompts( + *, + industry: str, + product_name: str, + category_lines: str, + cats_literal: str, +) -> Tuple[str, str]: + system = render("word_freq/analysis_system.md") + user = render( + "word_freq/analysis_user.md", + industry=industry, + product_name=product_name, + category_lines=category_lines, + cats_literal=cats_literal, + ) + return system, user + + +def sync_voc_report_constants(voc_report: Any) -> None: + """从 config.yaml / schema.yaml 同步 voc_report 模块级常量（每次 generate_report 前调用）。""" + cfg = get_config() + sch = get_schema() + wf = cfg.get("word_freq", {}) + rp = cfg.get("report", {}) + + voc_report.WORDCLOUD_TOP_N = int(wf.get("wordcloud_top_n", 180)) + voc_report.WORD_FREQ_TABLE_N = int(wf.get("word_freq_table_n", 180)) + voc_report.WORD_CATEGORY_CLASSIFY_N = int(wf.get("word_category_classify_n", 180)) + voc_report.WORD_FREQ_PAGE_SIZE = int(wf.get("word_freq_page_size", 30)) + voc_report.WORD_FREQ_PAGES = int(wf.get("word_freq_pages", 6)) + voc_report.WORD_CATEGORY_ASSIGN_BATCH_SIZE = int( + wf.get("word_category_assign_batch_size", 40) + ) + voc_report.WORD_CATEGORY_ANALYSIS_MAX_WORDS = int( + wf.get("word_category_analysis_max_words", 25) + ) + voc_report.OUTLIER_LABEL_ZH = str(rp.get("outlier_label_zh", "未归类")) + voc_report.MAX_PHRASES_PER_CLUSTER = int(rp.get("max_phrases_per_cluster", 15)) + voc_report.WORD_CATEGORIES = word_categories(cfg) + voc_report.REPORT_MARKERS = report_markers(sch) + + +def validate_prompt_files() -> List[str]: + """检查外置文件能否正常加载与渲染；返回错误列表（空=通过）。""" + errors: List[str] = [] + try: + cfg = get_config() + sch = get_schema() + if not word_categories(cfg): + errors.append("config.yaml: word_categories 为空") + if len(report_markers(sch)) != 4: + errors.append("schema.yaml: report.markers 须为 4 项") + if not product_feedback_categories(sch): + errors.append("schema.yaml: extraction.product_feedback_categories 为空") + format_examples() + build_batch_extraction_system( + "Test Industry", + "Test Product", + n_keys=2, + keys_literal='"C1", "C2"', + ) + build_batch_extraction_user(n_keys=2, keys_literal='"C1", "C2"', tagged_input="[C1] hi") + build_single_extraction_template("Test", "Product") + build_report_system() + build_report_analysis_requirements( + product_name="P", + stage_1_audience="1_audience", + ) + build_report_output_format() + build_report_json_markers(cats_literal="成分/原料") + build_word_assign_prompts( + industry="I", + product_name="P", + word_lines="dog\t10", + cats_literal="成分/原料", + ) + build_category_analysis_prompts( + industry="I", + product_name="P", + category_lines="- 成分/原料：a(1)", + cats_literal="成分/原料", + ) + except Exception as e: + errors.append(f"渲染失败: {e}") + return errors diff --git a/prompts/report/analysis_requirements.md b/prompts/report/analysis_requirements.md new file mode 100644 index 0000000..5b29c82 --- /dev/null +++ b/prompts/report/analysis_requirements.md @@ -0,0 +1,8 @@ +**分析要求：** +1. **一、摘要**：四条——核心受众与场景、最核心的用户需求、显著的产品反馈特征（正/负/客观）、提炼核心改进建议与机会（一句话）。 +2. **二、{product_name}评论分析**：严格按 HTML 模版层级填写。 + - **（一）受众画像与分析**：1.受众画像 —（1）受众群体特征、（2）受众的主要需求（仅用 {stage_1_audience} 各簇）； + 2.分受众分析 — 对 top2 受众分别写（1）用户需求（2）正面（3）负面（4）客观（仅用对应 2a/2b stage）。 + - **（二）全部用户需求与产品反馈**：1.全部受众需求（3a）、2.正面、3.负面、4.客观（3b 三档）； + 每个
    只收纳对应 stage 的簇；占比=结构化短语数÷a；同 ul 内按短语数降序。正文禁用「观点 / 评价 / 痛点」。 +3. **三、改进建议与机会**：保持模版

    与四条
      结构；建议须可执行，第 4 条单独写可放大的产品/市场机会；覆盖未满足需求、负面反馈与客观描述中的风险，勿复述本条款文字。 diff --git a/prompts/report/correction.md b/prompts/report/correction.md new file mode 100644 index 0000000..cf3a283 --- /dev/null +++ b/prompts/report/correction.md @@ -0,0 +1,15 @@ +你上一次输出未通过程序自动校验，请修正后**重新输出完整的四段标记内容**（四段都要给出，即使某段上次已正确也请原样附上）。 + +【校验错误】 +{err_block} + +【修正要点】 +- 四个标记必须按顺序出现：{marker_word_zh}、{marker_word_category}、{marker_cluster_names}、{marker_report_html}（JSON 在前，HTML 最后） +- JSON 段必须是严格合法 JSON：键与字符串均用英文双引号；例如 "words":["dog","cat"]，禁止写成 "words:["dog"] 或 words:[ +- 不要用 Markdown 代码围栏包裹 JSON；WORD_CATEGORY_JSON 的键仅限：{cats_literal} +- WORD_CATEGORY_JSON 必须是**一个** JSON 对象，words 仅来自 Top{word_category_classify_n}；逐词尽量归类、不强制每类凑满；禁止「其他」键；禁止多段 `{{"某类":[...]}}, "另一类":[...]` +- 务必输出完整的 {marker_cluster_names} 段 +- REPORT_HTML 为完整 ... 文档，放在最后一段 + +【你上一次的完整输出（请对照修改）】 +{raw_body} diff --git a/prompts/report/json_markers.md b/prompts/report/json_markers.md new file mode 100644 index 0000000..04e0df1 --- /dev/null +++ b/prompts/report/json_markers.md @@ -0,0 +1,24 @@ +请严格按以下四段标记输出（标记外不要有任何文字；**顺序不可调换**）： + +{marker_word_zh} +一行合法 JSON 对象：Top{wordcloud_top_n} 英文词 -> 中文，如 {{"dog":"狗","cat":"猫"}}。 + +{marker_word_category} +**仅一个** JSON 对象（禁止拆成多段 `{{"某类":[...]}}, "另一类":[...]`）。 +键仅限：{cats_literal}；每类值为 {{"words":["英文词",...],"analysis":"中文解读"}} + +{category_definitions} + +要求： +- 对「词频分类词表 Top{word_category_classify_n}」**逐词**判断是否可归类；仅语义明确贴合时才写入对应类 words，一词可多类 +- **不强制**每类都有词、**不强制**覆盖全部词；无法判断或仅为噪声的词**不要**硬塞进任何类（程序会对未分类词二次尝试） +- words 必须全部来自 Top{word_category_classify_n} 列表；每类按词频降序；每类须写 analysis（基于已归入的词总结）；禁止「其他」键 + +{marker_cluster_names} +一行合法 JSON 对象：键为聚类原始 id（必须与上文 id 严格一致），值为中文簇名；整段仅一对最外层花括号 `{{...}}`，勿多写结尾 `}}`。 +簇标签 -1 统一命名为「{outlier_label_zh}」。 +命名规则：1_audience、3a_pain_global、3b_aspect_opinion_* 写完整业务簇名； +2a/2b 只写子主题（程序拼接受众名）；2b 子主题禁止含情感词。 + +{marker_report_html} +（最后输出）完整 HTML 文档，结构遵循上文模版；勿用 Markdown 代码围栏包裹。 diff --git a/prompts/report/output_format.md b/prompts/report/output_format.md new file mode 100644 index 0000000..ee009b9 --- /dev/null +++ b/prompts/report/output_format.md @@ -0,0 +1,8 @@ +**输出格式（必须严格遵守）：** +1. 只允许输出标准 HTML，不允许 Markdown。 +2. {marker_report_html} 内必须是完整 HTML 文档，包含 、、；可保留模版中的 ，浏览器不会显示。 +3. 允许标签：

        1. ；列表项格式须为 纯中文簇名 (XX.X%)：洞察。 +4. 禁止 Markdown（#、**、``` 等）及 等未列出的标签。 +5. 【红线】正文可见文字中禁止出现 1_audience|2、2a_pain_audience_c2|0、stage=、audience_c 等任何机器标识；仅 CLUSTER_NAMES JSON 键可保留原始 id。 +6. 不要输出解释性前后缀；REPORT_HTML 段内只放 HTML 文档本体。 +7. 四段标记顺序固定：先三个 JSON 段（WORD_ZH → WORD_CATEGORY → CLUSTER_NAMES），**最后**输出 REPORT_HTML，避免长 HTML 导致 JSON 被截断。 diff --git a/prompts/report/system.md b/prompts/report/system.md new file mode 100644 index 0000000..9cb3c67 --- /dev/null +++ b/prompts/report/system.md @@ -0,0 +1,4 @@ +你是一位非常资深且专业的亚马逊美国站运营师，正在撰写面向业务高管的 VOC 改进报告。 +报告必须专业、可读，正文中绝对禁止出现原始聚类 ID、stage 代码名或簇标签编号。 +展示用语统一：用户需求、产品反馈、产品客观描述；禁止观点、评价、痛点等旧称。 +你必须严格遵守 HTML 输出规范，禁止 Markdown。 diff --git a/prompts/schema.yaml b/prompts/schema.yaml new file mode 100644 index 0000000..9795be0 --- /dev/null +++ b/prompts/schema.yaml @@ -0,0 +1,31 @@ +# 🔒 程序锁定区：修改前请与开发确认，否则可能导致 JSON 解析失败 +# 字段名、英文 category、输出标记等与下游 SQLite / 聚类 / 报告解析绑定 + +extraction: + root_fields: + - audience + - pain_points + - product_feedback + product_feedback_fields: + - aspect + - opinion + - sentiment + - category + product_feedback_categories: + - Quality + - Function + - Appearance + - Logistics + - Customer Service + - Price + sentiments: + - Positive + - Negative + - Neutral + +report: + markers: + - "===WORD_ZH_JSON===" + - "===WORD_CATEGORY_JSON===" + - "===CLUSTER_NAMES_JSON===" + - "===REPORT_HTML===" diff --git a/prompts/smoke.py b/prompts/smoke.py new file mode 100644 index 0000000..56d82ce --- /dev/null +++ b/prompts/smoke.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +Prompt 验收脚本（修改 prompts/ 后运行）。 + + python3 prompts/smoke.py # 仅校验文件加载与渲染（无需 API Key） + python3 prompts/smoke.py --live # 调用模型跑 smoke/reviews.yaml（需 DASHSCOPE_API_KEY） +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +import yaml + +from prompts.loader import ( + build_batch_extraction_system, + build_batch_extraction_user, + get_schema, + product_feedback_categories, + validate_prompt_files, +) + + +def _load_smoke_reviews() -> dict: + path = Path(__file__).resolve().parent / "smoke/reviews.yaml" + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def run_dry() -> int: + errors = validate_prompt_files() + if errors: + print("❌ Prompt 文件校验失败：") + for e in errors: + print(f" - {e}") + return 1 + smoke = _load_smoke_reviews() + industry = smoke.get("industry", "Test") + product = smoke.get("product_name", "Test") + reviews = smoke.get("reviews") or [] + keys = [str(r["id"]) for r in reviews] + tagged = "\n".join(f"[{r['id']}] {r['text']}" for r in reviews) + keys_literal = ", ".join(json.dumps(k) for k in keys) + system = build_batch_extraction_system( + industry, + product, + n_keys=len(keys), + keys_literal=keys_literal, + ) + user = build_batch_extraction_user( + n_keys=len(keys), + keys_literal=keys_literal, + tagged_input=tagged, + ) + print("✅ 外置 Prompt 加载与渲染通过") + print(f" 示例评论数: {len(reviews)}") + print(f" system 长度: {len(system)} 字符") + print(f" user 长度: {len(user)} 字符") + return 0 + + +def run_live() -> int: + rc = run_dry() + if rc != 0: + return rc + try: + from 结构化_server import _call_dashscope_chat, _validate_extraction_strict + except ImportError as e: + print(f"❌ 无法导入结构化_server: {e}") + return 1 + + smoke = _load_smoke_reviews() + industry = smoke["industry"] + product = smoke["product_name"] + reviews = smoke["reviews"] + keys = [str(r["id"]) for r in reviews] + tagged = "\n".join(f"[{r['id']}] {r['text']}" for r in reviews) + keys_literal = ", ".join(json.dumps(k) for k in keys) + system = build_batch_extraction_system( + industry, + product, + n_keys=len(keys), + keys_literal=keys_literal, + ) + user = build_batch_extraction_user( + n_keys=len(keys), + keys_literal=keys_literal, + tagged_input=tagged, + ) + print("⏳ 调用模型进行 smoke 结构化…") + try: + raw = _call_dashscope_chat( + [{"role": "system", "content": system}, {"role": "user", "content": user}], + max_tokens=8192, + ) + except Exception as e: + print(f"❌ 模型调用失败: {e}") + return 1 + + if not isinstance(raw, dict): + print(f"❌ 期望 JSON 对象，得到: {type(raw).__name__}") + return 1 + + ok = 0 + for k in keys: + if k not in raw: + print(f"❌ 缺少键 {k}") + continue + try: + _validate_extraction_strict(raw[k], k) + ok += 1 + print(f"✅ {k} 校验通过") + except ValueError as e: + print(f"❌ {k} 校验失败: {e}") + + schema_cats = product_feedback_categories(get_schema()) + print(f" 锁定 category 枚举: {', '.join(sorted(schema_cats))}") + return 0 if ok == len(keys) else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Prompt smoke 验收") + parser.add_argument( + "--live", + action="store_true", + help="调用 DashScope 跑 smoke 评论（需 API Key）", + ) + args = parser.parse_args() + return run_live() if args.live else run_dry() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/prompts/smoke/reviews.yaml b/prompts/smoke/reviews.yaml new file mode 100644 index 0000000..d8f927f --- /dev/null +++ b/prompts/smoke/reviews.yaml @@ -0,0 +1,12 @@ +# Smoke 验收用评论（修改 prompt 后运行 python prompts/smoke.py 验证） + +industry: "Pet supplements" +product_name: "Turkey tail mushroom for dogs" + +reviews: + - id: C1 + text: "My senior dog has weak immunity. This turkey tail powder mixes easily and he eats it without fuss. Shipping was fast." + - id: C2 + text: "Good quality mushrooms but the bag seal broke and powder spilled everywhere. Too expensive for the amount." + - id: C3 + text: "Random spam about phone cases, not related to dog supplements at all." diff --git a/prompts/word_freq/analysis_system.md b/prompts/word_freq/analysis_system.md new file mode 100644 index 0000000..dbb186b --- /dev/null +++ b/prompts/word_freq/analysis_system.md @@ -0,0 +1 @@ +你是亚马逊 VOC 词频分析专家。根据每类已归入的英文高频词，写一两句简洁的中文解读（说明该类词反映的用户关注点，勿罗列词表）。 diff --git a/prompts/word_freq/analysis_user.md b/prompts/word_freq/analysis_user.md new file mode 100644 index 0000000..0c3dab6 --- /dev/null +++ b/prompts/word_freq/analysis_user.md @@ -0,0 +1,9 @@ +行业：{industry} +产品：{product_name} + +【须撰写 analysis 的类别及已归入词（word(count)）】 +{category_lines} + +请输出**仅一个** JSON 对象：键为类别名（仅限：{cats_literal}），值为 1～2 句中文解读字符串。 +- 只输出上述列出的类别；不要 Markdown 围栏 +- 示例：{{"成分/原料":"用户高度关注火鸡尾等药用真菌原料…","受众/使用对象":"以老年犬与患病犬为主…"}} diff --git a/prompts/word_freq/assign_system.md b/prompts/word_freq/assign_system.md new file mode 100644 index 0000000..86401d0 --- /dev/null +++ b/prompts/word_freq/assign_system.md @@ -0,0 +1,4 @@ +你是亚马逊 VOC 词频分类专家。对给定英文词**逐一**判断是否属于以下类别之一或多个：{cats_literal}。 +拿不准、仅为噪声或与品类无关的词返回空数组 []，不要硬分类。 + +{category_definitions} diff --git a/prompts/word_freq/assign_user.md b/prompts/word_freq/assign_user.md new file mode 100644 index 0000000..4b33d0f --- /dev/null +++ b/prompts/word_freq/assign_user.md @@ -0,0 +1,12 @@ +行业：{industry} +产品：{product_name} + +【本批待分类词（word\tcount，拼写须原样作为 JSON 键）】 +{word_lines} + +请输出**仅一个** JSON 对象：键为本批每个英文词（与上表拼写完全一致），值为类别名数组。 +- 值仅限：{cats_literal} 中的 0～多个类别；一词可多类 +- 无法明确归类时值为 []，不要编造类别 +- 不要输出「其他」；不要 Markdown 围栏 + +示例：{{"dog":["受众/使用对象"],"turmeric":["成分/原料","功效/功能"],"powder":["剂型"],"xyz":[]}} diff --git a/prompts/word_freq/category_definitions.md b/prompts/word_freq/category_definitions.md new file mode 100644 index 0000000..b895bec --- /dev/null +++ b/prompts/word_freq/category_definitions.md @@ -0,0 +1,10 @@ +## 词频各类定义（分类时严格遵守） + +- **成分/原料**：核心成分、原料、主材名词（如 turmeric、mushroom、beta-glucan）；定语或形容词禁止写入此类（如natural、organic 等）。 +- **剂型**：产品的物理形态与给药/食用形式（如 powder、capsule、chew、tablet、soft chew、liquid、treat、granule、sachet、drop）。与成分本身、功效描述区分；定语或形容词不归此类。 +- **受众/使用对象**：使用者或购买对象（dog、puppy、senior cat…）；定语或形容词不归此类。 +- **功效/功能**：作用、效果、功能（immunity、digestion、joint support…），不是成分名或剂型。 +- **需求/场景**：使用场景、未满足需求（allergy、anxiety、appetite…）。 +- **品质/体验**：口感、气味、质地、纯度等体验词（taste、smell、organic、natural…）。 +- **价格/价值**：价格、性价比相关（price、worth、expensive…）。 +- **物流/包装**：包装、运输、到货（packaging、shipping、box…）。 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..154f959 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "voc-llm-structured" +version = "0.1.0" +description = "Structured VOC review extraction via DashScope qwen3.6-flash" +requires-python = ">=3.10" +dependencies = [ + "mcp>=1.2.0", + "openai>=1.40.0", + "pyyaml>=6.0", +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..431d4da --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +openai>=1.40.0 +numpy>=1.24.0 +umap-learn>=0.5.5 +hdbscan>=0.8.33 +scikit-learn>=1.3.0 +spacy>=3.7.0 +pyyaml>=6.0 +jieba>=0.42.1 diff --git a/voc_report.py b/voc_report.py new file mode 100644 index 0000000..84caa38 --- /dev/null +++ b/voc_report.py @@ -0,0 +1,2415 @@ +""" +基于聚类结果与词频 CSV 生成 output/voc_report.html（词云 + 词频 + AI 报告 + 各簇表述）。 + +用法:: + python3 main_voc分析.py + python3 main_voc分析.py --only-step 7 --industry "..." --product "..." + # 仅纳入本 stage 内评论占比 ≥10% 的簇： + python3 main_voc分析.py --only-step 7 --filter-small-clusters ... + # 调试：保存报告 LLM 完整原文到 output/report_llm_raw.txt + python3 main_voc分析.py --only-step 7 --save-llm-raw --industry "..." --product "..." +""" +from __future__ import annotations + +import csv +import html +import json +import logging +import os +import re +import sqlite3 +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Sequence, Set, Tuple + +from openai import OpenAI + +from prompts.loader import ( + build_category_analysis_prompts, + build_report_analysis_requirements, + build_report_correction_message, + build_report_json_markers, + build_report_output_format, + build_report_system, + build_word_assign_prompts, + sync_voc_report_constants, +) + +logger = logging.getLogger("voc_report") + +PROJECT_ROOT = Path(__file__).resolve().parent +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +STRUCTURED_APPENDIX_SAMPLE_N = 15 +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" +MODEL_NAME = os.environ.get("DASHSCOPE_MODEL", "qwen3.6-flash").strip() + +WORDCLOUD_TOP_N = 180 +WORD_FREQ_TABLE_N = 180 # 页面词频表、排名展示上限 +WORD_CATEGORY_CLASSIFY_N = 180 # LLM 词频分类词表范围（≥ 词表展示数） +WORD_FREQ_PAGE_SIZE = 30 +WORD_FREQ_PAGES = 6 +# 词云字号映射：0.5 次幂缓解「超高频词过大、长尾过小」 +WORDCLOUD_SIZE_POWER = 0.5 +WORDCLOUD_SIZE_MIN = 14 +WORDCLOUD_SIZE_MAX = 96 +MAX_PHRASES_PER_CLUSTER = 15 +# 启用簇筛选时的默认阈值（本 stage 内去重评论占比） +DEFAULT_CLUSTER_MIN_REVIEW_RATIO = 0.10 +OUTLIER_LABEL_ZH = "未归类" + +_STAGE_3B_TITLES: Dict[str, str] = { + "3b_aspect_opinion_positive": "全量正面产品反馈", + "3b_aspect_opinion_negative": "全量负面产品反馈", + "3b_aspect_opinion_neutral": "全量产品客观描述", +} +_STAGE_2B_BLOCK_TITLE: Dict[str, str] = { + "positive": "正面产品反馈", + "negative": "负面产品反馈", + "neutral": "产品客观描述", +} +# 词频分类展示名迁移（兼容旧 LLM 输出） +_WORD_CATEGORY_DISPLAY_ALIASES: Dict[str, str] = { + "痛点/场景": "需求/场景", +} +_STAGE_SORT_KEY: Dict[str, int] = { + "3a_pain_global": 0, + "3b_aspect_opinion_positive": 1, + "3b_aspect_opinion_negative": 2, + "3b_aspect_opinion_neutral": 3, +} +# 与 聚类.py 流程一致的 stage 前缀（用于模版与映射说明） +STAGE_1_AUDIENCE = "1_audience" +STAGE_2A_PAIN_PREFIX = "2a_pain_audience_c" +STAGE_2B_AO_PREFIX = "2b_aspect_opinion_" +STAGE_3A_PAIN = "3a_pain_global" +STAGE_3B_NEGATIVE = "3b_aspect_opinion_negative" +STAGE_3B_AO_PREFIX = "3b_aspect_opinion_" +# 底部「聚类效果验证」固定展示的 stage（写死，不含 -1 离群簇） +PHRASE_APPENDIX_STAGES: Tuple[str, ...] = (STAGE_3A_PAIN, STAGE_3B_NEGATIVE) +STEP2_TOP2_META_STAGE = "step2_filter" +_AUDIENCE_RANK_ZH: Tuple[str, ...] = ("第一", "第二") + +WORD_CATEGORIES: Tuple[str, ...] = ( + "成分/原料", + "剂型", + "受众/使用对象", + "功效/功能", + "需求/场景", + "品质/体验", + "价格/价值", + "物流/包装", +) + +# 词云按分类配色（与 WORD_CATEGORIES 顺序一一对应） +CATEGORY_COLORS: Dict[str, str] = { + "成分/原料": "#dc2626", + "剂型": "#db2777", + "受众/使用对象": "#2563eb", + "功效/功能": "#16a34a", + "需求/场景": "#ea580c", + "品质/体验": "#7c3aed", + "价格/价值": "#ca8a04", + "物流/包装": "#0891b2", +} +WORDCLOUD_UNCATEGORIZED_COLOR = "#9ca3af" + +TRANSLATE_CHARS_PER_REQUEST = 200_000 +TRANSLATE_MAX_OUTPUT_TOKENS = 65536 +REPORT_MAX_OUTPUT_TOKENS = 65536 +TRANSLATE_LLM_TIMEOUT_SEC = 600.0 +REPORT_LLM_TIMEOUT_SEC = 600.0 +REPORT_PARSE_MAX_RETRIES = 2 +# JSON 段放前、HTML 放后：输出被 max_tokens 截断时优先保留可解析的 JSON +REPORT_MARKERS: Tuple[str, ...] = ( + "===WORD_ZH_JSON===", + "===WORD_CATEGORY_JSON===", + "===CLUSTER_NAMES_JSON===", + "===REPORT_HTML===", +) +WORD_CATEGORY_ASSIGN_BATCH_SIZE = 40 +WORD_CATEGORY_ASSIGN_MAX_TOKENS = 8192 +WORD_CATEGORY_ANALYSIS_MAX_WORDS = 25 # 生成 analysis 时每类最多展示的词条数 +WORD_CATEGORY_ANALYSIS_MAX_TOKENS = 4096 + + +@dataclass +class ClusterBundle: + section: str + stage: str + stage_title_zh: str + cluster_label: int + cluster_title_zh: str + anchor_id: str + review_count: int + phrase_count: int + ratio: float # 占本 stage 去重评论比（小簇过滤用） + ratio_global: float # 短语数 / 全部评论 a（报告洞察用） + embed_texts: List[str] = field(default_factory=list) + embed_texts_zh: List[str] = field(default_factory=list) + + +def _resolve_api_key() -> str: + v = os.environ.get("DASHSCOPE_API_KEY", "").strip() + if v: + return v + fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip() + if fp: + p = Path(fp).expanduser() + if p.is_file(): + return p.read_text(encoding="utf-8").strip().strip('"').strip("'") + local = PROJECT_ROOT / ".dashscope_key" + if local.is_file(): + return local.read_text(encoding="utf-8").strip().strip('"').strip("'") + return "" + + +def _strip_think(text: str) -> str: + if not text: + return text + text = re.sub( + r"[\s\S]*?", "", text, flags=re.IGNORECASE + ) + text = re.sub(r"", "", text, flags=re.IGNORECASE) + return text.strip() + + +def _call_llm_messages( + messages: Sequence[Dict[str, str]], + api_key: str, + *, + temperature: float = 0.3, + max_tokens: int = 16384, + timeout: float = 300.0, +) -> str: + client = OpenAI( + api_key=api_key, base_url=DASHSCOPE_BASE_URL, timeout=timeout + ) + extra_body: Dict[str, Any] = {} + if MODEL_NAME.lower().startswith(("qwen3.6", "qwen3.5", "qwen3")): + extra_body["enable_thinking"] = False + resp = client.chat.completions.create( + model=MODEL_NAME, + messages=list(messages), + temperature=temperature, + max_tokens=max_tokens, + **({"extra_body": extra_body} if extra_body else {}), + ) + msg = resp.choices[0].message + text = msg.content or getattr(msg, "reasoning_content", None) or "" + if not text.strip(): + raise RuntimeError("LLM 返回为空") + return _strip_think(text) + + +def _call_llm( + system: str, + user: str, + api_key: str, + *, + temperature: float = 0.3, + max_tokens: int = 16384, + timeout: float = 300.0, +) -> str: + return _call_llm_messages( + [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + api_key, + temperature=temperature, + max_tokens=max_tokens, + timeout=timeout, + ) + + +def _load_review_count(csv_path: Path) -> int: + n = 0 + with csv_path.open(encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + if (row.get("content") or "").strip(): + n += 1 + return n + + +def _load_review_texts_by_source_row(csv_path: Path) -> Dict[int, str]: + """与 结构化_server.load_reviews_from_file 一致：第 1 条数据行 source_row=1。""" + out: Dict[int, str] = {} + with csv_path.open(encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for i, row in enumerate(reader, start=1): + text = (row.get("content") or "").strip() + if text: + out[i] = text + return out + + +def _latest_structured_job_id(conn: sqlite3.Connection) -> int: + row = conn.execute( + "SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1" + ).fetchone() + if not row: + raise RuntimeError("voc_structured.sqlite 中无结构化任务") + return int(row[0]) + + +def _load_structured_samples( + structured_db: Path, + *, + job_id: int | None = None, + limit: int = STRUCTURED_APPENDIX_SAMPLE_N, +) -> List[Tuple[int, Dict[str, Any]]]: + conn = sqlite3.connect(structured_db) + try: + jid = job_id if job_id is not None else _latest_structured_job_id(conn) + cur = conn.execute( + """ + SELECT source_row, extraction_json + FROM comment_extractions + WHERE job_id = ? + ORDER BY source_row + LIMIT ? + """, + (jid, limit), + ) + return [(int(sr), json.loads(js)) for sr, js in cur.fetchall()] + finally: + conn.close() + + +def _extraction_to_display_zh(ext: Dict[str, Any]) -> Dict[str, Any]: + """将库内英文字段转为报告展示用中文键（与业务阅读一致）。""" + feedback: List[Dict[str, str]] = [] + for item in ext.get("product_feedback") or []: + if not isinstance(item, dict): + continue + feedback.append( + { + "方面": str(item.get("aspect", "")).strip(), + "观点": str(item.get("opinion", "")).strip(), + "态度": str(item.get("sentiment", "")).strip(), + "类别标签": str(item.get("category", "")).strip(), + } + ) + pains = ext.get("pain_points") or [] + if not isinstance(pains, list): + pains = [] + return { + "受众": str(ext.get("audience", "")).strip(), + "需求/痛点": [str(p).strip() for p in pains if str(p).strip()], + "产品反馈": feedback, + } + + +def _build_structured_appendix_samples( + *, + structured_db: Path, + cleaned_csv: Path, + job_id: int | None = None, + limit: int = STRUCTURED_APPENDIX_SAMPLE_N, +) -> List[Tuple[int, str, Dict[str, Any]]]: + if not structured_db.is_file(): + logger.warning("未找到结构化库 %s，跳过结构化提取效果验证", structured_db) + return [] + texts = _load_review_texts_by_source_row(cleaned_csv) + try: + rows = _load_structured_samples(structured_db, job_id=job_id, limit=limit) + except (RuntimeError, sqlite3.Error) as e: + logger.warning("读取结构化样本失败: %s", e) + return [] + out: List[Tuple[int, str, Dict[str, Any]]] = [] + for sr, ext in rows: + content = texts.get(sr, "").strip() or "（原文缺失）" + out.append((sr, content, _extraction_to_display_zh(ext))) + return out + + +def _load_word_freq(path: Path, top_n: int) -> List[Tuple[str, int]]: + rows: List[Tuple[str, int]] = [] + with path.open(encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + w = (row.get("word") or "").strip() + if not w: + continue + rows.append((w, int(row.get("count") or 0))) + rows.sort(key=lambda x: x[1], reverse=True) + return rows[:top_n] + + +def _stage_section(stage: str) -> str: + if stage == "1_audience" or stage.startswith("2a_") or stage.startswith("2b_"): + return "part1" + return "part2" + + +def _audience_cluster_from_stage(stage: str) -> int | None: + if stage.startswith("2a_pain_audience_c"): + try: + return int(stage.split("_c")[-1]) + except ValueError: + return None + m = re.match( + r"2b_aspect_opinion_(?:positive|negative|neutral)_audience_c(\d+)$", + stage, + ) + if m: + return int(m.group(1)) + return None + + +def _sentiment_suffix_from_2b_stage(stage: str) -> str | None: + m = re.match( + r"2b_aspect_opinion_(positive|negative|neutral)_audience_c\d+$", + stage, + ) + return m.group(1) if m else None + + +def _stage_title_zh(stage: str, audience_names: Dict[int, str]) -> str: + if stage == "1_audience": + return "受众画像聚类（阶段一）" + if stage in _STAGE_3B_TITLES: + return _STAGE_3B_TITLES[stage] + aud_c = _audience_cluster_from_stage(stage) + if aud_c is not None: + aud_name = audience_names.get(aud_c, f"受众{aud_c}") + if stage.startswith("2a_"): + return f"{aud_name} · 用户需求（阶段二）" + sent = _sentiment_suffix_from_2b_stage(stage) + if sent: + label = _STAGE_2B_BLOCK_TITLE.get(sent, sent) + return f"{aud_name} · {label}（阶段二）" + return f"{aud_name} · 产品反馈（阶段二）" + if stage == "3a_pain_global": + return "全量用户需求（阶段三）" + return stage + + +def _phrase_appendix_stage_title(stage: str) -> str: + if stage == STAGE_3A_PAIN: + return "全量用户需求（阶段三）" + if stage in _STAGE_3B_TITLES: + return _STAGE_3B_TITLES[stage] + return stage + + +def _bundles_for_phrase_appendix(bundles: List[ClusterBundle]) -> List[ClusterBundle]: + return [ + b + for b in bundles + if b.stage in PHRASE_APPENDIX_STAGES and b.cluster_label != -1 + ] + + +def _bundle_sort_key(b: ClusterBundle) -> Tuple[int, int, str, int]: + sec = 0 if b.section == "part1" else 1 + stage_ord = _STAGE_SORT_KEY.get(b.stage, 0) + lab = -1 if b.cluster_label == -1 else b.cluster_label + return (sec, stage_ord, b.stage, lab) + + +def _cluster_key(stage: str, label: int) -> str: + return f"{stage}|{label}" + + +def _anchor_id(stage: str, label: int) -> str: + safe = re.sub(r"[^a-zA-Z0-9_-]", "_", stage) + return f"cluster-{safe}-{label}" + + +def _latest_cluster_run(conn: sqlite3.Connection) -> Tuple[int, int]: + row = conn.execute( + "SELECT id, job_id FROM cluster_runs ORDER BY id DESC LIMIT 1" + ).fetchone() + if not row: + raise RuntimeError("voc_clustering.sqlite 中无聚类记录") + return int(row[0]), int(row[1]) + + +def _load_assignments( + cconn: sqlite3.Connection, run_id: int +) -> List[Tuple[str, int, int, str, int]]: + cur = cconn.execute( + """ + SELECT stage, cluster_label, source_row, embed_text, embedding_item_id + FROM cluster_assignments + WHERE run_id = ? + ORDER BY stage, cluster_label, source_row + """, + (run_id,), + ) + return [ + (str(s), int(lab), int(sr), str(et), int(eid)) + for s, lab, sr, et, eid in cur.fetchall() + ] + + +def build_cluster_bundles( + *, + cluster_db: Path, + cleaned_csv: Path, + embed_db: Path | None = None, # 保留参数兼容 VOC分析，不再使用 + min_cluster_review_ratio: float | None = None, +) -> Tuple[int, List[ClusterBundle]]: + del embed_db + total_reviews = _load_review_count(cleaned_csv) + + cconn = sqlite3.connect(cluster_db) + try: + run_id, _ = _latest_cluster_run(cconn) + raw = _load_assignments(cconn, run_id) + finally: + cconn.close() + + grouped: Dict[Tuple[str, int], List[Tuple[int, str, int]]] = defaultdict(list) + texts_by_key: Dict[Tuple[str, int], List[str]] = defaultdict(list) + stage_source_rows: Dict[str, set[int]] = defaultdict(set) + + for stage, lab, sr, et, _eid in raw: + key = (stage, lab) + grouped[key].append((sr, et, _eid)) + texts_by_key[key].append(et) + stage_source_rows[stage].add(sr) + + bundles: List[ClusterBundle] = [] + for (stage, lab), items in sorted(grouped.items(), key=lambda x: (x[0][0], x[0][1])): + sec = _stage_section(stage) + if sec not in ("part1", "part2"): + continue + unique_srs = sorted({sr for sr, _, _ in items}) + cnt = len(unique_srs) + stage_total = len(stage_source_rows.get(stage, set())) + phrase_cnt = len(items) + stage_ratio = cnt / stage_total if stage_total else 0.0 + ratio = stage_ratio + ratio_global = phrase_cnt / total_reviews if total_reviews else 0.0 + if ( + min_cluster_review_ratio is not None + and stage_total > 0 + and stage_ratio < min_cluster_review_ratio + ): + logger.info( + "报告跳过小簇 %s / %s：%s 条评论 (本步骤 %.1f%% < %.0f%%)", + stage, + lab, + cnt, + stage_ratio * 100, + min_cluster_review_ratio * 100, + ) + continue + text_ctr = Counter(texts_by_key[(stage, lab)]) + reps = [t for t, _ in text_ctr.most_common(MAX_PHRASES_PER_CLUSTER)] + + bundles.append( + ClusterBundle( + section=sec, + stage=stage, + stage_title_zh=stage, + cluster_label=lab, + cluster_title_zh=str(lab), + anchor_id=_anchor_id(stage, lab), + review_count=cnt, + phrase_count=phrase_cnt, + ratio=ratio, + ratio_global=ratio_global, + embed_texts=reps, + ) + ) + + bundles.sort(key=_bundle_sort_key) + return total_reviews, bundles + + +def _apply_cluster_names( + bundles: List[ClusterBundle], names: Dict[str, str] +) -> None: + audience_names: Dict[int, str] = {} + for b in bundles: + if b.stage != "1_audience": + continue + if b.cluster_label == -1: + audience_names[-1] = OUTLIER_LABEL_ZH + b.cluster_title_zh = OUTLIER_LABEL_ZH + continue + key = _cluster_key(b.stage, b.cluster_label) + title = names.get(key, "").strip() or f"受众群体 {b.cluster_label}" + b.cluster_title_zh = title + audience_names[b.cluster_label] = title + + for b in bundles: + b.stage_title_zh = _stage_title_zh(b.stage, audience_names) + if b.stage == "1_audience": + continue + if b.cluster_label == -1: + aud_c = _audience_cluster_from_stage(b.stage) + if aud_c is not None: + aud = audience_names.get(aud_c, f"受众{aud_c}") + b.cluster_title_zh = f"{aud} · {OUTLIER_LABEL_ZH}" + else: + b.cluster_title_zh = OUTLIER_LABEL_ZH + continue + key = _cluster_key(b.stage, b.cluster_label) + sub = names.get(key, "").strip() + aud_c = _audience_cluster_from_stage(b.stage) + if aud_c is not None: + aud = audience_names.get(aud_c, f"受众{aud_c}") + b.cluster_title_zh = f"{aud} · {sub}" if sub else f"{aud} · 子簇 {b.cluster_label}" + else: + b.cluster_title_zh = sub or f"主题簇 {b.cluster_label}" + + +def _parse_numbered_translations( + raw: str, count: int, *, originals: Sequence[str] +) -> List[str]: + out: List[str] = [] + for i in range(count): + m = re.search(rf"\[{i + 1}\]\s*([\s\S]*?)(?=\n\[{i + 2}\]|\Z)", raw) + out.append(m.group(1).strip() if m else originals[i]) + while len(out) < count: + j = len(out) + out.append(originals[j] if j < len(originals) else "") + return out[:count] + + +def _chunk_texts_for_translation(texts: List[str]) -> List[List[str]]: + batches: List[List[str]] = [] + current: List[str] = [] + size = 0 + for text in texts: + block_size = len(text) + 32 + if current and size + block_size > TRANSLATE_CHARS_PER_REQUEST: + batches.append(current) + current = [] + size = 0 + current.append(text) + size += block_size + if current: + batches.append(current) + return batches + + +def _translate_unique_phrases( + phrases: List[str], api_key: str +) -> Dict[str, str]: + unique: List[str] = [] + seen: set[str] = set() + for p in phrases: + t = p.strip() + if not t or t in seen: + continue + seen.add(t) + unique.append(t) + if not unique: + return {} + + merged: Dict[str, str] = {} + batches = _chunk_texts_for_translation(unique) + for bi, batch in enumerate(batches, start=1): + logger.info( + "批量翻译短语（第 %s/%s 批）：%s 条", + bi, + len(batches), + len(batch), + ) + body = "\n\n".join(f"[{i + 1}]\n{t}" for i, t in enumerate(batch)) + user = ( + f"将以下 {len(batch)} 条英文 VOC 结构化短语逐条译为流畅简体中文。\n" + "输出格式：仍用 [1]、[2]… 编号；不要解释、不要 Markdown。\n\n" + + body + ) + raw = _call_llm( + "你是专业英中翻译。按编号输出全部译文，不要遗漏。", + user, + api_key, + temperature=0.2, + max_tokens=TRANSLATE_MAX_OUTPUT_TOKENS, + timeout=TRANSLATE_LLM_TIMEOUT_SEC, + ) + zh_list = _parse_numbered_translations(raw, len(batch), originals=batch) + for en, zh in zip(batch, zh_list): + merged[en] = zh + return merged + + +def _fill_phrase_translations( + bundles: List[ClusterBundle], zh_map: Dict[str, str] +) -> None: + for b in bundles: + b.embed_texts_zh = [zh_map.get(t, t) for t in b.embed_texts] + + +def _bundles_for_prompt(bundles: List[ClusterBundle], section: str) -> str: + """按短语数降序，供 LLM 按规模分析各簇。""" + subset = [b for b in bundles if b.section == section] + subset.sort(key=lambda b: (-b.phrase_count, b.stage, b.cluster_label)) + lines: List[str] = [] + for b in subset: + pct_a = f"{b.ratio_global * 100:.1f}%" + pct_stage = f"{b.ratio * 100:.1f}%" + key = _cluster_key(b.stage, b.cluster_label) + lines.append( + f"- id={key}；阶段={b.stage}；簇标签={b.cluster_label}；" + f"结构化短语数={b.phrase_count}（占全部评论 a 的 {pct_a}）；" + f"去重评论数={b.review_count}（占本阶段 {pct_stage}）；" + f"代表短语：{' | '.join(b.embed_texts)}" + ) + return "\n".join(lines) if lines else "（无）" + + +def _audience_cluster_ids_from_bundles(bundles: Sequence[ClusterBundle]) -> List[int]: + ids: set[int] = set() + for b in bundles: + aud = _audience_cluster_from_stage(b.stage) + if aud is not None: + ids.add(aud) + return sorted(ids) + + +def _load_step2_top2_audience(cluster_db: Path) -> List[int]: + """读取聚类阶段二入选的前两个受众簇标签（与 聚类.py _top2_audience_clusters 一致）。""" + try: + cconn = sqlite3.connect(cluster_db) + try: + run_id, _ = _latest_cluster_run(cconn) + row = cconn.execute( + """ + SELECT meta_json FROM cluster_stage_meta + WHERE run_id = ? AND stage = ? + """, + (run_id, STEP2_TOP2_META_STAGE), + ).fetchone() + finally: + cconn.close() + if not row: + return [] + meta = json.loads(str(row[0])) + raw = meta.get("top2_audience_clusters") + if not isinstance(raw, list): + return [] + return [int(x) for x in raw] + except (json.JSONDecodeError, TypeError, ValueError, sqlite3.Error) as e: + logger.warning("读取 step2 top2 受众簇失败: %s", e) + return [] + + +def _ordered_step2_audiences( + bundles: Sequence[ClusterBundle], top2_from_db: Sequence[int] +) -> List[int]: + """阶段二模版顺序：优先 DB 中 top2 排名，再补齐 bundles 里出现的其它受众簇。""" + in_bundles = set(_audience_cluster_ids_from_bundles(bundles)) + ordered: List[int] = [] + for aud in top2_from_db: + if aud in in_bundles and aud not in ordered: + ordered.append(aud) + for aud in sorted(in_bundles): + if aud not in ordered: + ordered.append(aud) + return ordered + + +def _audience_rank_title(rank_index: int, audience_cluster: int) -> str: + del audience_cluster # 仅用于调用方传入映射注释，勿出现在正文标题 + rank_zh = ( + _AUDIENCE_RANK_ZH[rank_index] + if rank_index < len(_AUDIENCE_RANK_ZH) + else f"第{rank_index + 1}" + ) + return f"{rank_zh}大受众簇" + + +def _stages_in_bundles(bundles: Sequence[ClusterBundle]) -> List[str]: + return sorted({b.stage for b in bundles}) + + +def _cluster_stage_mapping_guide( + bundles: Sequence[ClusterBundle], + *, + top2_audience_clusters: Sequence[int], +) -> str: + """将本批次实际 stage 映射到 HTML 模版小节（与 聚类.py 五段流程一致）。""" + stages = _stages_in_bundles(bundles) + if not stages: + return "（本批次无聚类簇数据）" + aud_ids = _ordered_step2_audiences(bundles, top2_audience_clusters) + top2_txt = ( + "、".join(str(a) for a in top2_audience_clusters) + if top2_audience_clusters + else "（见各 2a/2b stage 后缀）" + ) + lines = [ + "阶段二入选规则（聚类.py）：1_audience 完成后，按各受众簇「去重评论数」降序，" + f"仅对规模最大的前 2 个簇（本批次 top2 簇标签={top2_txt}）分别做：", + " · 2a：簇内 pain_point → 报告中称「用户需求」（每受众 1 个 stage）", + " · 2b：簇内 aspect_opinion → 正面/负面产品反馈、产品客观描述（每受众 3 个 stage）", + "聚类 stage → HTML 模版小节（仅分析下列已出现的 stage；无数据的小节可省略
        2. ）：", + f"- {STAGE_1_AUDIENCE} → 二·（一）·1.受众画像 ·（1）受众群体特征 &（2）受众的主要需求", + ] + for i, aud in enumerate(aud_ids): + rank = _audience_rank_title(i, aud) + lines.append( + f"- {STAGE_2A_PAIN_PREFIX}{aud} → 二·（一）·2.分受众分析 · {rank} ·（1）用户需求（2a）" + ) + for suf, block_title in _STAGE_2B_BLOCK_TITLE.items(): + st = f"{STAGE_2B_AO_PREFIX}{suf}_audience_c{aud}" + slot = {"positive": "（2）", "negative": "（3）", "neutral": "（4）"}[suf] + sent_en = {"positive": "Positive", "negative": "Negative", "neutral": "Neutral"}[ + suf + ] + lines.append( + f"- {st} → 二·（一）·2.分受众分析 · {rank} ·{slot}{block_title}（2b，sentiment={sent_en}）" + ) + lines.append(f"- {STAGE_3A_PAIN} → 二·（二）·1.全部受众需求分析（3a）") + for slot, suf, title in ( + ("2.", "positive", "正面产品反馈"), + ("3.", "negative", "负面产品反馈"), + ("4.", "neutral", "产品客观描述"), + ): + st = f"{STAGE_3B_AO_PREFIX}{suf}" + marker = "（本批次未出现可省略）" if st not in stages else "" + lines.append(f"- {st} → 二·（二）·{slot}{title}（3b）{marker}") + lines.append( + "- 占比口径：括号内 (XX.X%) = 该簇结构化短语数 ÷ a；" + "同一
            内各
          • 按短语数从高到低排列。" + ) + lines.append( + "⚠️ 【排版与格式极度重要警告】：" + "这是面向高管的业务报告，直接暴露原始机器 ID（如 1_audience|2、2a_pain_audience_c2|0）" + "极其不专业且严重影响阅读！正文及任何列表标题中【绝对禁止】出现原始 ID、stage 名、" + "簇标签数字；必须且只能使用提炼后的「纯中文业务簇名」。" + ) + lines.append( + "- 簇 -1 请统一命名并使用「未归类」或「离群反馈」；" + "2a/2b 的 CLUSTER_NAMES JSON 中只写子主题，但在 HTML 正文排版时请自行补全为通顺纯中文" + "（例如：犬类受众-消化不适需求、适口性），不得保留 audience_c2 等代码片段；" + "2b 子主题禁止带「好评/差评/正面/负面」等情感词（情感由 stage 区分）。" + ) + lines.append( + "- 展示用语：全文只用「用户需求 / 产品反馈 / 产品客观描述」，禁止「观点 / 评价 / 痛点」等旧称。" + ) + return "\n".join(lines) + + +def _html_list_placeholder(n: int = 2) -> str: + items = "\n".join( + "
          • [纯中文业务簇名，绝不许带原始ID] " + "(XX.X%)：结合代表短语写深入的业务洞察。
            • " + for _ in range(n) + ) + return f"
              \n{items}\n
            " + + +def _html_step2_intro(top2_audience_clusters: Sequence[int]) -> str: + top2_txt = ( + "、".join(str(a) for a in top2_audience_clusters[:2]) + if top2_audience_clusters + else "见聚类库 step2_filter" + ) + return ( + f" \n" + "

            仅对评论量排名前 2 的受众进行分析。

            " + ) + + +def _html_audience_step2_blocks( + audience_ids: Sequence[int], + *, + top2_audience_clusters: Sequence[int], +) -> str: + if not audience_ids: + return ( + _html_step2_intro(top2_audience_clusters) + + "\n

            （本批次无「2.分受众分析」聚类结果，请先运行 聚类.py）

            \n" + + _html_list_placeholder(1) + ) + blocks: List[str] = [_html_step2_intro(top2_audience_clusters)] + for i, aud in enumerate(audience_ids): + rank_title = _audience_rank_title(i, aud) + blocks.append( + f""" +

            {rank_title}（正文请用「1.受众画像」CLUSTER_NAMES 中的纯中文受众名，勿写簇标签号）

            +

            （1）用户需求

            +{_html_list_placeholder(2)} +

            （2）正面产品反馈

            +{_html_list_placeholder(2)} +

            （3）负面产品反馈

            +{_html_list_placeholder(2)} +

            （4）产品客观描述

            +{_html_list_placeholder(2)}""" + ) + return "\n".join(blocks) + + +def _report_html_template( + product_name: str, + total_reviews: int, + bundles: Sequence[ClusterBundle], + *, + top2_audience_clusters: Sequence[int], +) -> str: + """规范化 HTML 报告骨架：摘要 → 评论分析（（一）受众 +（二）全量）→ 改进建议。""" + audience_ids = _ordered_step2_audiences(bundles, top2_audience_clusters) + step2_html = _html_audience_step2_blocks( + audience_ids, top2_audience_clusters=top2_audience_clusters + ) + return f""" + + + + {product_name}产品站内评论分析报告 + + +

            {product_name}产品改进建议报告

            + +

            一、摘要

            +

            数据清洗后，共 {total_reviews} 条有效用户评论。

            +
              +
            • 核心受众与场景：（说明文字，禁止出现原始 ID）
              • +
            • 最核心的用户需求：（说明文字）
              • +
            • 显著的产品反馈特征（正/负/客观）：（说明文字）
              • +
            • 提炼核心改进建议与机会：（一句话概括方向）
              • +
            + +

            二、{product_name}评论分析

            +

            统计口径：按结构化短语数占比（结构化短语数 ÷ 全部有效评论数）。

            + +

            （一）受众画像与分析

            + + +

            1.受众画像

            +

            （1）受众群体特征：

            +{_html_list_placeholder(2)} +

            （2）受众的主要需求：

            +{_html_list_placeholder(2)} + +

            2.分受众分析

            +{step2_html} + +

            （二）全部用户需求与产品反馈

            + + +

            1.全部受众需求分析

            +{_html_list_placeholder(3)} + + +

            2.正面产品反馈

            +{_html_list_placeholder(3)} + + +

            3.负面产品反馈

            +{_html_list_placeholder(3)} + + +

            4.产品客观描述

            +{_html_list_placeholder(3)} + +

            三、改进建议与机会

            +
              +
            1. 配方与成分优化：（针对负面反馈与未满足需求的可执行建议）
              2. +
            2. 包装与品控升级：（具体执行建议）
              3. +
            3. 说明书/Listing优化：（具体执行建议）
              4. +
            4. 市场与产品机会：（基于正面反馈与客观描述中的可放大卖点、人群或场景机会）
              5. +
            + +""" + + +def _normalize_report_html_fragment(html_text: str) -> str: + """从完整 HTML 文档中提取可嵌入页面的 body 片段。""" + text = html_text.strip() + m = re.search(r"]*>([\s\S]*?)", text, flags=re.I) + if m: + return m.group(1).strip() + if re.search(r"]", text, flags=re.I): + text = re.sub(r"]*>", "", text, flags=re.I) + text = re.sub(r"]*>[\s\S]*?", "", text, flags=re.I) + text = re.sub(r"]*>", "", text, flags=re.I) + return text.strip() + return text + + +def _strip_report_top_heading(html_fragment: str) -> str: + """去掉 LLM 报告正文开头的 h1，避免与页面级标题重复。""" + return re.sub( + r"^\s*]*>[\s\S]*?\s*", + "", + html_fragment, + count=1, + flags=re.I, + ) + + +def _build_report_prompt( + *, + product_name: str, + industry: str, + total_reviews: int, + bundles: List[ClusterBundle], + word_freq: List[Tuple[str, int]], + top2_audience_clusters: Sequence[int], +) -> Tuple[str, str]: + wf_table = word_freq[:WORD_FREQ_TABLE_N] + wf_classify = word_freq[:WORD_CATEGORY_CLASSIFY_N] + wf_table_lines = "\n".join(f"{w}\t{c}" for w, c in wf_table) + wf_classify_lines = "\n".join(f"{w}\t{c}" for w, c in wf_classify) + cats_literal = "、".join(WORD_CATEGORIES) + stage_map = _cluster_stage_mapping_guide( + bundles, top2_audience_clusters=top2_audience_clusters + ) + html_tpl = _report_html_template( + product_name, + total_reviews, + bundles, + top2_audience_clusters=top2_audience_clusters, + ) + system = build_report_system() + user = f"""请基于以下用户评论反馈的聚类结果，撰写一份详细的《{product_name}》产品改进建议报告。 + +行业背景：{industry} +数据清洗后有效评论总数 a = {total_reviews} + +【词频 Top{WORD_FREQ_TABLE_N}（word\\tcount，辅助理解品类与撰写报告，排名 1-{WORD_FREQ_TABLE_N}）】 +{wf_table_lines} + +【词频分类词表 Top{WORD_CATEGORY_CLASSIFY_N}（word\\tcount，**仅**用于 WORD_CATEGORY_JSON，排名 1-{WORD_CATEGORY_CLASSIFY_N}）】 +{wf_classify_lines} + +【聚类流程与报告小节映射】 +{stage_map} + +【聚类 · 二·（一）受众画像与分析（part1）】 +- {STAGE_1_AUDIENCE} → 1.受众画像 ·（1）受众群体特征、（2）受众的主要需求 +- top2 受众（本批次簇标签：{", ".join(str(x) for x in top2_audience_clusters) or "见 2a/2b stage 后缀"}）→ 2.分受众分析： + 2a 用户需求；2b 正面 / 负面 / 产品客观描述（Neutral 禁止写作「中性反馈」） +已按「结构化短语数」降序排列： +{_bundles_for_prompt(bundles, "part1")} + +【聚类 · 二·（二）全部用户需求与产品反馈（part2）】 +- {STAGE_3A_PAIN} → 1.全部受众需求分析 +- 3b → 2.正面产品反馈 / 3.负面产品反馈 / 4.产品客观描述 +已按「结构化短语数」降序排列： +{_bundles_for_prompt(bundles, "part2")} + +{build_report_analysis_requirements(product_name=product_name, stage_1_audience=STAGE_1_AUDIENCE)} + +{build_report_output_format()} + +【HTML 结构模版（请替换括号占位为真实分析，可增删
          • ，保持层级）】 +{html_tpl} + +{build_report_json_markers(cats_literal=cats_literal)} +""" + return system, user + +def _marker_body(raw: str, marker: str) -> str: + if marker not in raw: + return "" + part = raw.split(marker, 1)[1] + for end_marker in REPORT_MARKERS: + if end_marker != marker and end_marker in part: + part = part.split(end_marker, 1)[0] + part = part.strip() + part = re.sub(r"^```(?:json)?\s*", "", part, flags=re.I) + part = re.sub(r"\s*```\s*$", "", part) + return part + + +def _json_block_slice(raw: str, marker: str) -> Tuple[str, str]: + """返回 (marker 后正文, 用于 json.loads 的子串)。""" + part = _marker_body(raw, marker) + if not part: + return part, "" + start, end = part.find("{"), part.rfind("}") + if start != -1 and end > start: + return part, part[start : end + 1] + if start != -1: + return part, part[start:].strip() + return part, "" + + +def _coerce_word_category_data(data: Dict[str, Any]) -> Dict[str, Any]: + """将 LLM 各类别块统一为 {words: [...], analysis: str}。""" + out: Dict[str, Any] = {} + pending_analysis: str | None = None + for key, val in data.items(): + if key == "analysis": + if isinstance(val, str): + pending_analysis = val + continue + if not isinstance(val, (dict, list)): + continue + if isinstance(val, list): + out[key] = {"words": val, "analysis": ""} + else: + words = val.get("words") + if words is None: + words = val.get("word") + out[key] = { + "words": list(words) if isinstance(words, list) else [], + "analysis": str(val.get("analysis") or ""), + } + if pending_analysis and len(out) == 1: + only_key = next(iter(out)) + if not out[only_key].get("analysis"): + out[only_key]["analysis"] = pending_analysis + return out + + +def _trim_incomplete_json_tail(text: str) -> str: + """去掉截断在引号/逗号上的尾部，便于补全括号。""" + s = text.rstrip() + while s.endswith(","): + s = s[:-1].rstrip() + m = re.search(r'(,\s*|\[\s*)\"[^\"\\]*$', s) + if m: + s = s[: m.start()].rstrip() + if s.endswith(","): + s = s[:-1].rstrip() + return s + + +def _close_truncated_json_object(text: str) -> str: + """为截断在数组/对象中间的 JSON 片段补全括号（尽力而为）。""" + s = _close_truncated_json_braces(text) + if s and '"analysis"' not in s: + s = s.rstrip(", ") + ',"analysis":""' + if s.count("{") > s.count("}"): + s = s + "}" + return s + + +def _repair_word_category_json(js: str) -> Dict[str, Any] | None: + """ + 修复 LLM 将各类别写成多段伪对象的情况，例如： + {"成分/原料":[...],"analysis":"..."}, "受众/使用对象":[...], "analysis":"..."}, ... + 亦支持输出被截断、缺少最外层闭合括号的情形。 + """ + text = js.strip() + if not text: + return None + if not text.startswith("{"): + text = "{" + text + parts = re.split(r"\}\s*,\s*(?=\")", text) + merged: Dict[str, Any] = {} + for i, part in enumerate(parts): + chunk = part.strip().rstrip(",").strip() + if not chunk: + continue + if i == 0: + if not chunk.endswith("}"): + chunk = _close_truncated_json_object(chunk) + else: + if not chunk.startswith("{"): + chunk = "{" + chunk + if not chunk.endswith("}"): + chunk = _close_truncated_json_object(chunk) + try: + obj = json.loads(chunk) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + merged.update(_coerce_word_category_data(obj)) + return merged or None + + +def _extract_word_category_blocks_regex(text: str) -> Dict[str, Any] | None: + """从截断/脏文本中抽取完整的「类别 + words + analysis」块。""" + if not text or "{" not in text: + return None + merged: Dict[str, Any] = {} + cat_alt = "|".join(re.escape(c) for c in WORD_CATEGORIES) + pattern = ( + rf'\{{\s*"({cat_alt})"\s*:\s*(\[[^\]]*\])\s*,\s*' + r'"analysis"\s*:\s*"((?:[^"\\]|\\.)*)"\s*\}' + ) + for m in re.finditer(pattern, text): + cat, words_json, analysis = m.group(1), m.group(2), m.group(3) + try: + words = json.loads(words_json) + except json.JSONDecodeError: + continue + if not isinstance(words, list): + continue + merged[cat] = { + "words": [str(w) for w in words if str(w).strip()], + "analysis": analysis.replace('\\"', '"'), + } + return merged or None + + +def _try_parse_word_category_json_block(raw: str) -> Tuple[Dict[str, Any], str | None]: + marker = "===WORD_CATEGORY_JSON===" + if marker not in raw: + return {}, f"缺少标记 {marker}" + part = _marker_body(raw, marker) + _part, js = _json_block_slice(raw, marker) + payload = js or (part[part.find("{") :] if "{" in part else part) + data: Dict[str, Any] | None = None + if payload: + try: + parsed = json.loads(payload) + if isinstance(parsed, dict): + data = _coerce_word_category_data(parsed) + except json.JSONDecodeError: + data = _repair_word_category_json(payload) + if data: + logger.info("已自动修复 %s 的非标准 JSON 结构", marker) + if not data and payload: + closed = _close_truncated_json_object(payload) + if closed != payload: + data = _repair_word_category_json(closed) + if data: + logger.info("已补全截断的 %s 并解析出 %s 类", marker, len(data)) + if not data and part: + data = _extract_word_category_blocks_regex(part) + if data: + logger.info("已从 %s 截断文本中按类别块正则抽取 %s 类", marker, len(data)) + if not data: + preview = (part or _part)[:200].replace("\n", " ") + return ( + {}, + f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}", + ) + return data, None + + +def _close_truncated_json_braces(text: str) -> str: + """仅补全 [] / {{}}，不注入 analysis 等字段。""" + s = _trim_incomplete_json_tail(text.strip()) + if not s: + return s + if not s.startswith("{"): + s = "{" + s + if s.count("[") > s.count("]"): + s = s + "]" + if s.count("{") > s.count("}"): + s = s + "}" + return s + + +def _extract_json_kv_regex(text: str) -> Dict[str, str]: + """从截断的 JSON 对象文本中提取已完整的 key:value 字符串对。""" + out: Dict[str, str] = {} + for m in re.finditer( + r'"((?:[^"\\]|\\.)+)"\s*:\s*"((?:[^"\\]|\\.)*)"', + text, + ): + k = m.group(1).replace('\\"', '"') + v = m.group(2).replace('\\"', '"') + if k.strip() and v.strip(): + out[k] = v + return out + + +def _loads_json_object_loose(payload: str) -> Tuple[Dict[str, Any] | None, str]: + """ + 解析单个 JSON 对象。兼容尾部多余 `}}`、Extra data、以及真实截断时的补全/正则抽取。 + 返回 (dict, repair_note)；repair_note 非空时仅用于日志。 + """ + text = payload.strip() + if not text or "{" not in text: + return None, "" + + def _loads_once(s: str) -> Dict[str, Any] | None: + try: + obj = json.loads(s) + except json.JSONDecodeError as e: + if e.msg == "Extra data" or "Extra data" in e.msg: + try: + obj, _idx = json.JSONDecoder().raw_decode(s) + except json.JSONDecodeError: + return None + else: + return None + return obj if isinstance(obj, dict) else None + + obj = _loads_once(text) + if obj is not None: + tail = text[text.rfind("}") + 1 :].strip() + if tail: + return obj, "ignore_trailing_garbage" + return obj, "" + + trimmed = text + while trimmed.endswith("}") and trimmed.count("{") < trimmed.count("}"): + trimmed = trimmed[:-1].rstrip() + obj = _loads_once(trimmed) + if obj is not None: + return obj, "trim_extra_brace" + + closed = _close_truncated_json_braces(text) + if closed != text: + obj = _loads_once(closed) + if obj is not None: + return obj, "close_truncated" + + kv = _extract_json_kv_regex(text) + if kv: + return kv, "regex_kv" + return None, "" + + +def _try_parse_json_block(raw: str, marker: str) -> Tuple[Dict[str, Any], str | None]: + if marker not in raw: + return {}, f"缺少标记 {marker}" + part = _marker_body(raw, marker) + _part, js = _json_block_slice(raw, marker) + payload = js or (part[part.find("{") :] if "{" in part else "") + if not payload: + preview = (part or _part)[:200].replace("\n", " ") + return {}, f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}" + data, repair = _loads_json_object_loose(payload) + if data is not None and repair: + if repair == "regex_kv": + logger.info("已从损坏的 %s 中 regex 抽取 %s 个键值对", marker, len(data)) + elif repair == "close_truncated": + logger.info("已补全截断的 %s（%s 个键）", marker, len(data)) + elif repair in ("ignore_trailing_garbage", "trim_extra_brace"): + logger.debug("已宽松解析 %s（%s 个键，%s）", marker, len(data), repair) + if data is None: + try: + json.loads(payload) + except json.JSONDecodeError as e: + pos = e.pos if e.pos is not None else 0 + ctx = payload[max(0, pos - 50) : pos + 50] + return ( + {}, + f"{marker} JSON 解析失败: {e.msg}（行{e.lineno}列{e.colno}）；" + f"错误附近: ...{ctx!r}...", + ) + return {}, f"{marker} 必须是 JSON 对象" + if not isinstance(data, dict): + return {}, f"{marker} 必须是 JSON 对象，实际为 {type(data).__name__}" + return data, None + + +def _extract_json_block(raw: str, marker: str) -> Dict[str, Any]: + data, err = _try_parse_json_block(raw, marker) + if err: + logger.warning("解析 %s 失败: %s", marker, err) + return data + + +def _extract_report_html_fragment(raw: str) -> str: + if "===REPORT_HTML===" not in raw: + return "" + part = raw.split("===REPORT_HTML===", 1)[1] + for marker in REPORT_MARKERS[1:]: + if marker in part: + part = part.split(marker, 1)[0] + report_html = part.strip() + report_html = re.sub(r"```html?", "", report_html, flags=re.I) + report_html = report_html.replace("```", "").strip() + return _normalize_report_html_fragment(report_html) + + +def _validate_word_zh_json(data: Dict[str, Any], marker: str) -> List[str]: + errs: List[str] = [] + if not data: + errs.append(f"{marker} 为空对象") + return errs + for k, v in data.items(): + if not str(k).strip(): + errs.append(f"{marker} 含空键名") + if not isinstance(v, str) or not str(v).strip(): + errs.append(f"{marker} 键 {k!r} 的值必须为非空字符串") + break + return errs + + +def _validate_word_category_json(data: Dict[str, Any], marker: str) -> List[str]: + errs: List[str] = [] + if not data: + errs.append(f"{marker} 为空对象") + return errs + has_any_words = False + for cat, block in data.items(): + if not isinstance(block, dict): + errs.append(f"{marker} 类别 {cat!r} 的值必须是对象") + continue + words = block.get("words") + if words is not None: + if not isinstance(words, list): + errs.append(f"{marker} 类别 {cat!r}.words 必须是数组") + elif words: + has_any_words = True + analysis = block.get("analysis") + if analysis is not None and not isinstance(analysis, str): + errs.append(f"{marker} 类别 {cat!r}.analysis 必须是字符串") + if not has_any_words: + errs.append(f"{marker} 所有类别的 words 均为空") + return errs + + +def _validate_cluster_names_json( + data: Dict[str, Any], marker: str, *, expect_keys: bool +) -> List[str]: + errs: List[str] = [] + if expect_keys and not data: + errs.append(f"{marker} 为空对象（需要为聚类簇命名）") + return errs + for k, v in data.items(): + if not str(k).strip(): + errs.append(f"{marker} 含空键名") + if not isinstance(v, str) or not str(v).strip(): + errs.append(f"{marker} 键 {k!r} 的簇名必须为非空字符串") + break + return errs + + +def _dedupe_errors(errors: List[str]) -> List[str]: + return list(dict.fromkeys(errors)) + + +def _validate_report_response( + raw: str, *, expect_cluster_names: bool +) -> List[str]: + errors: List[str] = [] + for marker in REPORT_MARKERS: + if marker not in raw: + if marker == "===CLUSTER_NAMES_JSON===" and expect_cluster_names: + logger.warning( + "缺少 %s（将使用程序默认簇名）；若频繁出现请检查模型 max_tokens 是否截断输出", + marker, + ) + else: + errors.append(f"缺少标记 {marker}") + + report_html = _extract_report_html_fragment(raw) + if not report_html: + errors.append("===REPORT_HTML=== 内容为空或无法提取 片段") + elif " 标题") + elif " 章节") + + wzh, err = _try_parse_json_block(raw, "===WORD_ZH_JSON===") + if err: + errors.append(err) + else: + errors.extend(_validate_word_zh_json(wzh, "===WORD_ZH_JSON===")) + + cats, err = _try_parse_word_category_json_block(raw) + if err: + errors.append(err) + else: + errors.extend(_validate_word_category_json(cats, "===WORD_CATEGORY_JSON===")) + + if "===CLUSTER_NAMES_JSON===" in raw: + names, err = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===") + if err: + errors.append(err) + else: + errors.extend( + _validate_cluster_names_json( + names, + "===CLUSTER_NAMES_JSON===", + expect_keys=expect_cluster_names and not names, + ) + ) + return _dedupe_errors(errors) + + +def _build_report_correction_user_message(errors: List[str], raw: str) -> str: + err_block = "\n".join(f"- {e}" for e in errors) + max_chars = 100_000 + raw_body = raw if len(raw) <= max_chars else raw[:max_chars] + "\n\n...(上文已截断)..." + cats_literal = "、".join(WORD_CATEGORIES) + return build_report_correction_message( + err_block=err_block, + raw_body=raw_body, + cats_literal=cats_literal, + ) + + +def _fetch_report_llm_raw_with_retry( + *, + system: str, + user: str, + api_key: str, + expect_cluster_names: bool, + max_retries: int = REPORT_PARSE_MAX_RETRIES, +) -> str: + messages: List[Dict[str, str]] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + raw = _call_llm_messages( + messages, + api_key, + max_tokens=REPORT_MAX_OUTPUT_TOKENS, + timeout=REPORT_LLM_TIMEOUT_SEC, + ) + for attempt in range(max_retries + 1): + errors = _validate_report_response(raw, expect_cluster_names=expect_cluster_names) + if not errors: + if attempt > 0: + logger.info("报告 LLM 输出校验通过（第 %s 次修正后）", attempt + 1) + return raw + if attempt >= max_retries: + logger.warning( + "报告解析校验仍失败（已重试 %s 次），继续使用最后一次输出: %s", + max_retries, + errors, + ) + return raw + logger.warning( + "报告解析校验失败（第 %s/%s 次），请求 LLM 修正: %s", + attempt + 1, + max_retries, + errors, + ) + messages.append({"role": "assistant", "content": raw}) + messages.append( + { + "role": "user", + "content": _build_report_correction_user_message(errors, raw), + } + ) + raw = _call_llm_messages( + messages, + api_key, + max_tokens=REPORT_MAX_OUTPUT_TOKENS, + timeout=REPORT_LLM_TIMEOUT_SEC, + ) + return raw + + +def _parse_report_response( + raw: str, +) -> Tuple[str, Dict[str, str], Dict[str, Any], Dict[str, str]]: + report_html = _extract_report_html_fragment(raw) + word_zh_raw, _ = _try_parse_json_block(raw, "===WORD_ZH_JSON===") + word_zh = {str(k).lower(): str(v) for k, v in word_zh_raw.items()} + categories, _ = _try_parse_word_category_json_block(raw) + cluster_names_raw, _ = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===") + cluster_names = {str(k): str(v) for k, v in cluster_names_raw.items()} + return report_html, word_zh, categories, cluster_names + + +def _zh_for_word(word: str, word_zh: Dict[str, str]) -> str: + return word_zh.get(word.lower(), word_zh.get(word, word)) + + +def _normalize_word_category_data(category_data: Dict[str, Any]) -> Dict[str, Any]: + """兼容旧版词频分类名「痛点/场景」→「需求/场景」。""" + if not category_data: + return category_data + out = dict(category_data) + legacy = out.pop("痛点/场景", None) + if legacy is not None and "需求/场景" not in out: + out["需求/场景"] = legacy + out.pop("其他", None) + return out + + +def _ensure_word_category_skeleton(category_data: Dict[str, Any]) -> Dict[str, Any]: + """保证七个类别键存在且值为 {{words, analysis}} 结构。""" + base = _normalize_word_category_data(category_data or {}) + out: Dict[str, Any] = {} + for cat in WORD_CATEGORIES: + block = base.get(cat) + if isinstance(block, dict): + words = block.get("words") if isinstance(block.get("words"), list) else [] + analysis = str(block.get("analysis") or "") + else: + words, analysis = [], "" + out[cat] = {"words": list(words), "analysis": analysis} + return out + + +def _category_pool_count_map( + word_freq: Sequence[Tuple[str, int]], +) -> Tuple[List[Tuple[str, int]], Dict[str, Tuple[str, int]], Set[str]]: + top = list(word_freq[:WORD_CATEGORY_CLASSIFY_N]) + count_map = {w.lower(): (w, c) for w, c in top} + keys = set(count_map.keys()) + return top, count_map, keys + + +def _valid_category_words_in_pool( + block: Any, + count_map: Dict[str, Tuple[str, int]], + top_keys: Set[str], +) -> List[str]: + if not isinstance(block, dict): + return [] + out: List[str] = [] + seen: Set[str] = set() + for w in block.get("words") or []: + key = str(w).lower().strip() + if not key or key in seen or key not in top_keys: + continue + en, cnt = count_map[key] + if cnt <= 0: + continue + seen.add(key) + out.append(en) + return out + + +def _unclassified_pool_words( + category_data: Dict[str, Any], + word_freq: Sequence[Tuple[str, int]], +) -> List[str]: + """Top{WORD_CATEGORY_CLASSIFY_N} 中尚未归入任何类别的词（按词频降序）。""" + top, count_map, _top_keys = _category_pool_count_map(word_freq) + classified = set(_build_word_category_map(category_data).keys()) + return [en for en, _ in top if en.lower() not in classified] + + +def _parse_word_assign_json(text: str) -> Dict[str, List[str]]: + """解析逐词分类结果：{{"word": ["类别", ...], ...}}，空数组表示不分类。""" + text = _strip_think(text).replace("```json", "").replace("```", "").strip() + start, end = text.find("{"), text.rfind("}") + if start == -1 or end <= start: + return {} + try: + obj = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + obj, _ = _loads_json_object_loose(text[start : end + 1]) + if not isinstance(obj, dict): + return {} + out: Dict[str, List[str]] = {} + for word, cats in obj.items(): + w = str(word).strip() + if not w: + continue + if isinstance(cats, list): + out[w] = [str(c).strip() for c in cats if str(c).strip()] + elif isinstance(cats, str) and cats.strip(): + out[w] = [cats.strip()] + else: + out[w] = [] + return out + + +def _build_word_assign_prompt( + *, + industry: str, + product_name: str, + batch_words: Sequence[str], + word_freq: Sequence[Tuple[str, int]], +) -> Tuple[str, str]: + _top, count_map, _top_keys = _category_pool_count_map(word_freq) + lines: List[str] = [] + for w in batch_words: + key = w.lower() + cnt = count_map.get(key, (w, 0))[1] + lines.append(f"{w}\t{cnt}") + cats_literal = "、".join(WORD_CATEGORIES) + return build_word_assign_prompts( + industry=industry, + product_name=product_name, + word_lines="\n".join(lines), + cats_literal=cats_literal, + ) + + +def _merge_word_assignments( + category_data: Dict[str, Any], + assignments: Dict[str, List[str]], + word_freq: Sequence[Tuple[str, int]], +) -> Dict[str, Any]: + _top, count_map, top_keys = _category_pool_count_map(word_freq) + out = _ensure_word_category_skeleton(category_data) + cat_set = set(WORD_CATEGORIES) + for word, cats in assignments.items(): + key = str(word).lower().strip() + if not key or key not in top_keys: + continue + en, cnt = count_map[key] + if cnt <= 0: + continue + valid_cats = [c for c in cats if c in cat_set] + if not valid_cats: + continue + for cat in valid_cats: + block = out[cat] + existing = _valid_category_words_in_pool(block, count_map, top_keys) + if any(e.lower() == key for e in existing): + continue + block["words"] = list(existing) + [en] + return out + + +def _classify_remaining_words_in_pool( + category_data: Dict[str, Any], + word_freq: Sequence[Tuple[str, int]], + *, + industry: str, + product_name: str, + api_key: str, +) -> Dict[str, Any]: + """对 Top{WORD_CATEGORY_CLASSIFY_N} 中尚未分类的词分批逐词尽量分类（可不归类）。""" + data = _ensure_word_category_skeleton(category_data) + pending = _unclassified_pool_words(data, word_freq) + top_n = WORD_CATEGORY_CLASSIFY_N + classified_n = top_n - len(pending) + if not pending: + logger.info( + "词频分类：Top%s 已全部有类别（%s 个词）", + top_n, + classified_n, + ) + return data + + logger.info( + "词频分类：Top%s 已分类 %s 个，待逐词补充分类 %s 个（每批 %s）", + top_n, + classified_n, + len(pending), + WORD_CATEGORY_ASSIGN_BATCH_SIZE, + ) + batches = [ + pending[i : i + WORD_CATEGORY_ASSIGN_BATCH_SIZE] + for i in range(0, len(pending), WORD_CATEGORY_ASSIGN_BATCH_SIZE) + ] + for bi, batch in enumerate(batches, start=1): + system, user = _build_word_assign_prompt( + industry=industry, + product_name=product_name, + batch_words=batch, + word_freq=word_freq, + ) + raw = _call_llm_messages( + [{"role": "system", "content": system}, {"role": "user", "content": user}], + api_key, + temperature=0.2, + max_tokens=WORD_CATEGORY_ASSIGN_MAX_TOKENS, + timeout=REPORT_LLM_TIMEOUT_SEC, + ) + before_keys = set(_build_word_category_map(data).keys()) + assign = _parse_word_assign_json(raw) + if assign: + data = _merge_word_assignments(data, assign, word_freq) + after_keys = set(_build_word_category_map(data).keys()) + newly = len(after_keys - before_keys) + logger.info( + "词频逐词分类 第 %s/%s 批：本批 %s 词，新归类 %s 个", + bi, + len(batches), + len(batch), + newly, + ) + + still = _unclassified_pool_words(data, word_freq) + logger.info( + "词频分类完成：Top%s 共归类 %s 个，未分类 %s 个（词云显示为未分类）", + top_n, + top_n - len(still), + len(still), + ) + return data + + +def _categories_needing_analysis( + category_data: Dict[str, Any], + word_freq: Sequence[Tuple[str, int]], +) -> List[str]: + """有词条但 analysis 为空的类别。""" + _top, count_map, top_keys = _category_pool_count_map(word_freq) + data = _ensure_word_category_skeleton(category_data) + need: List[str] = [] + for cat in WORD_CATEGORIES: + words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys) + if words and not str(data[cat].get("analysis") or "").strip(): + need.append(cat) + return need + + +def _parse_category_analysis_json(text: str) -> Dict[str, str]: + """解析 {{\"类别\": \"一段中文解读\", ...}}。""" + text = _strip_think(text).replace("```json", "").replace("```", "").strip() + start, end = text.find("{"), text.rfind("}") + if start == -1 or end <= start: + return {} + try: + obj = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + obj, _ = _loads_json_object_loose(text[start : end + 1]) + if not isinstance(obj, dict): + return {} + out: Dict[str, str] = {} + for k, v in obj.items(): + cat = str(k).strip() + if cat in WORD_CATEGORIES and isinstance(v, str) and v.strip(): + out[cat] = v.strip() + return out + + +def _build_category_analysis_prompt( + *, + industry: str, + product_name: str, + word_freq: Sequence[Tuple[str, int]], + category_data: Dict[str, Any], + need_cats: Sequence[str], +) -> Tuple[str, str]: + _top, count_map, top_keys = _category_pool_count_map(word_freq) + data = _ensure_word_category_skeleton(category_data) + lines: List[str] = [] + for cat in need_cats: + words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys)[ + :WORD_CATEGORY_ANALYSIS_MAX_WORDS + ] + parts = [] + for w in words: + key = w.lower() + cnt = count_map.get(key, (w, 0))[1] + parts.append(f"{w}({cnt})") + lines.append(f"- {cat}：{', '.join(parts) if parts else '（无）'}") + cats_literal = "、".join(need_cats) + return build_category_analysis_prompts( + industry=industry, + product_name=product_name, + category_lines="\n".join(lines), + cats_literal=cats_literal, + ) + + +def _merge_category_analysis( + category_data: Dict[str, Any], + analyses: Dict[str, str], +) -> Dict[str, Any]: + out = _ensure_word_category_skeleton(category_data) + for cat, text in analyses.items(): + if cat not in WORD_CATEGORIES: + continue + if text.strip() and not str(out[cat].get("analysis") or "").strip(): + out[cat]["analysis"] = text.strip() + return out + + +def _fill_category_analysis( + category_data: Dict[str, Any], + word_freq: Sequence[Tuple[str, int]], + *, + industry: str, + product_name: str, + api_key: str, +) -> Dict[str, Any]: + """为有词但缺少 analysis 的类别自动生成简短中文解读。""" + data = _ensure_word_category_skeleton(category_data) + need = _categories_needing_analysis(data, word_freq) + if not need: + return data + + logger.info("词频分类：为 %s 个类别补写 analysis: %s", len(need), need) + system, user = _build_category_analysis_prompt( + industry=industry, + product_name=product_name, + word_freq=word_freq, + category_data=data, + need_cats=need, + ) + for attempt in range(2): + raw = _call_llm_messages( + [{"role": "system", "content": system}, {"role": "user", "content": user}], + api_key, + temperature=0.3, + max_tokens=WORD_CATEGORY_ANALYSIS_MAX_TOKENS, + timeout=REPORT_LLM_TIMEOUT_SEC, + ) + parsed = _parse_category_analysis_json(raw) + if parsed: + data = _merge_category_analysis(data, parsed) + still = _categories_needing_analysis(data, word_freq) + if not still: + logger.info("词频分类各类 analysis 已补全") + return data + need = still + user = _build_category_analysis_prompt( + industry=industry, + product_name=product_name, + word_freq=word_freq, + category_data=data, + need_cats=need, + )[1] + logger.warning( + "词频 analysis 仍有 %s 类未生成（第 %s 次重试）: %s", + len(still), + attempt + 1, + still, + ) + + logger.warning("词频分类 analysis 仍未补全: %s", still) + return data + + +def _build_word_category_map(category_data: Dict[str, Any]) -> Dict[str, List[str]]: + """英文词（小写）-> 所属分类列表（按 WORD_CATEGORIES 顺序，支持一词多类）。""" + category_data = _normalize_word_category_data(category_data) + by_word: Dict[str, List[str]] = defaultdict(list) + for cat in WORD_CATEGORIES: + block = category_data.get(cat) + if not isinstance(block, dict): + continue + for w in block.get("words") or []: + key = str(w).lower().strip() + if key and cat not in by_word[key]: + by_word[key].append(cat) + return dict(by_word) + + +def _wordcloud_color_for_categories(categories: List[str]) -> str: + if not categories: + return WORDCLOUD_UNCATEGORIZED_COLOR + return CATEGORY_COLORS.get(categories[0], WORDCLOUD_UNCATEGORIZED_COLOR) + + +def _render_wordcloud_legend() -> str: + items = "".join( + f'' + f'' + f"{html.escape(c)}" + for c in WORD_CATEGORIES + ) + items += ( + f'' + f'' + f"未分类" + ) + return f'
            {items}
            ' + + +def _wordcloud_data( + word_freq: List[Tuple[str, int]], + word_zh: Dict[str, str], + category_data: Dict[str, Any], +) -> List[dict]: + items = word_freq[:WORDCLOUD_TOP_N] + if not items: + return [] + word_cats = _build_word_category_map(category_data) + counts = [c for _, c in items] + c_min, c_max = min(counts), max(counts) + span = c_max - c_min + out: List[dict] = [] + for rank, (word, count) in enumerate(items, start=1): + norm = 1.0 if span <= 0 else (count - c_min) / span + value = max(1, int((norm**WORDCLOUD_SIZE_POWER) * 1000)) + zh = _zh_for_word(word, word_zh) + cats = word_cats.get(word.lower(), []) + color = _wordcloud_color_for_categories(cats) + out.append( + { + "name": word, + "value": value, + "count": count, + "rank": rank, + "en": word, + "zh": zh, + "category": cats[0] if cats else "", + "categories": cats, + "textStyle": {"color": color}, + } + ) + return out + + +def _freq_rank_map(word_freq: List[Tuple[str, int]], n: int) -> Dict[str, int]: + return {w.lower(): i + 1 for i, (w, _) in enumerate(word_freq[:n])} + + +def _category_top_words( + category_data: Dict[str, Any], + word_freq: List[Tuple[str, int]], + *, + top_n: int = 10, +) -> Dict[str, List[Tuple[str, int, str]]]: + category_data = _normalize_word_category_data(category_data) + count_map = {w.lower(): (w, c) for w, c in word_freq} + rank_map = _freq_rank_map(word_freq, WORD_CATEGORY_CLASSIFY_N) + out: Dict[str, List[Tuple[str, int, str]]] = {} + for cat in WORD_CATEGORIES: + block = category_data.get(cat) + if not isinstance(block, dict): + out[cat] = [] + continue + words = block.get("words") or [] + rows: List[Tuple[str, int, str]] = [] + seen: set[str] = set() + for w in words: + key = str(w).lower().strip() + if not key or key in seen: + continue + seen.add(key) + if key in count_map: + en, cnt = count_map[key] + else: + en, cnt = str(w), 0 + if cnt <= 0: + continue + rows.append((en, cnt, str(rank_map.get(key, "")))) + rows.sort(key=lambda x: (-x[1], x[0])) + out[cat] = rows[:top_n] + return out + + +def _render_category_blocks( + category_data: Dict[str, Any], + category_words: Dict[str, List[Tuple[str, int, str]]], + word_zh: Dict[str, str], +) -> str: + category_data = _normalize_word_category_data(category_data) + parts = [ + '
            ', + "

            词频分类洞察

            ", + ] + for cat in WORD_CATEGORIES: + block = category_data.get(cat) + analysis = "" + if isinstance(block, dict): + analysis = str(block.get("analysis") or "").strip() + rows = category_words.get(cat, []) + parts.append(f'

            {html.escape(cat)}

            ') + if analysis: + parts.append(f"

            {html.escape(analysis)}

            ") + if rows: + parts.append( + "
          3. " + ) + for en, cnt, rank in rows: + zh = _zh_for_word(en, word_zh) + rank_cell = rank if rank else "—" + parts.append( + f"" + f"" + f"" + ) + parts.append("
          排名英文中文次数
          {html.escape(str(rank_cell))}{html.escape(en)}{html.escape(zh)}{cnt}
          ") + else: + parts.append('

          （本类暂无词条）

          ') + parts.append("") + parts.append("") + return "\n".join(parts) + + +def _render_freq_table_pages( + word_freq: List[Tuple[str, int]], word_zh: Dict[str, str] +) -> str: + top = word_freq[:WORD_FREQ_TABLE_N] + total = sum(c for _, c in top) or 1 + max_cnt = max((c for _, c in top), default=1) + row_lines: List[str] = [] + for rank, (en, cnt) in enumerate(top, start=1): + page = (rank - 1) // WORD_FREQ_PAGE_SIZE + 1 + zh = _zh_for_word(en, word_zh) + pct = cnt / total * 100.0 + bar_w = max(4, int(cnt / max_cnt * 100)) + row_lines.append( + f'' + f"{rank}" + f'{html.escape(en)}' + f"{html.escape(zh)}" + f'' + f"{cnt}" + f"{pct:.2f}%" + ) + n_pages = min( + WORD_FREQ_PAGES, + (len(top) + WORD_FREQ_PAGE_SIZE - 1) // WORD_FREQ_PAGE_SIZE or 1, + ) + tabs = "".join( + f'' + for i in range(n_pages) + ) + body = f""" + {tabs} +
          + + 第 1 / {n_pages} 页 + +
          +
          + + + + + + {chr(10).join(row_lines)} + +
          排名英文中文次数占比
          +
          +

          占比 = 该词次数 / Top{WORD_FREQ_TABLE_N} 词次数之和（相对占比）。

          +""" + return body.replace(" str: + category_words = _category_top_words(category_data, word_freq, top_n=10) + cat_html = _render_category_blocks(category_data, category_words, word_zh) + table_html = _render_freq_table_pages(word_freq, word_zh) + return ( + f'
          \n' + f"

          词频分析

          \n" + f'

          词云 Top {WORDCLOUD_TOP_N} · 词表 Top {WORD_FREQ_TABLE_N} · ' + f"分类词表 Top {WORD_CATEGORY_CLASSIFY_N}

          \n" + f"

          词云图

          \n" + f" {_render_wordcloud_legend()}\n" + f'
          \n' + f" {cat_html}\n" + f"

          全部词频表

          \n" + f" {table_html}\n" + f"
          " + ) + + +def _render_phrase_sections(appendix_bundles: List[ClusterBundle]) -> str: + by_stage: Dict[str, List[ClusterBundle]] = defaultdict(list) + for b in appendix_bundles: + by_stage[b.stage].append(b) + + parts = ['
          '] + for stage in PHRASE_APPENDIX_STAGES: + title = _phrase_appendix_stage_title(stage) + parts.append(f'

          {html.escape(title)}

          ') + rows = sorted(by_stage.get(stage, []), key=lambda b: b.cluster_label) + if not rows: + parts.append('

          （暂无）

          ') + continue + for b in rows: + pct = f"{b.ratio * 100:.1f}%" + parts.append( + f'

          {html.escape(b.cluster_title_zh)}' + f' （{b.review_count} 条评论 · {pct}）

          ' + ) + display = b.embed_texts_zh or b.embed_texts + parts.append( + f'
          展开本簇 {len(display)} 条代表性表述（中文）
            ' + ) + for txt in display: + parts.append(f"

          1. {html.escape(txt)}

            2. ") + if not display: + parts.append("

          2. （无可用表述）

            3. ") + parts.append("
          ") + parts.append("
          ") + return "\n".join(parts) + + +def _render_structured_validation_section( + samples: List[Tuple[int, str, Dict[str, Any]]], +) -> str: + parts = [ + '
          ', + f'

          展示前 {STRUCTURED_APPENDIX_SAMPLE_N} 条评论原文及对应结构化结果（按 source_row 升序）。

          ', + ] + if not samples: + parts.append('

          （暂无结构化样本，请先执行结构化步骤。）

          ') + parts.append("
          ") + return "\n".join(parts) + + for i, (sr, content, display) in enumerate(samples, start=1): + js = json.dumps(display, ensure_ascii=False, separators=(",", ":")) + parts.append(f'
          ') + parts.append(f"

          样本 {i} （source_row={sr}）

          ") + parts.append("

          评论原文：

          ") + parts.append(f'

          {html.escape(content)}

          ') + parts.append("

          结构化内容：

          ") + parts.append(f'
          {html.escape(js)}
          ') + parts.append("
          ") + parts.append("") + return "\n".join(parts) + + +def _wrap_ai_verify_panel(title: str, inner_html: str) -> str: + """将子版块包进可折叠面板（默认收起）。""" + return ( + f'
          \n' + f" {html.escape(title)}\n" + f" {inner_html}\n" + f"
          " + ) + + +def _render_ai_validation_section( + structured_html: str, + phrases_html: str, +) -> str: + """AI 分析效果验证：结构化 + 聚类，机器固定顺序、可展开。""" + structured_panel = _wrap_ai_verify_panel("结构化提取效果验证", structured_html) + phrases_panel = _wrap_ai_verify_panel("聚类效果验证", phrases_html) + return ( + '
          \n' + "

          AI分析效果验证

          \n" + '

          以下为结构化与聚类结果的抽样展示，点击标题展开查看。

          \n' + f" {structured_panel}\n" + f" {phrases_panel}\n" + "
          " + ) + + +def _assemble_html( + *, + product_name: str, + wordfreq_html: str, + report_html: str, + phrases_html: str, + structured_html: str, + wordcloud_data: List[dict], +) -> str: + """按固定顺序机器拼装最终 HTML（不由 LLM 决定版块顺序）。""" + data_json = json.dumps(wordcloud_data, ensure_ascii=False) + page_title = f"{product_name} · 评论分析报告" + report_body = _strip_report_top_heading(report_html) + report_section = ( + '
          \n' + "

          产品改进建议报告

          \n" + f" {report_body}\n" + "
          " + ) + ai_verify_section = _render_ai_validation_section(structured_html, phrases_html) + # 顺序固定：改进建议 → 词频 → AI 分析效果验证（内含结构化 / 聚类，可展开） + body_sections = "\n".join((report_section, wordfreq_html, ai_verify_section)) + return f""" + + + + + {page_title} + + + + + +

          {page_title}

          + {body_sections} + + + +""" + + +def generate_report( + *, + product_name: str, + industry: str, + cleaned_csv: Path, + cluster_db: Path, + embed_db: Path, + word_freq_csv: Path, + output_html: Path, + structured_db: Path = STRUCTURED_DB, + min_cluster_review_ratio: float | None = None, + save_llm_raw: bool = False, + llm_raw_path: Path | None = None, +) -> dict: + sync_voc_report_constants(sys.modules[__name__]) + api_key = _resolve_api_key() + if not api_key: + raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key") + + cleaned_csv = cleaned_csv.resolve() + word_freq = _load_word_freq( + word_freq_csv, + max(WORDCLOUD_TOP_N, WORD_FREQ_TABLE_N, WORD_CATEGORY_CLASSIFY_N), + ) + total_reviews, bundles = build_cluster_bundles( + cluster_db=cluster_db, + embed_db=embed_db, + cleaned_csv=cleaned_csv, + min_cluster_review_ratio=min_cluster_review_ratio, + ) + top2_audience = _load_step2_top2_audience(cluster_db) + if not top2_audience: + top2_audience = _ordered_step2_audiences(bundles, [])[:2] + logger.info("阶段二 top2 受众簇: %s", top2_audience) + + logger.info("调用 LLM 生成分析报告（含词频分类、簇命名、译文映射）…") + system, user = _build_report_prompt( + product_name=product_name, + industry=industry, + total_reviews=total_reviews, + bundles=bundles, + word_freq=word_freq, + top2_audience_clusters=top2_audience, + ) + raw = _fetch_report_llm_raw_with_retry( + system=system, + user=user, + api_key=api_key, + expect_cluster_names=len(bundles) > 0, + ) + saved_raw_path: str | None = None + if save_llm_raw: + raw_path = (llm_raw_path or output_html.parent / "report_llm_raw.txt").resolve() + raw_path.parent.mkdir(parents=True, exist_ok=True) + raw_path.write_text(raw, encoding="utf-8") + saved_raw_path = str(raw_path) + logger.info("已保存 LLM 原文 %s", raw_path) + + report_html, word_zh, category_data, cluster_names = _parse_report_response(raw) + category_data = _classify_remaining_words_in_pool( + category_data, + word_freq, + industry=industry, + product_name=product_name, + api_key=api_key, + ) + category_data = _fill_category_analysis( + category_data, + word_freq, + industry=industry, + product_name=product_name, + api_key=api_key, + ) + _apply_cluster_names(bundles, cluster_names) + + appendix_bundles = _bundles_for_phrase_appendix(bundles) + all_phrases: List[str] = [] + for b in appendix_bundles: + all_phrases.extend(b.embed_texts) + zh_map = _translate_unique_phrases(all_phrases, api_key) + _fill_phrase_translations(appendix_bundles, zh_map) + + wc_data = _wordcloud_data(word_freq, word_zh, category_data) + wordfreq_sec = _render_wordfreq_section(word_freq, word_zh, category_data) + phrases_sec = _render_phrase_sections(appendix_bundles) + struct_samples = _build_structured_appendix_samples( + structured_db=structured_db.resolve(), + cleaned_csv=cleaned_csv, + ) + structured_sec = _render_structured_validation_section(struct_samples) + html = _assemble_html( + product_name=product_name, + wordfreq_html=wordfreq_sec, + report_html=report_html, + phrases_html=phrases_sec, + structured_html=structured_sec, + wordcloud_data=wc_data, + ) + output_html.parent.mkdir(parents=True, exist_ok=True) + output_html.write_text(html, encoding="utf-8") + logger.info("已写入 %s", output_html) + return { + "report_html": str(output_html), + "total_reviews": total_reviews, + "cluster_groups": len(bundles), + "min_cluster_review_ratio": min_cluster_review_ratio, + "structured_samples": len(struct_samples), + "llm_raw_path": saved_raw_path, + } diff --git a/合并评论数据.py b/合并评论数据.py new file mode 100644 index 0000000..f123ecf --- /dev/null +++ b/合并评论数据.py @@ -0,0 +1,103 @@ +""" +合并指定目录下所有表头一致的 CSV 文件为一个 CSV。 +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import pandas as pd + +_PROJECT_ROOT = Path(__file__).resolve().parent +DEFAULT_INPUT_DIR = _PROJECT_ROOT / "turkey tail mushroom-voc" +DEFAULT_OUTPUT_PATH = _PROJECT_ROOT / "merged_reviews.csv" + + +def merge_csv_directory( + input_dir: str | Path, + output_path: str | Path, + *, + pattern: str = "*.csv", + recursive: bool = False, +) -> pd.DataFrame: + """ + 读取目录内所有匹配的 CSV（表头须一致），纵向合并后写入 output_path。 + + :param input_dir: 含多个 CSV 的目录 + :param output_path: 合并结果输出路径 + :param pattern: 文件名 glob，默认 *.csv + :param recursive: 是否包含子目录中的 CSV + :return: 合并后的 DataFrame + """ + input_dir = Path(input_dir).resolve() + output_path = Path(output_path).resolve() + + if not input_dir.is_dir(): + raise NotADirectoryError(f"目录不存在: {input_dir}") + + globber = input_dir.rglob if recursive else input_dir.glob + csv_files = sorted( + p for p in globber(pattern) if p.is_file() and p.resolve() != output_path + ) + + if not csv_files: + raise FileNotFoundError(f"未在 {input_dir} 找到匹配 {pattern!r} 的 CSV 文件") + + frames: list[pd.DataFrame] = [] + expected_columns: list[str] | None = None + + for path in csv_files: + df = pd.read_csv(path, dtype=str, keep_default_na=False) + if expected_columns is None: + expected_columns = list(df.columns) + elif list(df.columns) != expected_columns: + raise ValueError( + f"表头不一致: {path.name}\n" + f" 期望: {expected_columns}\n" + f" 实际: {list(df.columns)}" + ) + frames.append(df) + + merged = pd.concat(frames, ignore_index=True) + output_path.parent.mkdir(parents=True, exist_ok=True) + merged.to_csv(output_path, index=False, encoding="utf-8-sig") + + print(f"已合并 {len(csv_files)} 个文件，共 {len(merged)} 行 -> {output_path}") + for path in csv_files: + print(f" - {path.name}") + + return merged + + +def main() -> None: + parser = argparse.ArgumentParser(description="合并目录内表头相同的 CSV 文件") + parser.add_argument( + "-i", + "--input", + type=Path, + default=DEFAULT_INPUT_DIR, + metavar="DIR", + help=f"输入目录（默认: {DEFAULT_INPUT_DIR}）", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + default=DEFAULT_OUTPUT_PATH, + metavar="FILE", + help=f"输出 CSV 路径（默认: {DEFAULT_OUTPUT_PATH}）", + ) + parser.add_argument( + "-r", + "--recursive", + action="store_true", + help="是否递归搜索子目录", + ) + args = parser.parse_args() + + merge_csv_directory(args.input, args.output, recursive=args.recursive) + + +if __name__ == "__main__": + main() diff --git a/向量化.py b/向量化.py new file mode 100644 index 0000000..485bf12 --- /dev/null +++ b/向量化.py @@ -0,0 +1,534 @@ +""" +从 voc_structured.sqlite 最新 job 展开 audience / pain_points / aspect / opinion / +aspect_opinion，调用 DashScope text-embedding-v4（256 维）写入 voc_embeddings.sqlite。 + +溯源：source_row 与结构化时一致（CSV 第 1 条数据行=1）；对应 merged_reviews_cleaned.csv +物理行号 = source_row + 1（第 1 行为表头），content 取自该数据行。 + +用法（项目根目录）:: + + python3 向量化.py + python3 向量化.py --workers 8 + python3 向量化.py --job-id 4 --csv merged_reviews_cleaned.csv + +说明：DashScope text-embedding-v4 单次请求最多 10 条文本；脚本按批调用 API， +默认多线程并行多批（--workers），并非逐条请求。 +""" +from __future__ import annotations + +import argparse +import json +import logging +import os +import sqlite3 +import struct +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Sequence, Tuple + +from csv import DictReader +from openai import OpenAI + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_embed") + +PROJECT_ROOT = Path(__file__).resolve().parent +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite" +DEFAULT_CSV = PROJECT_ROOT / "merged_reviews_cleaned.csv" +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" +EMBEDDING_MODEL = "text-embedding-v4" +EMBEDDING_DIMENSIONS = 256 +EMBED_BATCH_SIZE = 10 # DashScope text-embedding-v4 单请求 input 数组上限 10 +EMBED_DEFAULT_WORKERS = 6 # 并行批次数（每批最多 10 条） +ENTITY_TYPES = ( + "audience", + "pain_point", + "aspect", + "opinion", + "aspect_opinion", +) + + +@dataclass +class EmbedTask: + job_id: int + extraction_id: int + source_row: int + entity_type: str + entity_index: int + embed_text: str + audience: str + aspect: str | None + opinion: str | None + category: str | None + sentiment: str | None + content: str + + +def _resolve_api_key() -> str: + v = os.environ.get("DASHSCOPE_API_KEY", "").strip() + if v: + return v + fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip() + if fp: + p = Path(fp).expanduser() + if p.is_file(): + return p.read_text(encoding="utf-8").strip().strip('"').strip("'") + local = PROJECT_ROOT / ".dashscope_key" + if local.is_file(): + return local.read_text(encoding="utf-8").strip().strip('"').strip("'") + return "" + + +def _latest_job_id(conn: sqlite3.Connection) -> int: + row = conn.execute( + "SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1" + ).fetchone() + if not row: + raise RuntimeError("analysis_jobs 为空，请先运行结构化") + return int(row[0]) + + +def _load_content_by_source_row(csv_path: Path) -> Dict[int, str]: + """source_row（1 起）-> content；与结构化_server._load_csv_reviews 行号一致。""" + with csv_path.open(encoding="utf-8-sig", newline="") as f: + reader = DictReader(f) + if not reader.fieldnames: + raise ValueError(f"CSV 无表头: {csv_path}") + headers = [h.strip() for h in reader.fieldnames] + lower_map = {h.lower(): h for h in headers} + col = None + for name in ("content", "review", "评论"): + if name.lower() in lower_map: + col = lower_map[name.lower()] + break + if col is None: + if len(headers) == 1: + col = headers[0] + else: + raise ValueError(f"CSV 缺少 content 列: {headers}") + out: Dict[int, str] = {} + for idx, row in enumerate(reader, start=1): + cell = row.get(col) + t = str(cell or "").strip() + if t: + out[idx] = t + return out + + +def _pack_embedding(vec: Sequence[float]) -> bytes: + return struct.pack(f"{len(vec)}f", *vec) + + +def _init_embed_schema(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS embedding_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL, + extraction_id INTEGER NOT NULL, + source_row INTEGER NOT NULL, + entity_type TEXT NOT NULL, + entity_index INTEGER NOT NULL DEFAULT 0, + embed_text TEXT NOT NULL, + audience TEXT, + aspect TEXT, + opinion TEXT, + category TEXT, + sentiment TEXT, + content TEXT NOT NULL, + dimensions INTEGER NOT NULL, + embedding BLOB NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_embed_job_type + ON embedding_items(job_id, entity_type); + CREATE INDEX IF NOT EXISTS idx_embed_source_row + ON embedding_items(job_id, source_row); + CREATE INDEX IF NOT EXISTS idx_embed_extraction + ON embedding_items(extraction_id, entity_type, entity_index); + """ + ) + + +def _should_skip_extraction(audience: str, pain_points: List[Any], feedback: List[Any]) -> bool: + aud = (audience or "").strip().lower() + return aud == "unknown" and not pain_points and not feedback + + +def _build_tasks( + job_id: int, + rows: List[sqlite3.Row], + content_map: Dict[int, str], +) -> List[EmbedTask]: + tasks: List[EmbedTask] = [] + for row in rows: + ext_id = int(row["id"]) + source_row = int(row["source_row"]) + try: + data = json.loads(row["extraction_json"]) + except json.JSONDecodeError as e: + logger.warning("跳过无效 JSON extraction_id=%s: %s", ext_id, e) + continue + + audience = str(data.get("audience", "unknown")).strip() or "unknown" + pain_points = data.get("pain_points") or [] + feedback = data.get("product_feedback") or [] + if not isinstance(pain_points, list): + pain_points = [] + if not isinstance(feedback, list): + feedback = [] + + if _should_skip_extraction(audience, pain_points, feedback): + continue + + content = content_map.get(source_row, "") + if not content: + logger.warning( + "source_row=%s 在 CSV 中无 content，仍写入向量但 content 为空", + source_row, + ) + + aud_norm = audience.strip() + if aud_norm.lower() != "unknown": + tasks.append( + EmbedTask( + job_id=job_id, + extraction_id=ext_id, + source_row=source_row, + entity_type="audience", + entity_index=0, + embed_text=aud_norm, + audience=aud_norm, + aspect=None, + opinion=None, + category=None, + sentiment=None, + content=content, + ) + ) + + for i, p in enumerate(pain_points): + text = str(p).strip() + if not text: + continue + tasks.append( + EmbedTask( + job_id=job_id, + extraction_id=ext_id, + source_row=source_row, + entity_type="pain_point", + entity_index=i, + embed_text=text, + audience=aud_norm, + aspect=None, + opinion=None, + category=None, + sentiment=None, + content=content, + ) + ) + + for i, item in enumerate(feedback): + if not isinstance(item, dict): + continue + aspect = str(item.get("aspect", "")).strip() + opinion = str(item.get("opinion", "")).strip() + category = str(item.get("category", "")).strip() or None + sentiment = str(item.get("sentiment", "")).strip() or None + if not aspect and not opinion: + continue + if aspect: + tasks.append( + EmbedTask( + job_id=job_id, + extraction_id=ext_id, + source_row=source_row, + entity_type="aspect", + entity_index=i, + embed_text=aspect, + audience=aud_norm, + aspect=aspect, + opinion=opinion or None, + category=category, + sentiment=sentiment, + content=content, + ) + ) + if opinion: + tasks.append( + EmbedTask( + job_id=job_id, + extraction_id=ext_id, + source_row=source_row, + entity_type="opinion", + entity_index=i, + embed_text=opinion, + audience=aud_norm, + aspect=aspect or None, + opinion=opinion, + category=category, + sentiment=sentiment, + content=content, + ) + ) + if aspect and opinion: + merged = f"{aspect}, {opinion}" + tasks.append( + EmbedTask( + job_id=job_id, + extraction_id=ext_id, + source_row=source_row, + entity_type="aspect_opinion", + entity_index=i, + embed_text=merged, + audience=aud_norm, + aspect=aspect, + opinion=opinion, + category=category, + sentiment=sentiment, + content=content, + ) + ) + return tasks + + +def _embed_one_api_batch(api_key: str, texts: List[str]) -> List[bytes]: + """单次 API 调用（最多 batch_size 条文本）。""" + client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL) + resp = client.embeddings.create( + model=EMBEDDING_MODEL, + input=texts, + dimensions=EMBEDDING_DIMENSIONS, + ) + if len(resp.data) != len(texts): + raise RuntimeError( + f"embedding 返回条数 {len(resp.data)} != 请求 {len(texts)}" + ) + ordered = sorted(resp.data, key=lambda d: d.index) + blobs: List[bytes] = [] + for item in ordered: + vec = item.embedding + if len(vec) != EMBEDDING_DIMENSIONS: + raise RuntimeError(f"维度 {len(vec)} != 期望 {EMBEDDING_DIMENSIONS}") + blobs.append(_pack_embedding(vec)) + return blobs + + +def _embed_batches( + api_key: str, + tasks: List[EmbedTask], + *, + batch_size: int = EMBED_BATCH_SIZE, + workers: int = EMBED_DEFAULT_WORKERS, +) -> List[bytes]: + """ + 将任务切成每批最多 batch_size 条，并行请求 DashScope。 + 返回与 tasks 顺序一致的 embedding BLOB 列表。 + """ + if batch_size < 1 or batch_size > 10: + raise ValueError("batch_size 须在 1–10 之间（DashScope 单请求上限 10）") + workers = max(1, workers) + + chunks: List[List[EmbedTask]] = [ + tasks[i : i + batch_size] for i in range(0, len(tasks), batch_size) + ] + n_chunks = len(chunks) + if n_chunks == 0: + return [] + + logger.info( + "共 %s 条文本，%s 批（每批≤%s 条），并行 workers=%s", + len(tasks), + n_chunks, + batch_size, + min(workers, n_chunks), + ) + + # 单线程：逻辑简单，便于限流环境 + if workers == 1: + all_blobs: List[bytes] = [] + for i, chunk in enumerate(chunks, start=1): + blobs = _embed_one_api_batch(api_key, [t.embed_text for t in chunk]) + all_blobs.extend(blobs) + if i == n_chunks or i % 20 == 0: + logger.info("Embedding 进度 %s/%s 批", i, n_chunks) + return all_blobs + + results: List[List[bytes] | None] = [None] * n_chunks + done = 0 + with ThreadPoolExecutor(max_workers=min(workers, n_chunks)) as pool: + future_map = { + pool.submit( + _embed_one_api_batch, + api_key, + [t.embed_text for t in chunk], + ): idx + for idx, chunk in enumerate(chunks) + } + for fut in as_completed(future_map): + idx = future_map[fut] + results[idx] = fut.result() + done += 1 + if done == n_chunks or done % 20 == 0: + logger.info("Embedding 进度 %s/%s 批", done, n_chunks) + + return [blob for batch in results for blob in batch] # type: ignore[union-attr] + + +def run_embed( + *, + job_id: int | None = None, + csv_path: Path = DEFAULT_CSV, + structured_db: Path = STRUCTURED_DB, + embed_db: Path = EMBED_DB, + batch_size: int = EMBED_BATCH_SIZE, + workers: int = EMBED_DEFAULT_WORKERS, + reset_db: bool = True, +) -> Dict[str, Any]: + api_key = _resolve_api_key() + if not api_key: + raise RuntimeError( + "缺少 DASHSCOPE_API_KEY 或项目根 .dashscope_key" + ) + + csv_path = csv_path.expanduser().resolve() + if not csv_path.is_file(): + raise FileNotFoundError(csv_path) + if not structured_db.is_file(): + raise FileNotFoundError(structured_db) + + content_map = _load_content_by_source_row(csv_path) + sconn = sqlite3.connect(structured_db) + sconn.row_factory = sqlite3.Row + try: + jid = job_id if job_id is not None else _latest_job_id(sconn) + ext_rows = sconn.execute( + """ + SELECT id, source_row, extraction_json + FROM comment_extractions + WHERE job_id = ? + ORDER BY source_row + """, + (jid,), + ).fetchall() + if not ext_rows: + raise RuntimeError(f"job_id={jid} 无 comment_extractions 记录") + finally: + sconn.close() + + tasks = _build_tasks(jid, ext_rows, content_map) + if not tasks: + raise RuntimeError("没有可向量化的条目（检查过滤规则与结构化结果）") + + by_type: Dict[str, int] = {} + for t in tasks: + by_type[t.entity_type] = by_type.get(t.entity_type, 0) + 1 + logger.info( + "job_id=%s：%s 条评论结构化记录 -> %s 条向量任务 %s", + jid, + len(ext_rows), + len(tasks), + by_type, + ) + + blobs = _embed_batches( + api_key, + tasks, + batch_size=batch_size, + workers=workers, + ) + + econn = sqlite3.connect(embed_db) + try: + _init_embed_schema(econn) + if reset_db: + econn.execute("DELETE FROM embedding_items") + else: + econn.execute("DELETE FROM embedding_items WHERE job_id = ?", (jid,)) + econn.commit() + insert_sql = """ + INSERT INTO embedding_items ( + job_id, extraction_id, source_row, entity_type, entity_index, + embed_text, audience, aspect, opinion, category, sentiment, + content, dimensions, embedding + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """ + rows = [ + ( + task.job_id, + task.extraction_id, + task.source_row, + task.entity_type, + task.entity_index, + task.embed_text, + task.audience, + task.aspect, + task.opinion, + task.category, + task.sentiment, + task.content, + EMBEDDING_DIMENSIONS, + blob, + ) + for task, blob in zip(tasks, blobs) + ] + econn.executemany(insert_sql, rows) + econn.commit() + total = econn.execute("SELECT COUNT(*) FROM embedding_items").fetchone()[0] + finally: + econn.close() + + summary = { + "job_id": jid, + "structured_db": str(structured_db), + "embed_db": str(embed_db), + "csv": str(csv_path), + "model": EMBEDDING_MODEL, + "dimensions": EMBEDDING_DIMENSIONS, + "extractions": len(ext_rows), + "vectors": total, + "by_entity_type": by_type, + } + logger.info("完成：%s", summary) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 结构化结果向量化") + parser.add_argument("--job-id", type=int, default=None, help="默认最新 job") + parser.add_argument("--csv", type=Path, default=DEFAULT_CSV) + parser.add_argument("--structured-db", type=Path, default=STRUCTURED_DB) + parser.add_argument("--embed-db", type=Path, default=EMBED_DB) + parser.add_argument( + "--batch-size", + type=int, + default=EMBED_BATCH_SIZE, + help="每批 API 请求条数，最大 10（DashScope 限制）", + ) + parser.add_argument( + "--workers", + type=int, + default=EMBED_DEFAULT_WORKERS, + help="并行批次数；设为 1 则串行", + ) + args = parser.parse_args() + summary = run_embed( + job_id=args.job_id, + csv_path=args.csv, + structured_db=args.structured_db, + embed_db=args.embed_db, + batch_size=args.batch_size, + workers=args.workers, + ) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/结构化_Prompt.py b/结构化_Prompt.py new file mode 100644 index 0000000..c14e588 --- /dev/null +++ b/结构化_Prompt.py @@ -0,0 +1,59 @@ +""" +结构化提取 Prompt 入口（外置 prompts/ 目录，每次调用重读磁盘）。 + +对外 API 保持不变，供 结构化_server.py 动态加载。 +""" +from __future__ import annotations + +import json +from typing import List, Tuple + +from prompts.loader import ( + build_batch_extraction_system, + build_batch_extraction_user, + build_single_extraction_template, + format_examples, +) + + +def build_batch_review_analysis_prompts( + industry: str, + product_name: str, + keys: List[str], + tagged_input: str, +) -> Tuple[str, str]: + """ + 生成批量评论分析用的 (system_prompt, user_prompt)。 + tagged_input 形如多行 "[C1] ...\\n[C2] ..."；模型须输出仅含这些键的 JSON 对象。 + """ + keys_literal = ", ".join(json.dumps(k) for k in keys) + n_keys = len(keys) + system_prompt = build_batch_extraction_system( + industry, + product_name, + n_keys=n_keys, + keys_literal=keys_literal, + ) + user_prompt = build_batch_extraction_user( + n_keys=n_keys, + keys_literal=keys_literal, + tagged_input=tagged_input, + ) + return system_prompt, user_prompt + + +def generate_extraction_prompt_template(industry: str = "行业", product_name: str = "产品") -> str: + """ + 根据行业和产品名，动态生成适合大模型结构化提取的 Prompt 模版。 + 返回的字符串中包含 {review_content} 占位符，供后续批量处理时填入真实评论。 + """ + return build_single_extraction_template(industry, product_name) + + +if __name__ == "__main__": + general_prompt = generate_extraction_prompt_template( + industry="线上教育及宠物用品", + product_name="综合商品", + ) + print("============== Prompt 预览 ==============\n") + print(general_prompt) diff --git a/结构化_server.py b/结构化_server.py new file mode 100644 index 0000000..9c08697 --- /dev/null +++ b/结构化_server.py @@ -0,0 +1,1082 @@ +""" +VOC 评论结构化服务（命令行 / 直接调用，无 MCP）。 + +---------------------------------------------------------------------- +虚拟环境（推荐 Python 3.10+） + + cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化" + source 310py/bin/activate + pip install -r requirements.txt + +---------------------------------------------------------------------- +运行前提供密钥（任选其一；勿把密钥写进代码仓库）:: + + export DASHSCOPE_API_KEY="sk-xxx" + # 或项目根单行文件 .dashscope_key + # 或 export DASHSCOPE_API_KEY_FILE="/path/to/key.txt" + +---------------------------------------------------------------------- +用法:: + + python3 结构化_server.py --industry "Pet supplements" --product "Turkey tail mushroom for dogs" --file merged_reviews_cleaned.csv + + python3 结构化_server.py --smoke + +模型：默认 ``qwen3.6-flash``（OpenAI 兼容 Chat API；可通过环境变量 ``DASHSCOPE_MODEL`` 覆盖）。 + +地域：固定中国大陆华北2（北京）``https://dashscope.aliyuncs.com/compatible-mode/v1``，不使用新加坡/国际节点。 + +数据库：项目根目录 ``voc_structured.sqlite``；每次写入前会清理该库及下游 ``voc_embeddings.sqlite``、``voc_clustering.sqlite``（冒烟测试用临时库时不清理项目根文件）。 +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import logging +import os +import re +import sqlite3 +import sys +from csv import DictReader +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Tuple + +from prompts.loader import product_feedback_categories + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_structured") + +PROJECT_ROOT = Path(__file__).resolve().parent +DB_PATH = PROJECT_ROOT / "voc_structured.sqlite" +EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite" +CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite" +# 中国大陆华北2（北京）OpenAI 兼容模式 +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" +MODEL_NAME = os.environ.get("DASHSCOPE_MODEL", "qwen3.6-flash").strip() +REVIEW_COLUMN_NAMES = ("content", "review", "评论") +BATCH_FETCH_MAX_ATTEMPTS = 3 +BATCH_CORRECTION_MAX_ATTEMPTS = 3 + +_VALID_CATEGORIES = product_feedback_categories() # 模块加载时快照；校验时用 _get_valid_categories() + + +def _get_valid_categories() -> frozenset[str]: + return product_feedback_categories() + +# 模型上下文上限（保守估）；单批仍用下方 DEFAULT_MAX_* 控制，保证 JSON 稳定 +MODEL_MAX_INPUT_TOKENS = 991_800 +MODEL_MAX_OUTPUT_TOKENS = 65_530 +CHARS_PER_TOKEN_EST = 3.2 +DEFAULT_MAX_BATCH_INPUT_TOKENS = 32_000 +DEFAULT_MAX_BATCH_OUTPUT_TOKENS = 12_000 +DEFAULT_OUTPUT_TOKENS_PER_REVIEW = 450 +BATCH_COUNT_MIN = 1 +BATCH_COUNT_MAX = 50 +# 批量结构化单次请求输出 token（含 JSON 开销，上限不超过模型） +BATCH_OUTPUT_TOKEN_BUFFER = 1024 +BATCH_OUTPUT_TOKEN_FLOOR = 4096 + + +def _batch_max_output_tokens(review_count: int) -> int: + est = review_count * DEFAULT_OUTPUT_TOKENS_PER_REVIEW + BATCH_OUTPUT_TOKEN_BUFFER + return min(MODEL_MAX_OUTPUT_TOKENS, max(BATCH_OUTPUT_TOKEN_FLOOR, est)) + + +def _resolve_dashscope_api_key() -> str: + """环境变量 > DASHSCOPE_API_KEY_FILE > 项目根 .dashscope_key（均仅一行密钥，无引号）。""" + v = os.environ.get("DASHSCOPE_API_KEY", "").strip() + if v: + return v + fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip() + if fp: + p = Path(fp).expanduser() + if p.is_file(): + return p.read_text(encoding="utf-8").strip().strip('"').strip("'") + local = PROJECT_ROOT / ".dashscope_key" + if local.is_file(): + return local.read_text(encoding="utf-8").strip().strip('"').strip("'") + return "" + + +def _load_prompt_module(): + path = PROJECT_ROOT / "结构化_Prompt.py" + spec = importlib.util.spec_from_file_location("voc_structured_prompts", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot load prompt module from {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_PROMPTS = _load_prompt_module() + + +def _clean_voc_sqlite_databases(structured_db: Path) -> None: + """写入前删除 SQLite，避免与旧 job / 下游向量、聚类结果混用。""" + paths = [structured_db.resolve()] + if structured_db.resolve() == DB_PATH.resolve(): + paths.extend([EMBED_DB.resolve(), CLUSTER_DB.resolve()]) + for p in paths: + if p.is_file(): + p.unlink() + logger.info("已清理 SQLite: %s", p.name) + + +def init_sqlite_schema(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS analysis_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + industry TEXT NOT NULL, + product_name TEXT NOT NULL, + source_file TEXT NOT NULL, + batch_size INTEGER NOT NULL, + model TEXT NOT NULL, + full_result_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS comment_extractions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id INTEGER NOT NULL, + source_row INTEGER NOT NULL, + batch_index INTEGER NOT NULL, + comment_key TEXT NOT NULL, + extraction_json TEXT NOT NULL, + UNIQUE(job_id, source_row), + FOREIGN KEY (job_id) REFERENCES analysis_jobs(id) + ); + """ + ) + + +def load_reviews_from_file(file_path: str) -> List[Tuple[int, str]]: + """ + 返回 (source_row, text) 列表。 + CSV：source_row 为表头下的数据行序号（第 1 行数据=1），空单元格跳过。 + TXT：source_row 为文件物理行号（从 1 起），空行跳过。 + """ + p = Path(file_path).expanduser().resolve() + if not p.is_file(): + raise FileNotFoundError(f"File not found: {p}") + + suffix = p.suffix.lower() + if suffix == ".csv": + return _load_csv_reviews(p) + if suffix == ".txt" or suffix == ".text": + return _load_txt_reviews(p) + raise ValueError(f"Unsupported file type {suffix!r}; use .csv or .txt") + + +def _load_txt_reviews(p: Path) -> List[Tuple[int, str]]: + out: List[Tuple[int, str]] = [] + with p.open(encoding="utf-8-sig", newline="") as f: + for line_no, line in enumerate(f, start=1): + t = line.strip("\r\n").strip() + if not t: + continue + out.append((line_no, t)) + return out + + +def _load_csv_reviews(p: Path) -> List[Tuple[int, str]]: + with p.open(encoding="utf-8-sig", newline="") as f: + reader = DictReader(f) + if not reader.fieldnames: + raise ValueError("CSV has no header row") + headers = [h.strip() for h in reader.fieldnames] + lower_map = {h.lower(): h for h in headers} + col: str | None = None + for name in REVIEW_COLUMN_NAMES: + if name.lower() in lower_map: + col = lower_map[name.lower()] + break + if col is None: + if len(headers) == 1: + col = headers[0] + else: + raise ValueError( + "CSV must contain column 'content', 'review', or '评论', " + f"or be single-column. Found columns: {headers}" + ) + rows = list(reader) + out: List[Tuple[int, str]] = [] + for idx, row in enumerate(rows, start=1): + cell = row.get(col) + if cell is None: + cell = "" + t = str(cell).strip() + if not t: + continue + out.append((idx, t)) + return out + + +def _chunked(items: List[Tuple[int, str]], size: int) -> List[List[Tuple[int, str]]]: + return [items[i : i + size] for i in range(0, len(items), size)] + + +def _estimate_tokens(text: str) -> int: + """英文评论粗略估 token（偏保守，避免顶满上下文）。""" + return max(1, int(len(text) / CHARS_PER_TOKEN_EST)) + + +def _estimate_batch_input_tokens( + industry: str, + product_name: str, + batch: List[Tuple[int, str]], +) -> int: + tagged, keys, _ = _format_tagged_batch(batch) + system, user = _PROMPTS.build_batch_review_analysis_prompts( + industry=industry, + product_name=product_name, + keys=keys, + tagged_input=tagged, + ) + return _estimate_tokens(system) + _estimate_tokens(user) + + +def _estimate_batch_output_tokens(review_count: int) -> int: + return review_count * DEFAULT_OUTPUT_TOKENS_PER_REVIEW + + +def chunk_reviews_by_token_budget( + reviews: List[Tuple[int, str]], + industry: str, + product_name: str, + *, + max_input_tokens: int = DEFAULT_MAX_BATCH_INPUT_TOKENS, + max_output_tokens: int = DEFAULT_MAX_BATCH_OUTPUT_TOKENS, + max_reviews_per_batch: int = BATCH_COUNT_MAX, + min_reviews_per_batch: int = BATCH_COUNT_MIN, +) -> List[List[Tuple[int, str]]]: + """ + 按预估输入/输出 token 与单批条数上限，将评论动态打包。 + 短评可多条约一批，长评自动减少条数（单条超长则单独成批）。 + """ + max_input_tokens = min(max_input_tokens, MODEL_MAX_INPUT_TOKENS) + max_output_tokens = min(max_output_tokens, MODEL_MAX_OUTPUT_TOKENS) + max_reviews_per_batch = max(min_reviews_per_batch, max_reviews_per_batch) + + batches: List[List[Tuple[int, str]]] = [] + i = 0 + while i < len(reviews): + batch: List[Tuple[int, str]] = [] + while i < len(reviews): + candidate = batch + [reviews[i]] + if len(candidate) > max_reviews_per_batch: + break + + inp = _estimate_batch_input_tokens(industry, product_name, candidate) + out = _estimate_batch_output_tokens(len(candidate)) + + if batch and (inp > max_input_tokens or out > max_output_tokens): + break + + batch = candidate + i += 1 + + if inp > max_input_tokens: + logger.warning( + "单条评论过长（约 %s input tokens），单独成批: source_row=%s", + inp, + batch[0][0], + ) + break + + if not batch: + break + batches.append(batch) + + return batches + + +def _summarize_batch_plan( + batches: List[List[Tuple[int, str]]], + industry: str, + product_name: str, +) -> List[Dict[str, Any]]: + plan: List[Dict[str, Any]] = [] + for idx, batch in enumerate(batches): + chars = sum(len(t) for _, t in batch) + plan.append( + { + "batch_index": idx, + "review_count": len(batch), + "total_chars": chars, + "est_input_tokens": _estimate_batch_input_tokens( + industry, product_name, batch + ), + "est_output_tokens": _estimate_batch_output_tokens(len(batch)), + } + ) + return plan + + +def _format_tagged_batch(batch: List[Tuple[int, str]]) -> Tuple[str, List[str], Dict[str, int]]: + """返回 tagged 文本、键列表 C1..Cn、Ci -> source_row。""" + lines: List[str] = [] + keys: List[str] = [] + key_to_row: Dict[str, int] = {} + for i, (source_row, text) in enumerate(batch, start=1): + key = f"C{i}" + keys.append(key) + key_to_row[key] = source_row + safe = text.replace("\r\n", "\n").replace("\r", "\n") + lines.append(f"[{key}] {safe}") + return "\n".join(lines), keys, key_to_row + + +def _strip_code_fence(text: str) -> str: + t = text.strip() + if t.startswith("```"): + lines = t.split("\n") + if len(lines) >= 2 and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + t = "\n".join(lines).strip() + return t + + +def parse_model_json(raw: str) -> Any: + t = _strip_code_fence(raw) + try: + return json.loads(t) + except json.JSONDecodeError: + m = re.search(r"[\{\[][\s\S]*[\}\]]\s*$", t) + if not m: + raise + return json.loads(m.group(0)) + + +def parse_model_json_object(raw: str) -> Dict[str, Any]: + parsed = parse_model_json(raw) + if isinstance(parsed, dict): + return parsed + raise ValueError( + f"模型 JSON 应为对象，实际为 {type(parsed).__name__}；请使用批量对象格式 {{\"C1\": ...}}" + ) + + +def _call_dashscope_chat( + messages: List[Dict[str, str]], + *, + max_tokens: int | None = None, +) -> Any: + api_key = _resolve_dashscope_api_key() + if not api_key: + raise RuntimeError( + "Missing DashScope API key: set DASHSCOPE_API_KEY in the process environment, " + "or DASHSCOPE_API_KEY_FILE to a one-line key file, " + "or create a one-line file at project root: .dashscope_key" + ) + + from openai import OpenAI + + client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL) + extra_body: Dict[str, Any] = {} + model_lower = MODEL_NAME.lower() + if model_lower.startswith(("qwen3.6", "qwen3.5", "qwen3")) or model_lower.startswith( + "deepseek-v4" + ): + extra_body["enable_thinking"] = False + + out_tokens = max_tokens if max_tokens is not None else DEFAULT_MAX_BATCH_OUTPUT_TOKENS + out_tokens = min(MODEL_MAX_OUTPUT_TOKENS, max(BATCH_OUTPUT_TOKEN_FLOOR, out_tokens)) + + resp = client.chat.completions.create( + model=MODEL_NAME, + messages=messages, + response_format={"type": "json_object"}, + temperature=0.2, + max_tokens=out_tokens, + **({"extra_body": extra_body} if extra_body else {}), + ) + msg = resp.choices[0].message + choice = msg.content + if not choice and getattr(msg, "reasoning_content", None): + choice = msg.reasoning_content + if not choice: + raise RuntimeError("Empty model response") + return parse_model_json(choice) + + +def _call_dashscope_model(system: str, user: str) -> Dict[str, Any]: + return _call_dashscope_chat( + [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + ) + + +def _coalesce_batch_response(raw: Dict[str, Any], keys: List[str]) -> Dict[str, Any]: + """若模型把 C1..Cn 包在嵌套对象里，尝试展开到顶层。""" + if any(k in raw for k in keys): + return raw + for value in raw.values(): + if isinstance(value, dict) and any(k in value for k in keys): + return value + return raw + + +def _normalize_batch_response(raw: Any, keys: List[str]) -> Dict[str, Any]: + """ + 统一为 {C1: {...}, C2: {...}}。 + 兼容模型误返回 JSON 数组、嵌套对象或按序号的对象。 + """ + if isinstance(raw, list): + out: Dict[str, Any] = {} + for i, key in enumerate(keys): + if i >= len(raw): + break + item = raw[i] + if not isinstance(item, dict): + continue + if len(item) == 1: + sole_k, sole_v = next(iter(item.items())) + if sole_k in keys and isinstance(sole_v, dict): + out[sole_k] = sole_v + continue + if any(k in item for k in keys): + for k in keys: + if k in item and isinstance(item[k], dict): + out[k] = item[k] + continue + out[key] = item + if out: + return out + raise ValueError( + "模型返回 JSON 数组，但无法按顺序映射为 " + + ", ".join(keys[: min(len(raw), len(keys))]) + ) + + if not isinstance(raw, dict): + raise TypeError(f"批量 JSON 应为 object 或 array，实际为 {type(raw).__name__}") + + coalesced = _coalesce_batch_response(raw, keys) + if any(k in coalesced for k in keys): + return coalesced + + # 如 {"1": {...}, "2": {...}} 映射到 C1, C2 + remapped: Dict[str, Any] = {} + for i, key in enumerate(keys, start=1): + for alias in (str(i), f"C{i}"): + if alias in coalesced and isinstance(coalesced[alias], dict): + remapped[key] = coalesced[alias] + break + if remapped: + return remapped + + return coalesced + + +_SENTIMENT_CANONICAL = { + "positive": "Positive", + "negative": "Negative", + "neutral": "Neutral", +} + +# 模型常见误写 -> 合法 category（校验前自动映射，减少无效重试） +_CATEGORY_ALIASES: Dict[str, str] = { + "value": "Price", + "values": "Price", + "cost": "Price", + "pricing": "Price", + "value for money": "Price", + "worth": "Price", + "packaging": "Logistics", + "shipping": "Logistics", + "delivery": "Logistics", + "service": "Customer Service", + "customer support": "Customer Service", + "support": "Customer Service", + "design": "Appearance", + "look": "Appearance", + "performance": "Function", + "efficacy": "Function", + "effectiveness": "Function", +} + + +def _normalize_sentiment(raw: Any) -> str: + key = str(raw or "").strip().lower() + return _SENTIMENT_CANONICAL.get(key, "Neutral") + + +def _normalize_category(raw: Any) -> str | None: + valid = _get_valid_categories() + key = str(raw or "").strip() + if not key: + return None + if key in valid: + return key + mapped = _CATEGORY_ALIASES.get(key.lower()) + if mapped: + return mapped + lower = key.lower() + for v in valid: + if v.lower() == lower: + return v + return None + + +def _normalize_product_feedback(items: Any) -> List[Dict[str, str]]: + if not isinstance(items, list): + return [] + out: List[Dict[str, str]] = [] + for item in items: + if not isinstance(item, dict): + continue + aspect = str(item.get("aspect", "")).strip() + opinion = str(item.get("opinion", "")).strip() + category = _normalize_category(item.get("category")) + if aspect and opinion and category: + out.append( + { + "aspect": aspect, + "opinion": opinion, + "sentiment": _normalize_sentiment(item.get("sentiment")), + "category": category, + } + ) + return out + + +def _validate_extraction_strict(obj: Any, key_label: str = "") -> Dict[str, Any]: + """校验结构化结果；失败时抛出带键名的 ValueError，供回传模型修正。""" + prefix = f"{key_label}: " if key_label else "" + if not isinstance(obj, dict): + raise ValueError(f"{prefix}必须是 JSON 对象") + + for field in ("audience", "pain_points", "product_feedback"): + if field not in obj: + raise ValueError(f"{prefix}缺少必填字段 {field}") + + audience = obj.get("audience") + if not isinstance(audience, str) or not audience.strip(): + raise ValueError(f"{prefix}audience 必须为非空字符串") + + pain_points = obj.get("pain_points") + if not isinstance(pain_points, list): + raise ValueError(f"{prefix}pain_points 必须为数组") + for i, p in enumerate(pain_points): + if not isinstance(p, str) or not str(p).strip(): + raise ValueError(f"{prefix}pain_points[{i}] 必须为非空字符串") + + pf = obj.get("product_feedback") + if not isinstance(pf, list): + raise ValueError(f"{prefix}product_feedback 必须为数组") + for i, item in enumerate(pf): + if not isinstance(item, dict): + raise ValueError(f"{prefix}product_feedback[{i}] 必须是对象") + for sub in ("aspect", "opinion", "sentiment", "category"): + if not str(item.get(sub, "")).strip(): + raise ValueError(f"{prefix}product_feedback[{i}] 缺少或空的 {sub}") + sent_key = str(item.get("sentiment", "")).strip().lower() + if sent_key not in _SENTIMENT_CANONICAL: + raise ValueError( + f"{prefix}product_feedback[{i}].sentiment 必须为 " + "Positive、Negative 或 Neutral" + ) + raw_cat = str(item.get("category", "")).strip() + cat = _normalize_category(raw_cat) + if cat is None: + raise ValueError( + f"{prefix}product_feedback[{i}].category 非法: {raw_cat!r}，" + f"允许: {', '.join(sorted(_get_valid_categories()))}" + ) + + return _normalize_extraction(obj) + + +def _normalize_extraction(obj: Any) -> Dict[str, Any]: + """补齐缺省字段，避免模型漏写 pain_points / product_feedback 导致整条丢弃。""" + if not isinstance(obj, dict): + raise ValueError("Each extraction must be a JSON object") + + audience = obj.get("audience", "unknown") + if not isinstance(audience, str) or not str(audience).strip(): + audience = "unknown" + else: + audience = str(audience).strip() + + pain_points = obj.get("pain_points", []) + if pain_points is None: + pain_points = [] + if not isinstance(pain_points, list): + pain_points = [str(pain_points)] if str(pain_points).strip() else [] + pain_points = [str(p).strip() for p in pain_points if str(p).strip()] + + product_feedback = _normalize_product_feedback(obj.get("product_feedback", [])) + + missing = [ + f + for f in ("audience", "pain_points", "product_feedback") + if f not in obj + ] + if missing: + logger.info("Normalized missing fields %s in extraction", missing) + + return { + "audience": audience, + "pain_points": pain_points, + "product_feedback": product_feedback, + } + + +def _build_batch_correction_message(keys: List[str], errors: List[str]) -> str: + keys_literal = ", ".join(json.dumps(k) for k in keys) + err_block = "\n".join(f"- {e}" for e in errors) + return ( + "你上一次返回的 JSON 未通过服务端校验，请根据下列错误修正后重新输出。\n\n" + f"校验错误：\n{err_block}\n\n" + "要求：\n" + f"- 输出一个 JSON 对象，顶层键必须且仅能是：{keys_literal}\n" + "- 每个键的值必须包含 audience、pain_points、product_feedback\n" + "- product_feedback 每条须含 aspect、opinion、sentiment" + "（Positive/Negative/Neutral）、category（禁止 Value，性价比用 Price）\n" + "- 只输出一个 JSON 对象（不要用数组），不要 markdown 代码围栏或解释文字" + ) + + +def _parse_batch_with_model_correction( + system: str, + user: str, + sub_keys: List[str], + *, + batch_index: int, + attempt_label: str, +) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]], List[str]]: + """ + 调用模型并在校验失败时将错误信息附在对话中重试。 + 返回 (原始 JSON, 已通过校验的条目, 仍失败的错误列表)。 + """ + messages: List[Dict[str, str]] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + raw_obj: Dict[str, Any] = {} + ok: Dict[str, Dict[str, Any]] = {} + errors: List[str] = [] + + for correction in range(BATCH_CORRECTION_MAX_ATTEMPTS): + logger.info( + "请求模型 batch %s（%s），校验轮次 %s/%s…", + batch_index, + attempt_label, + correction + 1, + BATCH_CORRECTION_MAX_ATTEMPTS, + ) + try: + raw_parsed = _call_dashscope_chat( + messages, max_tokens=_batch_max_output_tokens(len(sub_keys)) + ) + raw_obj = _normalize_batch_response(raw_parsed, sub_keys) + except (RuntimeError, json.JSONDecodeError, ValueError, TypeError) as e: + errors = [f"模型响应无法解析为 JSON: {e}"] + ok = {} + if correction + 1 >= BATCH_CORRECTION_MAX_ATTEMPTS: + break + logger.warning( + "Batch %s parse error (correction %s): %s", + batch_index, + correction + 1, + e, + ) + messages.append( + { + "role": "assistant", + "content": json.dumps( + {"error": "invalid_json", "detail": str(e)}, + ensure_ascii=False, + ), + } + ) + messages.append( + { + "role": "user", + "content": _build_batch_correction_message(sub_keys, errors), + } + ) + continue + + ok = {} + errors = [] + for sub_ck in sub_keys: + if sub_ck not in raw_obj: + errors.append(f"键 {sub_ck}：响应 JSON 缺少该顶层键") + continue + try: + ok[sub_ck] = _validate_extraction_strict(raw_obj[sub_ck], sub_ck) + except ValueError as e: + errors.append(str(e)) + + if not errors: + return raw_obj, ok, [] + + if correction + 1 >= BATCH_CORRECTION_MAX_ATTEMPTS: + break + + logger.warning( + "Batch %s validation failed (correction %s/%s): %s", + batch_index, + correction + 1, + BATCH_CORRECTION_MAX_ATTEMPTS, + errors, + ) + messages.append( + {"role": "assistant", "content": json.dumps(raw_obj, ensure_ascii=False)} + ) + messages.append( + { + "role": "user", + "content": _build_batch_correction_message(sub_keys, errors), + } + ) + + return raw_obj, ok, errors + + +def _fetch_batch_extractions( + industry: str, + product_name: str, + batch: List[Tuple[int, str]], + b_idx: int, +) -> Dict[str, Any]: + """请求一批评论的结构化结果；对缺失/无效键自动缩小批次重试。""" + _, all_keys, key_to_row = _format_tagged_batch(batch) + row_to_text = {row: text for row, text in batch} + pending_keys: List[str] = list(all_keys) + merged: Dict[str, Any] = {} + + for attempt in range(BATCH_FETCH_MAX_ATTEMPTS): + if not pending_keys: + break + + sub_batch = [ + (key_to_row[ck], row_to_text[key_to_row[ck]]) for ck in pending_keys + ] + tagged, sub_keys, sub_key_to_row = _format_tagged_batch(sub_batch) + # 重试时子批次会重新编号为 C1..Cn，需映射回原始键名 + sub_to_orig = {sub_keys[i]: pending_keys[i] for i in range(len(sub_keys))} + + system, user = _PROMPTS.build_batch_review_analysis_prompts( + industry=industry, + product_name=product_name, + keys=sub_keys, + tagged_input=tagged, + ) + logger.info( + "请求模型 batch %s，第 %s 次尝试，本批 %s 条…", + b_idx, + attempt + 1, + len(sub_keys), + ) + _, ok_map, val_errors = _parse_batch_with_model_correction( + system, + user, + sub_keys, + batch_index=b_idx, + attempt_label=f"attempt {attempt + 1}", + ) + + next_pending: List[str] = [] + for sub_ck in sub_keys: + orig_ck = sub_to_orig[sub_ck] + if sub_ck in ok_map: + merged[orig_ck] = ok_map[sub_ck] + else: + next_pending.append(orig_ck) + + if val_errors: + logger.warning( + "Batch %s still has validation issues after correction (attempt %s): %s", + b_idx, + attempt + 1, + val_errors, + ) + + pending_keys = next_pending + if pending_keys and attempt + 1 < BATCH_FETCH_MAX_ATTEMPTS: + logger.info( + "Retrying %s review(s) in batch %s (attempt %s)", + len(pending_keys), + b_idx, + attempt + 2, + ) + + if pending_keys: + logger.warning( + "Batch %s: gave up on %s review(s) after %s attempts: %s", + b_idx, + len(pending_keys), + BATCH_FETCH_MAX_ATTEMPTS, + pending_keys, + ) + return merged + + +def run_analysis( + industry: str, + product_name: str, + file_path: str, + batch_size: int | None = None, + *, + clean_databases: bool = True, + max_batch_input_tokens: int = DEFAULT_MAX_BATCH_INPUT_TOKENS, + max_batch_output_tokens: int = DEFAULT_MAX_BATCH_OUTPUT_TOKENS, + max_reviews_per_batch: int = BATCH_COUNT_MAX, +) -> Dict[str, Any]: + src = str(Path(file_path).expanduser().resolve()) + reviews = load_reviews_from_file(src) + if not reviews: + raise ValueError("No non-empty reviews found in file") + + if batch_size is not None and batch_size > 0: + if batch_size < BATCH_COUNT_MIN: + raise ValueError(f"batch_size must be >= {BATCH_COUNT_MIN}, got {batch_size}") + batches = _chunked(reviews, batch_size) + batching_mode = "fixed" + batch_size_record = batch_size + else: + batches = chunk_reviews_by_token_budget( + reviews, + industry, + product_name, + max_input_tokens=max_batch_input_tokens, + max_output_tokens=max_batch_output_tokens, + max_reviews_per_batch=max_reviews_per_batch, + ) + batching_mode = "dynamic" + batch_size_record = max_reviews_per_batch + + batch_plan = _summarize_batch_plan(batches, industry, product_name) + total_batches = len(batches) + counts = [p["review_count"] for p in batch_plan] + logger.info( + "开始结构化：共 %s 条评论，模式=%s，共 %s 批，每批条数 min/med/max=%s/%s/%s", + len(reviews), + batching_mode, + total_batches, + min(counts) if counts else 0, + sorted(counts)[len(counts) // 2] if counts else 0, + max(counts) if counts else 0, + ) + for p in batch_plan: + logger.info( + " 批次 %s: %s 条, %s 字, 约 %s in / %s out tokens", + p["batch_index"] + 1, + p["review_count"], + p["total_chars"], + p["est_input_tokens"], + p["est_output_tokens"], + ) + extractions_by_row: Dict[str, Dict[str, Any]] = {} + batch_details: List[Dict[str, Any]] = [] + sqlite_rows: List[Tuple[int, int, str, Dict[str, Any]]] = [] + + for b_idx, batch in enumerate(batches): + logger.info("批次 %s/%s：处理 %s 条评论…", b_idx + 1, total_batches, len(batch)) + _, keys, key_to_row = _format_tagged_batch(batch) + raw_obj = _fetch_batch_extractions(industry, product_name, batch, b_idx) + logger.info( + "批次 %s/%s 完成：成功 %s/%s 条", + b_idx + 1, + total_batches, + len(raw_obj), + len(batch), + ) + + for ck in keys: + if ck not in raw_obj: + continue + validated = raw_obj[ck] + src_row = key_to_row[ck] + extractions_by_row[str(src_row)] = validated + sqlite_rows.append((src_row, b_idx, ck, validated)) + + batch_details.append( + { + "batch_index": b_idx, + "keys": keys, + "source_rows": [key_to_row[k] for k in keys], + "model_output": raw_obj, + } + ) + + created = datetime.now(timezone.utc).isoformat() + db_path = DB_PATH.resolve() + if clean_databases: + _clean_voc_sqlite_databases(db_path) + conn = sqlite3.connect(db_path) + try: + init_sqlite_schema(conn) + cur = conn.execute( + """ + INSERT INTO analysis_jobs + (created_at, industry, product_name, source_file, batch_size, model, full_result_json) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + created, + industry, + product_name, + src, + batch_size_record, + MODEL_NAME, + "{}", + ), + ) + job_id = int(cur.lastrowid) + + for src_row, b_idx, ck, payload in sqlite_rows: + conn.execute( + """ + INSERT INTO comment_extractions + (job_id, source_row, batch_index, comment_key, extraction_json) + VALUES (?, ?, ?, ?, ?) + """, + ( + job_id, + src_row, + b_idx, + ck, + json.dumps(payload, ensure_ascii=False), + ), + ) + + result: Dict[str, Any] = { + "job_id": job_id, + "sqlite_path": str(db_path), + "industry": industry, + "product_name": product_name, + "source_file": src, + "batching_mode": batching_mode, + "batch_size": batch_size_record, + "batch_plan": batch_plan, + "max_batch_input_tokens": max_batch_input_tokens, + "max_batch_output_tokens": max_batch_output_tokens, + "model": MODEL_NAME, + "extractions": extractions_by_row, + "batches": batch_details, + } + full_json = json.dumps(result, ensure_ascii=False) + conn.execute( + "UPDATE analysis_jobs SET full_result_json = ? WHERE id = ?", + (full_json, job_id), + ) + conn.commit() + finally: + conn.close() + + return result + + +def _run_smoke_test() -> None: + """ + 真实调用 DashScope 的冒烟：5 条英文评论、batch_size=5、一次模型请求。 + 使用临时目录下的 SQLite，不写项目根 voc_structured.sqlite。 + """ + import tempfile + + global DB_PATH + td = Path(tempfile.mkdtemp(prefix="voc_smoke_")) + csv_p = td / "reviews.csv" + csv_p.write_text( + "review\n" + "Soft ramp; zipper broke after one week.\n" + "Shipping box was crushed but product fine.\n" + "My senior dog uses it daily; worth the price.\n" + "Smells a bit chemical at first, smell fades.\n" + "Great for cats too, very stable.\n", + encoding="utf-8", + ) + saved_db = DB_PATH + DB_PATH = td / "smoke.sqlite" + try: + out = run_analysis( + industry="Pet supplies", + product_name="Pet bed ramp", + file_path=str(csv_p), + ) + slim = { + "ok": True, + "job_id": out.get("job_id"), + "model": out.get("model"), + "sqlite_path": out.get("sqlite_path"), + "extraction_row_keys": sorted(out.get("extractions", {}).keys()), + "extractions": out.get("extractions"), + } + print(json.dumps(slim, ensure_ascii=False, indent=2)) + finally: + DB_PATH = saved_db + + +def main() -> None: + parser = argparse.ArgumentParser( + description=f"VOC 评论结构化（{MODEL_NAME} + SQLite）" + ) + parser.add_argument("--industry", required=True, help="行业，如 Pet supplements") + parser.add_argument("--product", required=True, help="产品名") + parser.add_argument("--file", required=True, type=Path, help="评论 CSV/TXT 路径") + parser.add_argument( + "--batch-size", + type=int, + default=None, + help="固定每批条数；不指定则按评论长度与 token 预算动态分批", + ) + parser.add_argument( + "--max-batch-input-tokens", + type=int, + default=DEFAULT_MAX_BATCH_INPUT_TOKENS, + help=f"单批最大输入 token 估算上限（模型上限约 {MODEL_MAX_INPUT_TOKENS}）", + ) + parser.add_argument( + "--max-batch-output-tokens", + type=int, + default=DEFAULT_MAX_BATCH_OUTPUT_TOKENS, + help=f"单批最大输出 token 估算上限（模型上限约 {MODEL_MAX_OUTPUT_TOKENS}）", + ) + parser.add_argument( + "--max-reviews-per-batch", + type=int, + default=BATCH_COUNT_MAX, + help=f"动态模式下每批最多条数（短评可接近此值，默认 {BATCH_COUNT_MAX}）", + ) + parser.add_argument( + "-o", + "--output-json", + type=Path, + default=None, + help="可选：将完整结果 JSON 写入该文件", + ) + args = parser.parse_args() + + result = run_analysis( + industry=args.industry, + product_name=args.product, + file_path=str(args.file), + batch_size=args.batch_size, + max_batch_input_tokens=args.max_batch_input_tokens, + max_batch_output_tokens=args.max_batch_output_tokens, + max_reviews_per_batch=args.max_reviews_per_batch, + ) + text = json.dumps(result, ensure_ascii=False, indent=2) + if args.output_json: + args.output_json.write_text(text, encoding="utf-8") + logger.info("结果已写入 %s", args.output_json) + print(text) + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "--smoke": + _run_smoke_test() + else: + main() diff --git a/聚类.py b/聚类.py new file mode 100644 index 0000000..3f0a34f --- /dev/null +++ b/聚类.py @@ -0,0 +1,1226 @@ +""" +从 voc_embeddings.sqlite 读取向量，按五段流程 UMAP+HDBSCAN 聚类，结果写入 voc_clustering.sqlite。 + +流程： + 1. audience（LLM 自动调参 n_neighbors） + 2a. 前两 audience 簇各自独立：簇内 pain_point（LLM 自动调参） + 2b. 前两 audience 簇各自独立：簇内 aspect_opinion 按 Positive/Negative/Neutral 分桶聚类 + 3a. 全量 pain_point（LLM 自动调参） + 3b. 全量 aspect_opinion 按 Positive/Negative/Neutral 分桶聚类（各自 LLM 自动调参） + +规则： + - 聚类单元：每行 embedding_items + - n < 10：不跑 HDBSCAN，逐条独立簇标签（0..n-1） + - 步骤 1/2 的 -1：写入 SQL，不参与后续（步骤 1 的 -1 不进入步骤 2） + - 步骤 3 的 -1：写入 SQL，参与后续统计 + +用法:: + + python3 聚类.py + # 默认自动使用 voc_structured.sqlite 中最新 analysis_jobs.id（须已向量化） + python3 聚类.py --job-id 4 # 可选：手动指定 +""" +from __future__ import annotations + +import argparse +import csv +import json +import logging +import os +import random +import re +import sqlite3 +import struct +import sys +import time +import warnings +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import hdbscan +import numpy as np +import umap +from openai import OpenAI +from sklearn.metrics import silhouette_score + +# UMAP 设 random_state 时的单线程提示；轮廓系数在 extmath 中的 matmul 数值告警 +warnings.filterwarnings( + "ignore", + message=r"n_jobs value .* overridden .* by setting random_state", + category=UserWarning, + module=r"umap\.umap_", +) +warnings.filterwarnings( + "ignore", + category=RuntimeWarning, + module=r"sklearn\.utils\.extmath", +) + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_cluster") + +PROJECT_ROOT = Path(__file__).resolve().parent +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite" +CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite" +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" +LLM_MODEL = "qwen3.6-flash" + +UMAP_N_COMPONENTS = 30 +UMAP_MIN_DIST = 0.1 +UMAP_METRIC = "cosine" +UMAP_RANDOM_STATE = 42 +MIN_POINTS_FOR_HDBSCAN = 10 +# HDBSCAN 目标：非离群簇数不超过该值（通过提高 min_cluster_size 达成） +MAX_CLUSTERS = 20 + +VALID_SENTIMENTS: Tuple[str, ...] = ("Positive", "Negative", "Neutral") +_SENTIMENT_STAGE_SUFFIX: Dict[str, str] = { + "Positive": "positive", + "Negative": "negative", + "Neutral": "neutral", +} + +HDBSCAN_BASE = dict( + min_samples=1, + cluster_selection_method="eom", + prediction_data=True, +) + +INITIAL_N_NEIGHBORS = 10 +MAX_N_NEIGHBORS = 45 +CROSS_SIMILAR_RATIO_THRESHOLD = 0.10 +SILHOUETTE_STOP_THRESHOLD = 0.6 +SILHOUETTE_DECLINE_WINDOW = 8 # 连续 8 个轮廓值：后 7 个均小于第 1 个则停止 +SAMPLE_CAP = 30 +SAMPLE_RATIO = 0.6 + +# 步骤 2 未启用 LLM 调参时的保守默认（当前步骤 2 已启用自动调参） +STEP2_N_NEIGHBORS = 8 + +# 簇内去重评论数 / 本步骤参与聚类的去重评论总数 < 该比例则不写入库、不进入报告 +CLUSTER_MIN_REVIEW_RATIO = 0.10 + + +@dataclass +class EmbedRow: + id: int + job_id: int + extraction_id: int + source_row: int + entity_type: str + entity_index: int + embed_text: str + audience: str | None + aspect: str | None + opinion: str | None + category: str | None + sentiment: str | None + content: str + dimensions: int + embedding: np.ndarray + + +def _resolve_api_key() -> str: + v = os.environ.get("DASHSCOPE_API_KEY", "").strip() + if v: + return v + fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip() + if fp: + p = Path(fp).expanduser() + if p.is_file(): + return p.read_text(encoding="utf-8").strip().strip('"').strip("'") + local = PROJECT_ROOT / ".dashscope_key" + if local.is_file(): + return local.read_text(encoding="utf-8").strip().strip('"').strip("'") + return "" + + +def _unpack_embedding(blob: bytes, dimensions: int) -> np.ndarray: + n = dimensions + expected = n * 4 + if len(blob) != expected: + raise ValueError(f"BLOB 长度 {len(blob)} != {expected}（dim={n}）") + return np.array(struct.unpack(f"{n}f", blob), dtype=np.float32) + + +def _latest_job_id_from_structured(structured_db: Path) -> int | None: + """与 向量化.py 一致：analysis_jobs 表 id 最大者为最新 job。""" + if not structured_db.is_file(): + return None + conn = sqlite3.connect(structured_db) + try: + row = conn.execute( + "SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1" + ).fetchone() + return int(row[0]) if row else None + finally: + conn.close() + + +def _latest_job_id_from_embeddings(conn: sqlite3.Connection) -> int | None: + row = conn.execute( + "SELECT job_id FROM embedding_items ORDER BY job_id DESC LIMIT 1" + ).fetchone() + return int(row[0]) if row else None + + +def _count_embeddings(conn: sqlite3.Connection, job_id: int) -> int: + row = conn.execute( + "SELECT COUNT(*) FROM embedding_items WHERE job_id = ?", (job_id,) + ).fetchone() + return int(row[0]) if row else 0 + + +def _resolve_job_id( + job_id: int | None, + embed_conn: sqlite3.Connection, + structured_db: Path = STRUCTURED_DB, +) -> int: + """未传 job_id 时自动选最新结构化 job，并确认向量库中已有数据。""" + if job_id is not None: + jid = job_id + if _count_embeddings(embed_conn, jid) == 0: + raise RuntimeError( + f"job_id={jid} 在 {EMBED_DB.name} 中无向量，请先运行 向量化.py" + ) + logger.info("使用指定 job_id=%s（%s 条向量）", jid, _count_embeddings(embed_conn, jid)) + return jid + + jid = _latest_job_id_from_structured(structured_db) + source = "voc_structured.analysis_jobs" + if jid is None: + jid = _latest_job_id_from_embeddings(embed_conn) + source = "voc_embeddings.embedding_items" + if jid is None: + raise RuntimeError("无可用 job_id：请先运行 结构化_server.py 与 向量化.py") + + n = _count_embeddings(embed_conn, jid) + if n == 0: + latest_embed = _latest_job_id_from_embeddings(embed_conn) + if latest_embed is not None and latest_embed != jid: + logger.warning( + "结构化最新 job_id=%s 尚无向量，改用向量库最新 job_id=%s", + jid, + latest_embed, + ) + jid = latest_embed + n = _count_embeddings(embed_conn, jid) + source = "voc_embeddings.embedding_items（回退）" + else: + raise RuntimeError( + f"最新 job_id={jid} 在向量库中无数据，请先对应该 job 运行 向量化.py" + ) + + logger.info("自动选用最新 job_id=%s（来源: %s，%s 条向量）", jid, source, n) + return jid + + +def _load_embed_rows( + conn: sqlite3.Connection, job_id: int, entity_type: str +) -> List[EmbedRow]: + cur = conn.execute( + """ + SELECT id, job_id, extraction_id, source_row, entity_type, entity_index, + embed_text, audience, aspect, opinion, category, sentiment, + content, dimensions, embedding + FROM embedding_items + WHERE job_id = ? AND entity_type = ? + ORDER BY source_row, entity_index, id + """, + (job_id, entity_type), + ) + out: List[EmbedRow] = [] + for r in cur.fetchall(): + dims = int(r[13]) + out.append( + EmbedRow( + id=int(r[0]), + job_id=int(r[1]), + extraction_id=int(r[2]), + source_row=int(r[3]), + entity_type=str(r[4]), + entity_index=int(r[5]), + embed_text=str(r[6]), + audience=r[7], + aspect=r[8], + opinion=r[9], + category=r[10], + sentiment=r[11], + content=str(r[12] or ""), + dimensions=dims, + embedding=_unpack_embedding(r[14], dims), + ) + ) + return out + + +def _filter_by_source_rows( + rows: List[EmbedRow], allowed: set[int] +) -> List[EmbedRow]: + return [r for r in rows if r.source_row in allowed] + + +def _strict_sentiment(raw: str | None) -> str | None: + """仅接受 Positive / Negative / Neutral，其余丢弃。""" + if raw is None: + return None + s = str(raw).strip() + if s in VALID_SENTIMENTS: + return s + return None + + +def _filter_by_sentiment(rows: List[EmbedRow], sentiment: str) -> List[EmbedRow]: + return [r for r in rows if _strict_sentiment(r.sentiment) == sentiment] + + +def _stage_3b(sentiment: str) -> str: + return f"3b_aspect_opinion_{_SENTIMENT_STAGE_SUFFIX[sentiment]}" + + +def _stage_2b(sentiment: str, audience_cluster: int) -> str: + return f"2b_aspect_opinion_{_SENTIMENT_STAGE_SUFFIX[sentiment]}_audience_c{audience_cluster}" + + +def _cluster_aspect_opinion_sentiment_stages( + cconn: sqlite3.Connection, + run_id: int, + ao_rows: List[EmbedRow], + *, + stage_for: Any, + outlier_participates: bool, + parent_audience_cluster: int | None, + skipped_reason_prefix: str, + client: OpenAI, +) -> Dict[str, int]: + """对 aspect_opinion 按三档情感各跑一 stage，返回 stage -> 条数。""" + counts: Dict[str, int] = {} + for sentiment in VALID_SENTIMENTS: + stage = stage_for(sentiment) + sub = _filter_by_sentiment(ao_rows, sentiment) + logger.info( + "[%s] %s 条 aspect_opinion（情感=%s）", + stage, + len(sub), + sentiment, + ) + if not sub: + _save_stage_meta( + cconn, + run_id, + stage, + { + "skipped": True, + "reason": f"{skipped_reason_prefix}_{_SENTIMENT_STAGE_SUFFIX[sentiment]}", + "sentiment": sentiment, + }, + ) + counts[stage] = 0 + continue + labels, meta = _cluster_stage( + sub, stage=stage, use_llm_tune=True, client=client + ) + parent_clusters = None + if parent_audience_cluster is not None: + parent_clusters = [parent_audience_cluster] * len(sub) + filt = _save_assignments( + cconn, + run_id, + stage, + sub, + labels, + n_neighbors=meta.get("n_neighbors"), + outlier_participates=outlier_participates, + parent_clusters=parent_clusters, + ) + meta["cluster_filter"] = filt + meta["sentiment"] = sentiment + _save_stage_meta(cconn, run_id, stage, meta) + if meta.get("tuning_log"): + _save_tuning_logs(cconn, run_id, stage, meta["tuning_log"]) + counts[stage] = len(sub) + return counts + + +def _source_rows_for_audience_cluster( + row_to_aud: Dict[int, int], audience_cluster: int +) -> set[int]: + return {sr for sr, lab in row_to_aud.items() if lab == audience_cluster} + + +def _stage_name_step2_pain(audience_cluster: int) -> str: + return f"2a_pain_audience_c{audience_cluster}" + + +def _cluster_step2_for_audience( + cconn: sqlite3.Connection, + run_id: int, + *, + audience_cluster: int, + pain_rows: List[EmbedRow], + ao_rows: List[EmbedRow], + row_to_aud: Dict[int, int], + client: OpenAI, +) -> Dict[str, int]: + """对单个 audience 簇分别聚类 pain 与 aspect_opinion，返回各 stage 条数。""" + allowed = _source_rows_for_audience_cluster(row_to_aud, audience_cluster) + counts: Dict[str, int] = {} + pain_sub = _filter_by_source_rows(pain_rows, allowed) + ao_sub = _filter_by_source_rows(ao_rows, allowed) + + stage_pain = _stage_name_step2_pain(audience_cluster) + logger.info( + "[%s] audience 簇 %s：%s 条 pain（%s 条评论）", + stage_pain, + audience_cluster, + len(pain_sub), + len(allowed), + ) + if pain_sub: + labels_pain, meta_pain = _cluster_stage( + pain_sub, stage=stage_pain, use_llm_tune=True, client=client + ) + filt = _save_assignments( + cconn, + run_id, + stage_pain, + pain_sub, + labels_pain, + n_neighbors=meta_pain.get("n_neighbors"), + outlier_participates=False, + parent_clusters=[audience_cluster] * len(pain_sub), + ) + meta_pain["cluster_filter"] = filt + _save_stage_meta(cconn, run_id, stage_pain, meta_pain) + if meta_pain.get("tuning_log"): + _save_tuning_logs(cconn, run_id, stage_pain, meta_pain["tuning_log"]) + counts[stage_pain] = len(pain_sub) + else: + _save_stage_meta( + cconn, + run_id, + stage_pain, + {"skipped": True, "reason": "no_pain_points", "audience_cluster": audience_cluster}, + ) + counts[stage_pain] = 0 + + ao_counts = _cluster_aspect_opinion_sentiment_stages( + cconn, + run_id, + ao_sub, + stage_for=lambda s: _stage_2b(s, audience_cluster), + outlier_participates=False, + parent_audience_cluster=audience_cluster, + skipped_reason_prefix="no_aspect_opinion", + client=client, + ) + counts.update(ao_counts) + + return counts + + +def _adaptive_min_cluster_size(n: int) -> int: + """初始 min_cluster_size = max(2, n // MAX_CLUSTERS)，无上限；簇数仍 > MAX_CLUSTERS 时再迭代增大。""" + if n < MIN_POINTS_FOR_HDBSCAN: + return 2 + return max(2, n // MAX_CLUSTERS) + + +def _n_clusters_from_labels(labels: np.ndarray) -> int: + labs = {int(x) for x in labels.tolist()} + labs.discard(-1) + return len(labs) + + +def _stack_embeddings(rows: Sequence[EmbedRow]) -> np.ndarray: + return np.vstack([r.embedding for r in rows]).astype(np.float32) + + +def _fit_umap(embeddings: np.ndarray, n_neighbors: int) -> np.ndarray: + n = len(embeddings) + n_neighbors = min(n_neighbors, max(2, n - 1)) + reducer = umap.UMAP( + n_components=min(UMAP_N_COMPONENTS, max(2, n - 2)), + n_neighbors=n_neighbors, + min_dist=UMAP_MIN_DIST, + metric=UMAP_METRIC, + random_state=UMAP_RANDOM_STATE, + ) + return reducer.fit_transform(embeddings) + + +def _fit_hdbscan(umap_emb: np.ndarray, min_cluster_size: int) -> np.ndarray: + clusterer = hdbscan.HDBSCAN( + min_cluster_size=min_cluster_size, + **HDBSCAN_BASE, + ) + return clusterer.fit_predict(umap_emb) + + +def _run_umap_hdbscan_capped( + embeddings: np.ndarray, + n_neighbors: int, + *, + max_clusters: int = MAX_CLUSTERS, +) -> Tuple[np.ndarray, np.ndarray, int]: + """UMAP + HDBSCAN；若簇数 > max_clusters 则增大 min_cluster_size 直至满足或无法再增。""" + n = len(embeddings) + umap_emb = _fit_umap(embeddings, n_neighbors) + min_cs = _adaptive_min_cluster_size(n) + labels = _fit_hdbscan(umap_emb, min_cs) + n_clusters = _n_clusters_from_labels(labels) + + while n_clusters > max_clusters and min_cs < n: + step = max(1, (n_clusters - max_clusters + 1) // 2) + next_cs = min(min_cs + step, n) + if next_cs <= min_cs: + break + logger.info( + "簇数 %s > %s，min_cluster_size %s -> %s", + n_clusters, + max_clusters, + min_cs, + next_cs, + ) + min_cs = next_cs + labels = _fit_hdbscan(umap_emb, min_cs) + n_clusters = _n_clusters_from_labels(labels) + + if n_clusters > max_clusters: + logger.warning( + "簇数 %s 仍 > %s（min_cluster_size=%s, n=%s）", + n_clusters, + max_clusters, + min_cs, + n, + ) + return labels, umap_emb, min_cs + + +def _labels_all_zero(n: int) -> np.ndarray: + return np.zeros(n, dtype=int) + + +def _strip_think(text: str) -> str: + if not text: + return text + text = re.sub( + r"[\s\S]*?", "", text, flags=re.IGNORECASE + ) + text = re.sub(r"", "", text, flags=re.IGNORECASE) + return text.strip() + + +def _parse_json_from_llm(text: str) -> dict: + text = _strip_think(text or "") + text = text.replace("```json", "").replace("```JSON", "").replace("```", "").strip() + start = text.find("{") + end = text.rfind("}") + if start == -1 or end <= start: + raise ValueError("未找到 JSON 对象") + return json.loads(text[start : end + 1]) + + +def _sample_from_clusters( + sentences: List[str], cluster_labels: np.ndarray, rng: random.Random +) -> Dict[int, List[str]]: + groups: Dict[int, List[str]] = {} + for sent, lab in zip(sentences, cluster_labels.tolist()): + groups.setdefault(int(lab), []).append(sent) + samples: Dict[int, List[str]] = {} + for lab, sents in groups.items(): + if lab == -1: + continue + n = len(sents) + k = min(SAMPLE_CAP, max(1, int(n * SAMPLE_RATIO))) + k = min(k, n) + samples[lab] = rng.sample(sents, k) if n > k else list(sents) + return samples + + +def _ai_evaluate_cluster_samples( + client: OpenAI, cluster_samples: Dict[int, List[str]], max_retries: int = 2 +) -> dict: + if not cluster_samples: + return {"cross_similar_count": 0, "total_sentences": 0} + lines = [] + total = 0 + for lab in sorted(cluster_samples.keys()): + lines.append(f"【聚类{lab}】") + for i, sent in enumerate(cluster_samples[lab], 1): + lines.append(f" {i}. {sent}") + total += 1 + sample_text = "\n".join(lines) + prompt = f"""你是 VOC 评论短语聚类质量评估助手。以下是多个聚类类别的抽样短语。 + +{sample_text} + +请统计 cross_similar_count：不同聚类类别之间、语义相似的短语条数（每句最多计 1）。 + +只输出 JSON： +{{ + "cross_similar_count": 整数, + "total_sentences": {total} +}}""" + for attempt in range(max_retries): + try: + resp = client.chat.completions.create( + model=LLM_MODEL, + messages=[ + {"role": "system", "content": "只输出合法 JSON。"}, + {"role": "user", "content": prompt}, + ], + max_tokens=1500, + temperature=0.0, + extra_body={"enable_thinking": False}, + ) + data = _parse_json_from_llm(resp.choices[0].message.content) + data["total_sentences"] = int(data.get("total_sentences", total) or total) + data["cross_similar_count"] = int(data.get("cross_similar_count", 0)) + return data + except Exception as e: + logger.warning("AI 评估失败(第%s次): %s", attempt + 1, e) + if attempt < max_retries - 1: + time.sleep(1) + return {"cross_similar_count": total, "total_sentences": total} + + +def _silhouette_decline_should_stop(scores: List[float]) -> bool: + """最近 SILHOUETTE_DECLINE_WINDOW 个有效轮廓系数中，后 N-1 个是否都严格小于第一个。""" + if len(scores) < SILHOUETTE_DECLINE_WINDOW: + return False + window = scores[-SILHOUETTE_DECLINE_WINDOW:] + first = window[0] + return all(s < first for s in window[1:]) + + +def _best_silhouette_in_window( + snapshots: List[Tuple[int, np.ndarray, float]], +) -> Tuple[int, np.ndarray, float] | None: + """在最近轮廓窗口内取轮廓系数最高的一轮；无快照时返回 None。""" + if not snapshots: + return None + pool = snapshots[-SILHOUETTE_DECLINE_WINDOW:] + return max(pool, key=lambda x: x[2]) + + +def _auto_tune_n_neighbors( + embeddings: np.ndarray, + sentences: List[str], + client: OpenAI, + stage: str, +) -> Tuple[np.ndarray, int, List[dict], Optional[float], int]: + rng = random.Random(UMAP_RANDOM_STATE) + n_neighbors = INITIAL_N_NEIGHBORS + tuning_log: List[dict] = [] + labels: np.ndarray | None = None + min_cs = 2 + silhouette_avg: float | None = None + silhouette_history: List[float] = [] + # (n_neighbors, labels, silhouette) 仅在有有效轮廓时入栈，供早停回退最优轮次 + silhouette_snapshots: List[Tuple[int, np.ndarray, float]] = [] + n = len(embeddings) + + while True: + logger.info("[%s] 调参轮次 n_neighbors=%s", stage, n_neighbors) + labels, umap_emb, min_cs = _run_umap_hdbscan_capped(embeddings, n_neighbors) + n_clusters = _n_clusters_from_labels(labels) + n_noise = int((labels == -1).sum()) + silhouette_avg = None + if n_clusters > 1: + mask = labels != -1 + if int(mask.sum()) > 1 and len(set(labels[mask].tolist())) > 1: + silhouette_avg = float( + silhouette_score(umap_emb[mask], labels[mask]) + ) + logger.info( + "[%s] 簇数=%s 离群=%s 轮廓=%s", + stage, + n_clusters, + n_noise, + f"{silhouette_avg:.4f}" if silhouette_avg is not None else "N/A", + ) + + samples = _sample_from_clusters(sentences, labels, rng) + total_sample = sum(len(v) for v in samples.values()) + log_entry: dict = { + "stage": stage, + "n_neighbors": n_neighbors, + "min_cluster_size": min_cs, + "n_clusters": n_clusters, + "n_noise": n_noise, + "silhouette": silhouette_avg, + "total_sample": total_sample, + } + + if silhouette_avg is not None: + silhouette_history.append(silhouette_avg) + silhouette_snapshots.append((n_neighbors, labels.copy(), silhouette_avg)) + if _silhouette_decline_should_stop(silhouette_history): + window = silhouette_history[-SILHOUETTE_DECLINE_WINDOW:] + best = _best_silhouette_in_window(silhouette_snapshots) + assert best is not None + best_nn, best_labels, best_sil = best + n_neighbors = best_nn + labels = best_labels + silhouette_avg = best_sil + log_entry["silhouette_window"] = [round(s, 4) for s in window] + log_entry["stop_reason"] = ( + f"连续{SILHOUETTE_DECLINE_WINDOW}轮轮廓：后5个均低于窗口首值" + f"{window[0]:.4f}，回退至 n_neighbors={best_nn}（轮廓{best_sil:.4f}）" + ) + tuning_log.append(log_entry) + logger.info( + "[%s] 轮廓窗口 %s，早停并回退 n_neighbors=%s 轮廓=%.4f", + stage, + log_entry["silhouette_window"], + best_nn, + best_sil, + ) + break + + if n_neighbors > MAX_N_NEIGHBORS: + best = _best_silhouette_in_window(silhouette_snapshots) + if best is not None: + best_nn, best_labels, best_sil = best + n_neighbors = best_nn + labels = best_labels + silhouette_avg = best_sil + window = silhouette_history[-SILHOUETTE_DECLINE_WINDOW:] + log_entry["silhouette_window"] = [round(s, 4) for s in window] + log_entry["stop_reason"] = ( + f"n_neighbors 超过上限 {MAX_N_NEIGHBORS}，" + f"回退至 n_neighbors={best_nn}（轮廓{best_sil:.4f}）" + ) + logger.info( + "[%s] n_neighbors 超上限，回退 n_neighbors=%s 轮廓=%.4f", + stage, + best_nn, + best_sil, + ) + else: + log_entry["stop_reason"] = ( + f"n_neighbors 超过上限 {MAX_N_NEIGHBORS}（无有效轮廓快照，保留当前轮）" + ) + tuning_log.append(log_entry) + break + + ai_result = _ai_evaluate_cluster_samples(client, samples) + cross_count = ai_result["cross_similar_count"] + total = ai_result["total_sentences"] or total_sample or 1 + ratio = cross_count / total + log_entry.update( + { + "cross_similar_count": cross_count, + "cross_similar_ratio": round(ratio, 4), + } + ) + sil_ok = silhouette_avg is not None and silhouette_avg > SILHOUETTE_STOP_THRESHOLD + logger.info( + "[%s] AI: 跨类相似 %s/%s (%.1f%%，阈值≤%.0f%%) 轮廓=%s (停止需>%.1f)", + stage, + cross_count, + total, + ratio * 100, + CROSS_SIMILAR_RATIO_THRESHOLD * 100, + f"{silhouette_avg:.4f}" if silhouette_avg is not None else "N/A", + SILHOUETTE_STOP_THRESHOLD, + ) + + if ratio > CROSS_SIMILAR_RATIO_THRESHOLD: + log_entry["stop_reason"] = "跨类相似率>10%，继续调参" + tuning_log.append(log_entry) + n_neighbors += 1 + time.sleep(0.3) + continue + + if not sil_ok: + reason = ( + "轮廓系数不可用，继续调参" + if silhouette_avg is None + else f"轮廓系数{silhouette_avg:.4f}≤{SILHOUETTE_STOP_THRESHOLD}，继续调参" + ) + log_entry["stop_reason"] = reason + tuning_log.append(log_entry) + n_neighbors += 1 + time.sleep(0.3) + continue + + log_entry["stop_reason"] = "跨类相似率≤10%且轮廓>0.6，停止调参" + tuning_log.append(log_entry) + break + + assert labels is not None + return labels, n_neighbors, tuning_log, silhouette_avg, min_cs + + +def _cluster_fixed( + embeddings: np.ndarray, n_neighbors: int = STEP2_N_NEIGHBORS +) -> Tuple[np.ndarray, int, int]: + nn = min(n_neighbors, max(2, len(embeddings) - 1)) + labels, _, min_cs = _run_umap_hdbscan_capped(embeddings, nn) + return labels, nn, min_cs + + +def _cluster_stage( + rows: List[EmbedRow], + *, + stage: str, + use_llm_tune: bool, + client: OpenAI | None, +) -> Tuple[np.ndarray, dict]: + n = len(rows) + meta: dict = { + "stage": stage, + "n_items": n, + "used_hdbscan": False, + "n_neighbors": None, + "tuning_log": [], + "silhouette": None, + } + if n == 0: + return np.array([], dtype=int), meta + if n < MIN_POINTS_FOR_HDBSCAN: + logger.info( + "[%s] n=%s < %s，逐条独立簇标签，不跑 HDBSCAN", + stage, + n, + MIN_POINTS_FOR_HDBSCAN, + ) + meta["fallback"] = "per_item_labels_no_hdbscan" + return np.arange(n, dtype=int), meta + + embeddings = _stack_embeddings(rows) + sentences = [r.embed_text for r in rows] + meta["used_hdbscan"] = True + + if use_llm_tune: + if client is None: + raise RuntimeError(f"{stage} 需要 LLM 客户端") + labels, nn, tlog, sil, min_cs = _auto_tune_n_neighbors( + embeddings, sentences, client, stage + ) + meta["n_neighbors"] = nn + meta["min_cluster_size"] = min_cs + meta["max_clusters"] = MAX_CLUSTERS + meta["tuning_log"] = tlog + meta["silhouette"] = sil + else: + labels, nn, min_cs = _cluster_fixed(embeddings) + meta["n_neighbors"] = nn + meta["min_cluster_size"] = min_cs + meta["max_clusters"] = MAX_CLUSTERS + meta["fallback"] = "fixed_params" + + return labels, meta + + +def _init_cluster_db(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS cluster_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + job_id INTEGER NOT NULL, + embed_db TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS cluster_assignments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + stage TEXT NOT NULL, + embedding_item_id INTEGER NOT NULL, + cluster_label INTEGER NOT NULL, + is_outlier INTEGER NOT NULL DEFAULT 0, + participates_downstream INTEGER NOT NULL DEFAULT 1, + parent_audience_cluster INTEGER, + n_neighbors INTEGER, + embed_text TEXT NOT NULL, + entity_type TEXT NOT NULL, + source_row INTEGER NOT NULL, + extraction_id INTEGER NOT NULL, + entity_index INTEGER NOT NULL, + audience TEXT, + aspect TEXT, + opinion TEXT, + category TEXT, + sentiment TEXT, + content TEXT NOT NULL, + FOREIGN KEY (run_id) REFERENCES cluster_runs(id) + ); + CREATE TABLE IF NOT EXISTS cluster_stage_meta ( + run_id INTEGER NOT NULL, + stage TEXT NOT NULL, + meta_json TEXT NOT NULL, + PRIMARY KEY (run_id, stage) + ); + CREATE TABLE IF NOT EXISTS cluster_tuning_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id INTEGER NOT NULL, + stage TEXT NOT NULL, + round_index INTEGER NOT NULL, + log_json TEXT NOT NULL, + FOREIGN KEY (run_id) REFERENCES cluster_runs(id) + ); + CREATE INDEX IF NOT EXISTS idx_assign_run_stage ON cluster_assignments(run_id, stage); + CREATE INDEX IF NOT EXISTS idx_assign_cluster ON cluster_assignments(run_id, stage, cluster_label); + CREATE INDEX IF NOT EXISTS idx_assign_source ON cluster_assignments(run_id, source_row); + """ + ) + + +def _resolve_source_csv(structured_db: Path, job_id: int) -> Path: + conn = sqlite3.connect(structured_db) + try: + row = conn.execute( + "SELECT source_file FROM analysis_jobs WHERE id = ?", (job_id,) + ).fetchone() + finally: + conn.close() + if not row: + raise RuntimeError(f"analysis_jobs 无 job_id={job_id}") + p = Path(str(row[0])) + if p.is_file(): + return p.resolve() + cand = PROJECT_ROOT / p + if cand.is_file(): + return cand.resolve() + raise FileNotFoundError(f"找不到结构化来源 CSV: {row[0]}") + + +def _count_csv_reviews(csv_path: Path) -> int: + n = 0 + with csv_path.open(encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + if (row.get("content") or "").strip(): + n += 1 + return n + + +def _keep_cluster_labels( + labels: np.ndarray, + source_rows: Sequence[int], + total_reviews: int, + *, + min_ratio: float = CLUSTER_MIN_REVIEW_RATIO, +) -> Tuple[set[int], Dict[int, int]]: + """返回 (保留的簇标签, 被丢弃簇标签 -> 去重评论数)。""" + by_lab: Dict[int, set[int]] = defaultdict(set) + for lab, sr in zip(labels.tolist(), source_rows): + by_lab[int(lab)].add(int(sr)) + if total_reviews <= 0: + return set(by_lab.keys()), {} + kept: set[int] = set() + dropped: Dict[int, int] = {} + for lab, srs in by_lab.items(): + cnt = len(srs) + if cnt / total_reviews >= min_ratio: + kept.add(lab) + else: + dropped[lab] = cnt + return kept, dropped + + +def _reset_cluster_db(path: Path, *, reset: bool = True) -> sqlite3.Connection: + if reset and path.is_file(): + path.unlink() + conn = sqlite3.connect(path) + _init_cluster_db(conn) + return conn + + +def _save_assignments( + conn: sqlite3.Connection, + run_id: int, + stage: str, + rows: List[EmbedRow], + labels: np.ndarray, + *, + n_neighbors: int | None, + outlier_participates: bool, + parent_clusters: List[int | None] | None = None, +) -> dict: + if parent_clusters is not None and len(parent_clusters) != len(rows): + raise ValueError("parent_clusters 长度与 rows 不一致") + source_rows = [r.source_row for r in rows] + stage_total_reviews = len(set(source_rows)) + kept, dropped = _keep_cluster_labels(labels, source_rows, stage_total_reviews) + if dropped: + logger.info( + "[%s] 过滤小簇（去重评论占比 < %.0f%%，本步骤评论总数 %s）：%s", + stage, + CLUSTER_MIN_REVIEW_RATIO * 100, + stage_total_reviews, + { + lab: f"{cnt}条({cnt / stage_total_reviews * 100:.1f}%)" + for lab, cnt in sorted(dropped.items()) + if stage_total_reviews > 0 + }, + ) + sql = """ + INSERT INTO cluster_assignments ( + run_id, stage, embedding_item_id, cluster_label, is_outlier, + participates_downstream, parent_audience_cluster, n_neighbors, + embed_text, entity_type, source_row, extraction_id, entity_index, + audience, aspect, opinion, category, sentiment, content + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """ + saved = 0 + for i, row in enumerate(rows): + lab = int(labels[i]) + if lab not in kept: + continue + is_out = 1 if lab == -1 else 0 + if is_out: + participates = 1 if outlier_participates else 0 + else: + participates = 1 + parent = None + if parent_clusters is not None: + parent = parent_clusters[i] + conn.execute( + sql, + ( + run_id, + stage, + row.id, + lab, + is_out, + participates, + parent, + n_neighbors, + row.embed_text, + row.entity_type, + row.source_row, + row.extraction_id, + row.entity_index, + row.audience, + row.aspect, + row.opinion, + row.category, + row.sentiment, + row.content, + ), + ) + saved += 1 + return { + "kept_clusters": sorted(kept), + "dropped_clusters": dropped, + "saved_rows": saved, + "min_review_ratio": CLUSTER_MIN_REVIEW_RATIO, + "stage_total_reviews": stage_total_reviews, + } + + +def _save_stage_meta(conn: sqlite3.Connection, run_id: int, stage: str, meta: dict) -> None: + conn.execute( + """ + INSERT OR REPLACE INTO cluster_stage_meta (run_id, stage, meta_json) + VALUES (?, ?, ?) + """, + (run_id, stage, json.dumps(meta, ensure_ascii=False)), + ) + + +def _save_tuning_logs( + conn: sqlite3.Connection, run_id: int, stage: str, tuning_log: List[dict] +) -> None: + for i, entry in enumerate(tuning_log, start=1): + conn.execute( + """ + INSERT INTO cluster_tuning_log (run_id, stage, round_index, log_json) + VALUES (?, ?, ?, ?) + """, + (run_id, stage, i, json.dumps(entry, ensure_ascii=False)), + ) + + +def _top2_audience_clusters( + conn: sqlite3.Connection, run_id: int +) -> Tuple[List[int], Dict[int, int]]: + """返回 (前两簇标签列表, source_row -> audience簇标签)。""" + cur = conn.execute( + """ + SELECT cluster_label, source_row + FROM cluster_assignments + WHERE run_id = ? AND stage = '1_audience' + AND is_outlier = 0 AND participates_downstream = 1 + """, + (run_id,), + ) + row_to_cluster: Dict[int, int] = {} + cluster_rows: Dict[int, set[int]] = {} + for lab, sr in cur.fetchall(): + lab = int(lab) + sr = int(sr) + row_to_cluster[sr] = lab + cluster_rows.setdefault(lab, set()).add(sr) + ranked = sorted( + cluster_rows.items(), key=lambda x: len(x[1]), reverse=True + ) + top2 = [lab for lab, _ in ranked[:2]] + return top2, row_to_cluster + + +def run_clustering( + *, + job_id: int | None = None, + structured_db: Path = STRUCTURED_DB, + embed_db: Path = EMBED_DB, + cluster_db: Path = CLUSTER_DB, + reset_db: bool = True, +) -> dict: + api_key = _resolve_api_key() + if not api_key: + raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key") + + econn = sqlite3.connect(embed_db) + try: + jid = _resolve_job_id(job_id, econn, structured_db) + audience_rows = _load_embed_rows(econn, jid, "audience") + pain_rows = _load_embed_rows(econn, jid, "pain_point") + ao_rows = _load_embed_rows(econn, jid, "aspect_opinion") + finally: + econn.close() + + client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL) + cconn = _reset_cluster_db(cluster_db, reset=reset_db) + created = datetime.now(timezone.utc).isoformat() + try: + cur = cconn.execute( + "INSERT INTO cluster_runs (created_at, job_id, embed_db) VALUES (?,?,?)", + (created, jid, str(embed_db.resolve())), + ) + run_id = int(cur.lastrowid) + + csv_path = _resolve_source_csv(structured_db, jid) + total_reviews = _count_csv_reviews(csv_path) + logger.info("清洗后评论总数: %s", total_reviews) + + # --- 1 audience --- + labels1, meta1 = _cluster_stage( + audience_rows, stage="1_audience", use_llm_tune=True, client=client + ) + filt1 = _save_assignments( + cconn, + run_id, + "1_audience", + audience_rows, + labels1, + n_neighbors=meta1.get("n_neighbors"), + outlier_participates=False, + ) + meta1["cluster_filter"] = filt1 + _save_stage_meta(cconn, run_id, "1_audience", meta1) + if meta1.get("tuning_log"): + _save_tuning_logs(cconn, run_id, "1_audience", meta1["tuning_log"]) + + top2, row_to_aud = _top2_audience_clusters(cconn, run_id) + meta_top2 = { + "top2_audience_clusters": top2, + "per_cluster_source_rows": { + str(lab): len(_source_rows_for_audience_cluster(row_to_aud, lab)) + for lab in top2 + }, + "step2_mode": "per_audience_cluster_llm_auto_tune", + } + _save_stage_meta(cconn, run_id, "step2_filter", meta_top2) + logger.info("Audience 前两簇（将分别聚类）: %s", top2) + + step2_counts: Dict[str, int] = {} + for aud_lab in top2: + step2_counts.update( + _cluster_step2_for_audience( + cconn, + run_id, + audience_cluster=aud_lab, + pain_rows=pain_rows, + ao_rows=ao_rows, + row_to_aud=row_to_aud, + client=client, + ) + ) + + # --- 3a pain global --- + labels3a, meta3a = _cluster_stage( + pain_rows, stage="3a_pain_global", use_llm_tune=True, client=client + ) + filt3a = _save_assignments( + cconn, + run_id, + "3a_pain_global", + pain_rows, + labels3a, + n_neighbors=meta3a.get("n_neighbors"), + outlier_participates=True, + ) + meta3a["cluster_filter"] = filt3a + _save_stage_meta(cconn, run_id, "3a_pain_global", meta3a) + if meta3a.get("tuning_log"): + _save_tuning_logs(cconn, run_id, "3a_pain_global", meta3a["tuning_log"]) + + # --- 3b aspect_opinion by sentiment --- + step3b_counts = _cluster_aspect_opinion_sentiment_stages( + cconn, + run_id, + ao_rows, + stage_for=_stage_3b, + outlier_participates=True, + parent_audience_cluster=None, + skipped_reason_prefix="no_aspect_opinion", + client=client, + ) + + cconn.commit() + summary = { + "run_id": run_id, + "job_id": jid, + "total_reviews": total_reviews, + "min_cluster_review_ratio": CLUSTER_MIN_REVIEW_RATIO, + "cluster_db": str(cluster_db.resolve()), + "top2_audience_clusters": top2, + "counts": { + "1_audience": len(audience_rows), + **step2_counts, + "3a_pain_global": len(pain_rows), + **step3b_counts, + }, + } + logger.info("聚类完成: %s", summary) + return summary + finally: + cconn.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 向量聚类") + parser.add_argument( + "--job-id", + type=int, + default=None, + help="不指定则自动使用 voc_structured.sqlite 中最新 analysis_jobs.id", + ) + parser.add_argument("--structured-db", type=Path, default=STRUCTURED_DB) + parser.add_argument("--embed-db", type=Path, default=EMBED_DB) + parser.add_argument("--cluster-db", type=Path, default=CLUSTER_DB) + args = parser.parse_args() + summary = run_clustering( + job_id=args.job_id, + structured_db=args.structured_db, + embed_db=args.embed_db, + cluster_db=args.cluster_db, + ) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/词频.py b/词频.py new file mode 100644 index 0000000..058b29b --- /dev/null +++ b/词频.py @@ -0,0 +1,608 @@ +""" +词频统计：从最新结构化任务读取产品与 CSV，两步连跑。 + + 1. 随机 25 条 content → LLM 归纳「产品专有名词」与「Amazon/产品专属停用词」 + 2. spaCy 全量 content 分词 + 词频 → output/word_freq.csv（专有名词按完整短语统计，不拆词） + +用法:: + + python3 词频词云.py + python3 词频词云.py --skip-llm # 复用 output/voc_terms.json，仅跑第 2 步 +""" +from __future__ import annotations + +import argparse +import csv +import json +import logging +import os +import random +import re +import sqlite3 +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Dict, Iterable, List, Sequence, Set, Tuple + +from openai import OpenAI +from spacy.lang.en.stop_words import STOP_WORDS as EN_STOP_WORDS + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_wordfreq") + +PROJECT_ROOT = Path(__file__).resolve().parent +STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite" +OUTPUT_DIR = PROJECT_ROOT / "output" +TERMS_JSON = OUTPUT_DIR / "voc_terms.json" +WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv" + +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1" +MODEL_NAME = os.environ.get("DASHSCOPE_MODEL", "qwen3.6-flash").strip() + +SAMPLE_SIZE = 25 +SAMPLE_SEED = 42 + + +def _resolve_api_key() -> str: + v = os.environ.get("DASHSCOPE_API_KEY", "").strip() + if v: + return v + fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip() + if fp: + p = Path(fp).expanduser() + if p.is_file(): + return p.read_text(encoding="utf-8").strip().strip('"').strip("'") + local = PROJECT_ROOT / ".dashscope_key" + if local.is_file(): + return local.read_text(encoding="utf-8").strip().strip('"').strip("'") + return "" + + +def _strip_think(text: str) -> str: + if not text: + return text + text = re.sub( + r"[\s\S]*?", "", text, flags=re.IGNORECASE + ) + text = re.sub(r"", "", text, flags=re.IGNORECASE) + return text.strip() + + +def _latest_job(conn: sqlite3.Connection) -> Tuple[int, str, str, str]: + row = conn.execute( + """ + SELECT id, industry, product_name, source_file + FROM analysis_jobs + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + if not row: + raise RuntimeError(f"{STRUCTURED_DB} 中无 analysis_jobs 记录，请先跑结构化") + return int(row[0]), str(row[1]), str(row[2]), str(row[3]) + + +def _resolve_source_path(source_file: str) -> Path: + p = Path(source_file) + if p.is_file(): + return p.resolve() + candidate = PROJECT_ROOT / source_file + if candidate.is_file(): + return candidate.resolve() + raise FileNotFoundError(f"找不到评论 CSV: {source_file}") + + +def _load_contents(csv_path: Path) -> List[Tuple[int, str]]: + rows: List[Tuple[int, str]] = [] + with csv_path.open(encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + if not reader.fieldnames or "content" not in reader.fieldnames: + raise ValueError(f"{csv_path} 缺少 content 列") + for i, row in enumerate(reader, start=1): + text = (row.get("content") or "").strip() + if text: + rows.append((i, text)) + if not rows: + raise ValueError(f"{csv_path} 无有效 content") + return rows + + +def _sample_reviews( + rows: Sequence[Tuple[int, str]], *, n: int, seed: int +) -> List[Tuple[int, str]]: + rng = random.Random(seed) + k = min(n, len(rows)) + return rng.sample(list(rows), k) + + +def _build_terms_prompt( + industry: str, product_name: str, samples: Sequence[Tuple[int, str]] +) -> Tuple[str, str]: + blocks = [] + for row_id, text in samples: + blocks.append(f"[R{row_id}]\n{text}") + joined = "\n\n---\n\n".join(blocks) + system = ( + "你是亚马逊美国站英文评论与电商文本分析专家。" + "根据样本评论，列出两类词表，供后续英文分词与词频统计使用。" + ) + user = f"""行业：{industry} +产品：{product_name} + +以下为该产品 {len(samples)} 条随机评论（仅正文 content）： + +{joined} + +请输出两部分（可用 Markdown 标题、编号列表或逗号分隔，不必是 JSON）： + +## 产品专有名词（product_terms） +列出该产品评论中可能出现的**全部**专有表达，尽量合理扩展，包括但不限于： +成分/品类、宠物品种与病种、剂型包装、用法场景、Amazon 物流售后相关但**属于本产品语境**的词等。 +多词短语请保留完整形式（如 turkey tail mushroom powder）；后续词频**只统计完整短语**，不会拆成单词分别计数。 +**不要**把下面「专属停用词」里的词放进本列表。 + +## 专属停用词（stopwords） +列出 Amazon 美国站电商通用词，以及与本产品强相关、词频高但**分析价值低**的词（如泛化评价词、过泛产品描述词等）。 +多词短语可保留。 +注意：不要把产品名「{product_name}」**整句**列为停用词；也不要把其中的**实词/品类成分**（如 turkey、tail、mushroom）列为停用词。 +产品名拆词里**无分析价值的虚词/功能词**（如 for、the、a、of、and）可以且应列入停用词。 + +每条一行或逗号分隔即可。""" + return system, user + + +def _call_llm(system: str, user: str, api_key: str) -> str: + client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL) + extra_body: Dict[str, Any] = {} + if MODEL_NAME.lower().startswith(("qwen3.6", "qwen3.5", "qwen3")): + extra_body["enable_thinking"] = False + resp = client.chat.completions.create( + model=MODEL_NAME, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + temperature=0.3, + **({"extra_body": extra_body} if extra_body else {}), + ) + msg = resp.choices[0].message + text = msg.content or getattr(msg, "reasoning_content", None) or "" + if not text.strip(): + raise RuntimeError("LLM 返回为空") + return _strip_think(text) + + +def _normalize_term(s: str) -> str: + s = re.sub(r"\s+", " ", (s or "").strip().lower()) + return s.strip(".,;:\"'""''`-–—()[]{}") + + +def _split_list_items(line: str) -> List[str]: + line = re.sub(r"^[\s\d\.\)\-•\*]+", "", line.strip()) + if not line or line.startswith("#"): + return [] + parts = re.split(r"[,，;；、|]", line) + return [_normalize_term(p) for p in parts if _normalize_term(p)] + + +def _try_parse_json_terms(text: str) -> Tuple[List[str], List[str]] | None: + text = text.replace("```json", "").replace("```", "").strip() + start, end = text.find("{"), text.rfind("}") + if start == -1 or end <= start: + return None + try: + obj = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + return None + if not isinstance(obj, dict): + return None + + def pick(keys: Sequence[str]) -> List[str]: + for k in keys: + v = obj.get(k) + if isinstance(v, list): + return [_normalize_term(str(x)) for x in v if _normalize_term(str(x))] + if isinstance(v, str): + return _split_list_items(v) + return [] + + terms = pick(("product_terms", "专有名词", "terms", "keywords")) + stops = pick(("stopwords", "stop_words", "停用词", "custom_stopwords")) + if terms or stops: + return terms, stops + return None + + +def _parse_section_lines(text: str, mode: str) -> List[str]: + """mode: product | stop""" + lines = text.splitlines() + collecting = False + items: List[str] = [] + product_hdr = re.compile( + r"(专有名词|product[_\s-]*terms?|product\s+terms)", re.I + ) + stop_hdr = re.compile( + r"(专属停用|停用词|stop[_\s-]*words?|custom\s+stop)", re.I + ) + other_hdr = re.compile(r"^#{1,3}\s+") + + for line in lines: + raw = line.strip() + if not raw: + if collecting and items: + collecting = False + continue + if mode == "product" and product_hdr.search(raw): + collecting = True + inline = re.split(r"[:：]", raw, maxsplit=1) + if len(inline) > 1: + items.extend(_split_list_items(inline[1])) + continue + if mode == "stop" and stop_hdr.search(raw): + collecting = True + inline = re.split(r"[:：]", raw, maxsplit=1) + if len(inline) > 1: + items.extend(_split_list_items(inline[1])) + continue + if collecting and (other_hdr.match(raw) or ( + mode == "product" and stop_hdr.search(raw) + ) or (mode == "stop" and product_hdr.search(raw))): + collecting = False + continue + if collecting: + items.extend(_split_list_items(raw)) + return items + + +def _parse_llm_terms(text: str) -> Tuple[List[str], List[str]]: + parsed = _try_parse_json_terms(text) + if parsed: + return parsed + product = _parse_section_lines(text, "product") + stop = _parse_section_lines(text, "stop") + if not product and not stop: + # 兜底：按行提取非空短语 + for line in text.splitlines(): + items = _split_list_items(line) + if items and len(items) <= 8: + product.extend(items) + return _dedupe_terms(product), _dedupe_terms(stop) + + +def _dedupe_terms(items: Iterable[str]) -> List[str]: + seen: Set[str] = set() + out: List[str] = [] + for x in items: + if x and x not in seen: + seen.add(x) + out.append(x) + return out + + +def _product_name_tokens(product_name: str) -> Set[str]: + return { + t + for t in re.findall(r"[a-z0-9]+", product_name.lower()) + if t + } + + +# 产品标题拆词中可视为「不重要」、应参与停用的功能词（含 spaCy 英文停用词交集） +_PRODUCT_NAME_FILLER = frozenset( + { + "a", "an", "the", "and", "or", "but", "for", "nor", "so", "yet", + "at", "by", "in", "of", "on", "to", "up", "as", "is", "it", "be", + "with", "from", "into", "via", "per", "vs", "vs.", + } +) + + +def _product_name_filler_tokens(product_name: str) -> Set[str]: + """产品名拆词中的虚词/功能词 → 应停用。""" + tokens = _product_name_tokens(product_name) + return {t for t in tokens if t in EN_STOP_WORDS or t in _PRODUCT_NAME_FILLER} + + +def _product_name_core_tokens(product_name: str) -> Set[str]: + """产品名中有分析价值的实词 → 不停用。""" + fillers = _product_name_filler_tokens(product_name) + return _product_name_tokens(product_name) - fillers + + +def _is_pure_number(s: str) -> bool: + return bool(s) and s.isdigit() + + +def _finalize_term_lists( + product_terms: List[str], + stopwords: List[str], + product_name: str, +) -> Tuple[List[str], Set[str]]: + stop_set = set(EN_STOP_WORDS) + stop_set.update(_dedupe_terms(stopwords)) + # 产品名虚词（for/the/a 等）纳入停用；实词成分保持可统计 + stop_set.update(_product_name_filler_tokens(product_name)) + for tok in _product_name_core_tokens(product_name): + stop_set.discard(tok) + pn_lower = _normalize_term(product_name) + if pn_lower: + stop_set.discard(pn_lower) + + cleaned_terms: List[str] = [] + for t in _dedupe_terms(product_terms): + if t in stop_set: + continue + cleaned_terms.append(t) + # 长短语优先匹配 + cleaned_terms.sort(key=lambda x: (-len(x.split()), -len(x))) + return cleaned_terms, stop_set + + +def _load_spacy(): + import spacy + + try: + return spacy.load("en_core_web_sm", disable=["ner", "parser"]) + except OSError as e: + raise RuntimeError( + "未安装 spaCy 英文模型，请执行: python3 -m spacy download en_core_web_sm" + ) from e + + +def _phrase_pattern(phrase: str) -> re.Pattern[str]: + parts = [re.escape(p) for p in phrase.split()] + body = r"\s+".join(parts) + return re.compile(rf"(? List[Tuple[int, int]]: + if not spans: + return [] + ordered = sorted(spans) + merged: List[Tuple[int, int]] = [ordered[0]] + for start, end in ordered[1:]: + prev_s, prev_e = merged[-1] + if start <= prev_e: + merged[-1] = (prev_s, max(prev_e, end)) + else: + merged.append((start, end)) + return merged + + +def _overlaps_span(char_start: int, char_end: int, spans: Sequence[Tuple[int, int]]) -> bool: + for s, e in spans: + if char_start < e and char_end > s: + return True + return False + + +def _apply_product_terms( + lower: str, + product_terms: List[str], + counter: Counter[str], + stop_set: Set[str], +) -> List[Tuple[int, int]]: + """匹配专有名词短语：只计完整短语频次，并返回需屏蔽拆词统计的字符区间。""" + spans: List[Tuple[int, int]] = [] + for phrase in product_terms: + if not phrase or phrase in stop_set or _is_pure_number(phrase): + continue + pat = _phrase_pattern(phrase) + hits = 0 + for m in pat.finditer(lower): + spans.append((m.start(), m.end())) + hits += 1 + if hits: + counter[phrase] += hits + return _merge_spans(spans) + + +def _tokenize_doc( + nlp, text: str, protected_spans: Sequence[Tuple[int, int]] +) -> List[str]: + doc = nlp(text) + tokens: List[str] = [] + for tok in doc: + if tok.is_space or tok.is_punct: + continue + char_start = tok.idx + char_end = tok.idx + len(tok.text) + if _overlaps_span(char_start, char_end, protected_spans): + continue + lemma = (tok.lemma_ or tok.text).lower().strip() + if not lemma or _is_pure_number(lemma): + continue + if not re.search(r"[a-z]", lemma, re.I): + continue + tokens.append(lemma) + return tokens + + +_SINGLE_EN_WORD = re.compile(r"^[a-z]+$") + + +def _should_lemma_normalize(word: str) -> bool: + """仅对单个英文词做词形还原；多词短语、连字符短语保持原样。""" + w = word.strip().lower() + if not w or " " in w or "-" in w or "'" in w: + return False + return bool(_SINGLE_EN_WORD.match(w)) + + +def _lemma_form(word: str, nlp) -> str: + w = word.strip().lower() + if not _should_lemma_normalize(w): + return w + doc = nlp(w) + if not doc: + return w + tok = doc[0] + if tok.is_space or tok.is_punct: + return w + lemma = (tok.lemma_ or tok.text).lower().strip() + if not lemma or lemma == "-": + return w + return lemma + + +def _merge_word_forms(counter: Counter[str], nlp) -> Counter[str]: + """写入 CSV 前合并单复数/时态等词形（如 dogs→dog, bought→buy）。""" + merged: Counter[str] = Counter() + for word, count in counter.items(): + merged[_lemma_form(word, nlp)] += count + return merged + + +def _build_word_freq( + rows: Sequence[Tuple[int, str]], + product_terms: List[str], + stop_set: Set[str], +) -> Counter[str]: + nlp = _load_spacy() + counter: Counter[str] = Counter() + for _, text in rows: + if not text.strip(): + continue + lower = text.lower() + protected = _apply_product_terms(lower, product_terms, counter, stop_set) + for tok in _tokenize_doc(nlp, text, protected): + if tok in stop_set: + continue + counter[tok] += 1 + counter = Counter({k: v for k, v in counter.items() if not _is_pure_number(k)}) + return _merge_word_forms(counter, nlp) + + +def _save_word_freq(counter: Counter[str], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as f: + w = csv.writer(f) + w.writerow(["word", "count"]) + for word, count in counter.most_common(): + w.writerow([word, count]) + +def _step1_extract_terms( + *, + job_id: int, + industry: str, + product_name: str, + samples: List[Tuple[int, str]], + api_key: str, +) -> dict: + system, user = _build_terms_prompt(industry, product_name, samples) + logger.info("第 1 步：调用 %s 提取专有名词与停用词（样本 %s 条）", MODEL_NAME, len(samples)) + raw = _call_llm(system, user, api_key) + product_raw, stop_raw = _parse_llm_terms(raw) + product_terms, _stop_set = _finalize_term_lists( + product_raw, stop_raw, product_name + ) + logger.info( + "解析得到专有名词 %s 条、专属停用词 %s 条", + len(product_terms), + len(stop_raw), + ) + payload = { + "job_id": job_id, + "industry": industry, + "product_name": product_name, + "sample_size": len(samples), + "sample_seed": SAMPLE_SEED, + "sample_row_ids": [r for r, _ in samples], + "model": MODEL_NAME, + "product_terms": product_terms, + "stopwords_custom": _dedupe_terms(stop_raw), + "llm_raw": raw, + } + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + TERMS_JSON.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) + logger.info("已保存 %s", TERMS_JSON) + return payload + + +def _load_terms_json() -> dict: + if not TERMS_JSON.is_file(): + raise FileNotFoundError(f"缺少 {TERMS_JSON}，请先运行完整流程或去掉 --skip-llm") + return json.loads(TERMS_JSON.read_text(encoding="utf-8")) + + +def run(*, skip_llm: bool = False) -> dict: + if not STRUCTURED_DB.is_file(): + raise FileNotFoundError(f"缺少 {STRUCTURED_DB}") + + conn = sqlite3.connect(STRUCTURED_DB) + try: + job_id, industry, product_name, source_file = _latest_job(conn) + finally: + conn.close() + + csv_path = _resolve_source_path(source_file) + rows = _load_contents(csv_path) + logger.info( + "job_id=%s product=%r 评论 %s 条，来源 %s", + job_id, + product_name, + len(rows), + csv_path.name, + ) + + if skip_llm: + meta = _load_terms_json() + product_terms, stop_set = _finalize_term_lists( + list(meta.get("product_terms") or []), + list(meta.get("stopwords_custom") or []), + product_name, + ) + logger.info("第 1 步跳过，复用 %s", TERMS_JSON) + else: + api_key = _resolve_api_key() + if not api_key: + raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key") + samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED) + meta = _step1_extract_terms( + job_id=job_id, + industry=industry, + product_name=product_name, + samples=samples, + api_key=api_key, + ) + product_terms, stop_set = _finalize_term_lists( + meta["product_terms"], + meta.get("stopwords_custom") or [], + product_name, + ) + + logger.info("第 2 步：spaCy 全量分词与词频统计") + counter = _build_word_freq(rows, product_terms, stop_set) + _save_word_freq(counter, WORD_FREQ_CSV) + logger.info("已写入 %s", WORD_FREQ_CSV) + + return { + "job_id": job_id, + "product_name": product_name, + "terms_json": str(TERMS_JSON), + "word_freq_csv": str(WORD_FREQ_CSV), + "unique_words": len(counter), + "total_tokens": sum(counter.values()), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 词频统计") + parser.add_argument( + "--skip-llm", + action="store_true", + help="跳过第 1 步，使用 output/voc_terms.json", + ) + args = parser.parse_args() + result = run(skip_llm=args.skip_llm) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/词频_jieba.py b/词频_jieba.py new file mode 100644 index 0000000..af597e0 --- /dev/null +++ b/词频_jieba.py @@ -0,0 +1,182 @@ +""" +词频统计（jieba 分词版）：流程与 词频.py 相同，第 2 步改用 jieba 分词。 + + 1. 随机样本 → LLM 归纳专有名词与停用词（与 spaCy 版共用 voc_terms.json） + 2. jieba 全量 content 分词 + 词频 → output/word_freq.csv + +用法:: + + python3 词频_jieba.py + python3 词频_jieba.py --skip-llm +""" +from __future__ import annotations + +import argparse +import json +import logging +import re +import sys +from collections import Counter +from pathlib import Path +from typing import List, Sequence, Set, Tuple + +from 词频 import ( + OUTPUT_DIR, + SAMPLE_SEED, + SAMPLE_SIZE, + STRUCTURED_DB, + TERMS_JSON, + WORD_FREQ_CSV, + _apply_product_terms, + _finalize_term_lists, + _is_pure_number, + _latest_job, + _load_contents, + _load_spacy, + _load_terms_json, + _merge_word_forms, + _overlaps_span, + _resolve_api_key, + _resolve_source_path, + _sample_reviews, + _save_word_freq, + _step1_extract_terms, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr, +) +logger = logging.getLogger("voc_wordfreq_jieba") + +# 英文词或中文词均保留 +_TOKEN_OK = re.compile(r"[a-z\u4e00-\u9fff]", re.I) + + +def _prepare_jieba(product_terms: Sequence[str]) -> None: + import jieba + + for term in product_terms: + t = (term or "").strip() + if t: + jieba.add_word(t) + + +def _tokenize_jieba( + text: str, + protected_spans: Sequence[Tuple[int, int]], + stop_set: Set[str], +) -> List[str]: + import jieba + + tokens: List[str] = [] + for word, start, end in jieba.tokenize(text): + if _overlaps_span(start, end, protected_spans): + continue + w = word.strip().lower() + if not w or _is_pure_number(w) or w in stop_set: + continue + if not _TOKEN_OK.search(w): + continue + tokens.append(w) + return tokens + + +def _build_word_freq( + rows: Sequence[Tuple[int, str]], + product_terms: List[str], + stop_set: Set[str], +) -> Counter[str]: + _prepare_jieba(product_terms) + counter: Counter[str] = Counter() + for _, text in rows: + if not text.strip(): + continue + lower = text.lower() + protected = _apply_product_terms(lower, product_terms, counter, stop_set) + for tok in _tokenize_jieba(text, protected, stop_set): + counter[tok] += 1 + counter = Counter({k: v for k, v in counter.items() if not _is_pure_number(k)}) + nlp = _load_spacy() + return _merge_word_forms(counter, nlp) + + +def run(*, skip_llm: bool = False) -> dict: + if not STRUCTURED_DB.is_file(): + raise FileNotFoundError(f"缺少 {STRUCTURED_DB}") + + import sqlite3 + + conn = sqlite3.connect(STRUCTURED_DB) + try: + job_id, industry, product_name, source_file = _latest_job(conn) + finally: + conn.close() + + csv_path = _resolve_source_path(source_file) + rows = _load_contents(csv_path) + logger.info( + "job_id=%s product=%r 评论 %s 条，来源 %s（jieba 分词）", + job_id, + product_name, + len(rows), + csv_path.name, + ) + + if skip_llm: + meta = _load_terms_json() + product_terms, stop_set = _finalize_term_lists( + list(meta.get("product_terms") or []), + list(meta.get("stopwords_custom") or []), + product_name, + ) + logger.info("第 1 步跳过，复用 %s", TERMS_JSON) + else: + api_key = _resolve_api_key() + if not api_key: + raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key") + samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED) + meta = _step1_extract_terms( + job_id=job_id, + industry=industry, + product_name=product_name, + samples=samples, + api_key=api_key, + ) + product_terms, stop_set = _finalize_term_lists( + meta["product_terms"], + meta.get("stopwords_custom") or [], + product_name, + ) + + logger.info("第 2 步：jieba 全量分词与词频统计") + counter = _build_word_freq(rows, product_terms, stop_set) + _save_word_freq(counter, WORD_FREQ_CSV) + logger.info("已写入 %s", WORD_FREQ_CSV) + + return { + "job_id": job_id, + "product_name": product_name, + "tokenizer": "jieba", + "terms_json": str(TERMS_JSON), + "word_freq_csv": str(WORD_FREQ_CSV), + "unique_words": len(counter), + "total_tokens": sum(counter.values()), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="VOC 词频统计（jieba 分词）") + parser.add_argument( + "--skip-llm", + action="store_true", + help="跳过第 1 步，使用 output/voc_terms.json", + ) + args = parser.parse_args() + result = run(skip_llm=args.skip_llm) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main()