Initial commit: VOC LLM 结构化分析流水线
包含七步编排入口、结构化/向量化/聚类/词频/报告模块与 prompts 配置；忽略原始 CSV 与本地密钥。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
OnesvmWhoops
commit
2441c9119c
43 changed files with 8264 additions and 0 deletions
33
.gitignore
vendored
Normal file
33
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# macOS
|
||||
.DS_Store
|
||||
|
||||
# Python
|
||||
310py/
|
||||
.venv/
|
||||
venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
|
||||
# 密钥（勿提交）
|
||||
.dashscope_key
|
||||
.env
|
||||
.env.*
|
||||
|
||||
# 本地数据库
|
||||
voc_structured.sqlite
|
||||
voc_embeddings.sqlite
|
||||
voc_clustering.sqlite
|
||||
|
||||
# 流水线输出（含 word_freq.csv、报告 HTML 等）
|
||||
output/
|
||||
|
||||
# 原始 / 中间 CSV（不纳入版本库）
|
||||
*.csv
|
||||
|
||||
# 常见 --input-dir 原始数据目录（目录名因产品而异，按需追加）
|
||||
reviews_export/
|
||||
*-voc/
|
||||
cat deterrent indoor /
|
||||
153
content清洗.py
Normal file
153
content清洗.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
"""
|
||||
清洗合并后的评论 CSV（merged_reviews.csv 等同结构文件）。
|
||||
保留元数据列，对 content 列做文本清洗与过滤。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
DEFAULT_INPUT_PATH = _PROJECT_ROOT / "merged_reviews.csv"
|
||||
DEFAULT_OUTPUT_PATH = _PROJECT_ROOT / "merged_reviews_cleaned.csv"
|
||||
|
||||
CONTENT_COLUMN = "content"
|
||||
MIN_WORD_COUNT = 4
|
||||
|
||||
# 与 content 重复的正文列，输出时一律删除，避免一行存两遍长文本
|
||||
REDUNDANT_CONTENT_COLUMNS = (
|
||||
"content_raw",
|
||||
"content_cleaned",
|
||||
"Cleaned_Content",
|
||||
"cleaned_content",
|
||||
)
|
||||
|
||||
# 亚马逊「视频/图片无法加载」占位文案，后常跟大量空行
|
||||
_MEDIA_LOAD_NOISE = re.compile(
|
||||
r"The\s+media\s+could\s+not\s+be\s+loaded\.?\s*",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def strip_empty_lines(text) -> str:
|
||||
"""去掉仅含空白/空白的行，并将剩余行合并为单行文本。"""
|
||||
if pd.isna(text):
|
||||
return ""
|
||||
text = str(text).replace("\r\n", "\n").replace("\r", "\n")
|
||||
lines = [line.strip() for line in text.split("\n")]
|
||||
lines = [line for line in lines if line]
|
||||
return " ".join(lines)
|
||||
|
||||
|
||||
def clean_amazon_review(text) -> str:
|
||||
if pd.isna(text):
|
||||
return ""
|
||||
text = str(text).strip()
|
||||
|
||||
if text.startswith('"') and text.endswith('"'):
|
||||
text = text[1:-1]
|
||||
|
||||
text = strip_empty_lines(text)
|
||||
text = _MEDIA_LOAD_NOISE.sub("", text)
|
||||
|
||||
text = re.sub(r"<br\s*/?>", " ", text, flags=re.IGNORECASE)
|
||||
text = text.replace(" ", " ").replace("&", "&")
|
||||
|
||||
noise_prefixes = [
|
||||
r"Why did you pick this product vs others\?:",
|
||||
r"Quality:",
|
||||
r"Update:",
|
||||
]
|
||||
for prefix in noise_prefixes:
|
||||
text = re.sub(prefix, "", text, flags=re.IGNORECASE).strip()
|
||||
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def drop_redundant_content_columns(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""删除与 content 重复存储的正文列（如 content_raw），避免 CSV 一行两份长文本。"""
|
||||
drop_cols = [
|
||||
c for c in REDUNDANT_CONTENT_COLUMNS if c in df.columns and c != CONTENT_COLUMN
|
||||
]
|
||||
if drop_cols:
|
||||
df = df.drop(columns=drop_cols)
|
||||
return df
|
||||
|
||||
|
||||
def process_reviews(
|
||||
csv_file_path: str | Path,
|
||||
*,
|
||||
content_column: str = CONTENT_COLUMN,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
读取合并后的评论 CSV，清洗 content 列并过滤无效/重复行。
|
||||
输出保留原表头及元数据（_id, asin, rating, title 等）。
|
||||
"""
|
||||
csv_file_path = Path(csv_file_path)
|
||||
df = pd.read_csv(csv_file_path, dtype=str, keep_default_na=False)
|
||||
|
||||
if content_column not in df.columns:
|
||||
raise ValueError(
|
||||
f"缺少评论正文列 {content_column!r}，当前列: {list(df.columns)}"
|
||||
)
|
||||
|
||||
print(f"1. 原始数据量: {len(df)}")
|
||||
|
||||
df = drop_redundant_content_columns(df)
|
||||
df[content_column] = df[content_column].apply(clean_amazon_review)
|
||||
|
||||
df = df[df[content_column] != ""]
|
||||
|
||||
df = df[
|
||||
df[content_column].str.contains(
|
||||
r"[a-zA-Z0-9áéíóúñÁÉÍÓÚÑ]", regex=True, na=False
|
||||
)
|
||||
]
|
||||
|
||||
word_counts = df[content_column].apply(lambda x: len(str(x).split()))
|
||||
df = df[word_counts >= MIN_WORD_COUNT]
|
||||
|
||||
df = df.drop_duplicates(subset=[content_column], keep="first")
|
||||
df = drop_redundant_content_columns(df)
|
||||
|
||||
print(f"2. 清洗及过滤后数据量: {len(df)}")
|
||||
return df
|
||||
|
||||
|
||||
def save_cleaned_reviews(df: pd.DataFrame, output_path: str | Path) -> Path:
|
||||
output_path = Path(output_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.to_csv(output_path, index=False, encoding="utf-8-sig")
|
||||
print(f"3. 已写入: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="清洗合并后的评论 CSV")
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--input",
|
||||
type=Path,
|
||||
default=DEFAULT_INPUT_PATH,
|
||||
help=f"输入 CSV（默认: {DEFAULT_INPUT_PATH}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_PATH,
|
||||
help=f"输出 CSV（默认: {DEFAULT_OUTPUT_PATH}）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
df = process_reviews(args.input)
|
||||
save_cleaned_reviews(df, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
297
main_voc分析.md
Normal file
297
main_voc分析.md
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
# main_voc分析.py 说明文档
|
||||
|
||||
VOC（Voice of Customer）分析**全流程编排入口**：合并原始 CSV → 清洗 → LLM 结构化 → 向量化 →（聚类 + 词频）→ 生成 HTML 报告。
|
||||
|
||||
> 本文档与 `main_voc分析.py` 同目录放置，风格对齐 `prompts/README.md`（表格 + 可执行示例 + 路径约定）。
|
||||
|
||||
---
|
||||
|
||||
## 1. 流程总览
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
S1[步骤1 合并CSV] --> S2[步骤2 清洗]
|
||||
S2 --> S3[步骤3 结构化 LLM]
|
||||
S3 --> S4[步骤4 向量化 API]
|
||||
S4 --> S5[步骤5 聚类 UMAP+HDBSCAN]
|
||||
S4 --> S6[步骤6 词频 spaCy+LLM]
|
||||
S5 --> S7[步骤7 报告 HTML]
|
||||
S6 --> S7
|
||||
```
|
||||
|
||||
| 步骤 | 模块 | 是否调用 DashScope API |
|
||||
|------|------|------------------------|
|
||||
| 1 | `合并评论数据.py` | 否 |
|
||||
| 2 | `content清洗.py` | 否 |
|
||||
| 3 | `结构化_server.py` | **是**（Chat 结构化） |
|
||||
| 4 | `向量化.py` | **是**（Embedding） |
|
||||
| 5–6 | `聚类.py` ∥ `词频.py` | **是**（聚类调参评估 + 词频术语提取） |
|
||||
| 7 | `voc_report.py` | **是**（报告撰写、词频分类、翻译等） |
|
||||
|
||||
步骤 5 与 6 在 `from_step ≤ 5` 且两步均需执行时，由 `ThreadPoolExecutor(max_workers=2)` **并行**运行。
|
||||
|
||||
---
|
||||
|
||||
## 2. 命令行输入 / 输出
|
||||
|
||||
### 2.1 输入（CLI 参数）
|
||||
|
||||
| 参数 | 必填条件 | 默认值 | 说明 |
|
||||
|------|----------|--------|------|
|
||||
| `--input-dir` | 是 | — | 原始 VOC评论数据文件 CSV 目录（目录内的所有`*.csv` 表头须一致） |
|
||||
| `--industry` | 是 | `Pet Supplies` | 行业名，写入结构化任务 |
|
||||
| `--product` | 是 | — | 产品名 |
|
||||
| `--merged-csv` | 否 | `merged_reviews.csv` | 步骤 1 输出路径 |
|
||||
| `--cleaned-csv` | 否 | `merged_reviews_cleaned.csv` | 步骤 2 输出路径 |
|
||||
| `--from-step N` | 否 | `1` | 从第 N 步执行到结束（N=1…7） |
|
||||
| `--only-step N` | 否 | — | 仅执行第 N 步（需已有前置产物） |
|
||||
| `--keep-db` | 否 | 默认**会**清理库 | 保留已有三个 `.sqlite`，不覆盖删除 |
|
||||
| `--clean-intermediates` | 否 | 关闭 | 步骤 7 成功后删除中间 csv/sqlite，仅保留报告目录 |
|
||||
| `--skip-wordfreq-llm` | 否 | 关闭 | 词频复用已有 `output/voc_terms.json`，跳过 LLM 术语提取 |
|
||||
| `--filter-small-clusters` | 否 | 关闭 | 报告仅纳入簇内去重评论占比 ≥ 10% 的簇 |
|
||||
| `--save-llm-raw` | 否 | 关闭 | 将报告 LLM 完整原文写入 `{product}_voc_report.html` 同目录 `report_llm_raw.txt` |
|
||||
|
||||
### 2.2 输出
|
||||
|
||||
**标准输出（stdout）**：流程结束后打印 JSON 字典（`ensure_ascii=False`），键随实际执行步骤变化，常见字段：
|
||||
|
||||
| 键 | 含义 |
|
||||
|----|------|
|
||||
| `merged_csv` | 合并后 CSV 路径 |
|
||||
| `cleaned_csv` | 清洗后 CSV 路径 |
|
||||
| `structured_job_id` | 结构化任务 ID |
|
||||
| `structured_db` | `voc_structured.sqlite` |
|
||||
| `embed` | 向量化模块返回信息（含 `job_id` 等） |
|
||||
| `clustering` | 聚类模块返回信息 |
|
||||
| `wordfreq` | 词频模块返回信息 |
|
||||
| `report_html` | 报告 HTML 绝对/相对路径 |
|
||||
| `report` | `generate_report` 返回值 |
|
||||
|
||||
**磁盘产物（项目根为 `PROJECT_ROOT`，即本文件所在目录）**：
|
||||
|
||||
| 路径 | 产生步骤 | 说明 |
|
||||
|------|----------|------|
|
||||
| `merged_reviews.csv` | 1 | 多文件纵向合并 |
|
||||
| `merged_reviews_cleaned.csv` | 2 | 清洗、去重后的评论 |
|
||||
| `voc_structured.sqlite` | 3 | 结构化任务与逐条提取结果 |
|
||||
| `voc_embeddings.sqlite` | 4 | 各实体短语 256 维向量 |
|
||||
| `voc_clustering.sqlite` | 5 | 多阶段 UMAP+HDBSCAN 簇标签 |
|
||||
| `output/voc_terms.json` | 6 | LLM 归纳的产品专有名词 / 停用词 |
|
||||
| `output/word_freq.csv` | 6 | 全量词频表（`word`, `count`） |
|
||||
| `output/{product}/{product}_voc_report.html` | 7 | 最终 VOC 分析报告（产品名经路径安全化） |
|
||||
|
||||
`{product}` 目录名由 `_safe_product_dir_name` 生成：去除 `<>:"/\|?*` 等非法字符。
|
||||
|
||||
### 2.3 常用命令
|
||||
|
||||
```bash
|
||||
cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化"
|
||||
|
||||
# 全流程（默认写入 sqlite 前会清理旧库；加 --keep-db 则保留）
|
||||
python3 main_voc分析.py --input-dir "某目录" --product "产品名"
|
||||
|
||||
# 从步骤 4 续跑（product 可省略）
|
||||
python3 main_voc分析.py --from-step 4 --keep-db
|
||||
|
||||
# 仅重跑词频 LLM 之前的 spaCy 统计
|
||||
python3 main_voc分析.py --from-step 6 --skip-wordfreq-llm
|
||||
|
||||
# 仅生成报告
|
||||
python3 main_voc分析.py --only-step 7
|
||||
```
|
||||
|
||||
### 2.4 程序式调用
|
||||
|
||||
```python
|
||||
from main_voc分析 import run_voc_analysis # 别名 run_pipeline
|
||||
|
||||
result = run_voc_analysis(
|
||||
input_dir=Path("reviews_export"),
|
||||
industry="Pet Supplies",
|
||||
product_name="cat deterrent indoor",
|
||||
merged_csv=Path("merged_reviews.csv"),
|
||||
cleaned_csv=Path("merged_reviews_cleaned.csv"),
|
||||
from_step=1,
|
||||
clean_databases=True, # 对应 CLI 未加 --keep-db
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 七步数据流与数学 / 算法
|
||||
|
||||
### 步骤 1：合并 CSV（`合并评论数据.py`）
|
||||
|
||||
- **输入**：目录内表头一致的 `*.csv`。
|
||||
- **方法**：`pandas.concat` 纵向合并；表头不一致则报错。
|
||||
- **输出**：单表 `merged_reviews.csv`（`utf-8-sig`）。
|
||||
|
||||
### 步骤 2：清洗（`content清洗.py`）
|
||||
|
||||
- **输入**：合并 CSV，正文列 `content`（或兼容列名由下游结构化处理）。
|
||||
- **方法**（规则型，无机器学习）：
|
||||
- 去空行、亚马逊媒体占位文案、HTML 实体、常见噪声前缀；
|
||||
- 保留含字母/数字的行的词数 ≥ 4；
|
||||
- 按 `content` 去重（`keep="first"`）。
|
||||
- **输出**：`merged_reviews_cleaned.csv`。
|
||||
|
||||
### 步骤 3：结构化（`结构化_server.py` + `结构化_Prompt.py` + `prompts/`）
|
||||
|
||||
- **输入**：清洗 CSV、`industry`、`product_name`。
|
||||
- **方法**：DashScope **Chat Completions**（默认 `qwen3.6-flash`），按 token 估算批量调用，从评论中提取 JSON 字段（audience、pain_points、aspect、opinion、category、sentiment 等，以 `prompts/schema.yaml` 为准）。
|
||||
- **输出**：`voc_structured.sqlite`（`analysis_jobs`、`comment_extractions`）。
|
||||
|
||||
### 步骤 4：向量化（`向量化.py`）
|
||||
|
||||
- **输入**：最新或指定 `job_id` 的结构化实体；`embed_text` 由 audience / pain_point / aspect / opinion / aspect_opinion 等展开。
|
||||
- **方法**：
|
||||
- API：`text-embedding-v4`，**256 维**，余弦相似度空间中的稠密向量；
|
||||
- 存储：`float32` 打包为 BLOB（`struct.pack`）；
|
||||
- 批大小 ≤ 10/请求，默认 6 线程并行多批。
|
||||
- **输出**：`voc_embeddings.sqlite`（`embedding_items`）。
|
||||
|
||||
### 步骤 5：聚类（`聚类.py`）
|
||||
|
||||
- **输入**：`voc_embeddings.sqlite` 中向量与元数据。
|
||||
- **核心数学管线**（对每个聚类 stage 的子集）：
|
||||
|
||||
1. **UMAP 降维**（`umap-learn`）
|
||||
- `n_components = min(30, n-2)`
|
||||
- `metric = cosine`
|
||||
- `min_dist = 0.1`
|
||||
- `random_state = 42`
|
||||
- `n_neighbors` 由自动调参循环递增（初值 10，上限 45）
|
||||
|
||||
2. **HDBSCAN**（`hdbscan`）
|
||||
- `min_samples = 1`，`cluster_selection_method = eom`
|
||||
- 初始 `min_cluster_size = max(2, n // 20)`，若簇数 > 20 则增大 `min_cluster_size` 直至 ≤ 20 或无法再增
|
||||
- 标签 `-1` 表示离群点（各 stage 是否参与后续见模块内注释）
|
||||
|
||||
3. **轮廓系数**（`sklearn.metrics.silhouette_score`）
|
||||
- 在 UMAP 空间、非离群点上计算；用于早停：连续 8 轮中后 7 轮均低于窗口首值则回退轮廓最高的一轮。
|
||||
|
||||
4. **LLM 聚类质量评估**（非传统指标，辅助调参）
|
||||
- 对各簇抽样短语，统计「跨簇语义相似」比例；> 10% 则继续增大 `n_neighbors`；≤ 10% 且轮廓 > 0.6 则停止。
|
||||
|
||||
- **特例**：子集样本数 `n < 10` 时不跑 HDBSCAN，每条独立簇 `0..n-1`。
|
||||
- **输出**：`voc_clustering.sqlite`（多 stage：如 audience、簇内 pain_point、按情感分桶的 aspect_opinion 等）。
|
||||
|
||||
### 步骤 6：词频（`词频.py`）
|
||||
|
||||
- **第 1 步（LLM）**：固定种子 `SAMPLE_SEED=42` 随机 **25** 条 `content`，归纳产品专有名词与专属停用词 → `output/voc_terms.json`。
|
||||
- **第 2 步（spaCy）**：
|
||||
- 英文分词与停用词过滤（含 EN stop words + 自定义停用词）；
|
||||
- 专有名词按完整短语计数（protected spans）；
|
||||
- 词形归并：`_merge_word_forms` 将复数/时态等 lemmatize 后合并计数；
|
||||
- 纯数字词剔除。
|
||||
- **输出**：`output/word_freq.csv`。
|
||||
|
||||
### 步骤 7：报告（`voc_report.py` + `prompts/report/`）
|
||||
|
||||
- **输入**：`voc_clustering.sqlite`、`voc_embeddings.sqlite`、`merged_reviews_cleaned.csv`、`word_freq.csv`、可选 `voc_structured.sqlite`。
|
||||
- **方法**：
|
||||
- 从库中聚合各 stage 簇与代表短语；
|
||||
- 多次 LLM 调用生成分析正文、词频六类归类、短语翻译等；
|
||||
- 词云字号：`count^0.5` 映射到 `[14, 96]` px；
|
||||
- 可选 `min_cluster_review_ratio`（默认 0.10）过滤小簇。
|
||||
- **输出**：`output/{product}/{product}_voc_report.html`（内嵌词云、词频表、AI 报告、分簇表述）。
|
||||
|
||||
---
|
||||
|
||||
## 4. API Key 使用说明
|
||||
|
||||
`main_voc分析.py` **本身不读取、不持有 API Key**；密钥解析在各子模块内统一实现，优先级一致：
|
||||
|
||||
1. 环境变量 `DASHSCOPE_API_KEY`
|
||||
2. 环境变量 `DASHSCOPE_API_KEY_FILE` 指向的单行密钥文件
|
||||
3. 项目根文件 `.dashscope_key`（单行，无引号）
|
||||
|
||||
可选环境变量：`DASHSCOPE_MODEL`（默认各模块为 `qwen3.6-flash`）。
|
||||
|
||||
| 模块 | 使用 API Key 的位置 | API 类型 / 用途 |
|
||||
|------|---------------------|-----------------|
|
||||
| `结构化_server.py` | `_resolve_dashscope_api_key()` → `OpenAI(...)` | Chat：批量/单条评论结构化 |
|
||||
| `向量化.py` | `_resolve_api_key()` → `_embed_one_api_batch` | Embeddings：`text-embedding-v4` |
|
||||
| `聚类.py` | `_resolve_api_key()` → `run_clustering` 内 `OpenAI` | Chat：簇间相似度评估、调 `n_neighbors` |
|
||||
| `词频.py` | `_resolve_api_key()` → `_step1_extract_terms` / `_call_llm` | Chat：专有名词与停用词提取 |
|
||||
| `voc_report.py` | `_resolve_api_key()` → `generate_report` 及子函数 | Chat：报告生成、词频分类、翻译等 |
|
||||
| `prompts/smoke.py` | `--live` 时 | 冒烟测试（非 main 流程） |
|
||||
|
||||
**仓库安全规范**（见 `.gitignore`）：
|
||||
|
||||
- **禁止提交** `.dashscope_key`、`voc_structured.sqlite` 及含真实评论/密钥的敏感导出；
|
||||
- 密钥仅通过环境变量或本机未跟踪文件提供；
|
||||
- 文档与代码中勿写入真实 `sk-` 密钥。
|
||||
|
||||
---
|
||||
|
||||
## 5. 各模块职责
|
||||
|
||||
| 文件 | 职责 |
|
||||
|------|------|
|
||||
| `main_voc分析.py` | 七步编排、断点续跑、库清理策略、步骤 5/6 并行、CLI |
|
||||
| `合并评论数据.py` | 目录多 CSV 合并为一张表 |
|
||||
| `content清洗.py` | 评论正文清洗、过滤、去重 |
|
||||
| `结构化_server.py` | LLM 结构化入库、job 管理、库清理钩子 |
|
||||
| `结构化_Prompt.py` | 组装结构化 prompt（被 server 动态加载） |
|
||||
| `向量化.py` | 结构化实体 → 256 维向量库 |
|
||||
| `聚类.py` | UMAP + HDBSCAN 多阶段聚类 + LLM 调参 |
|
||||
| `词频.py` | LLM 术语 + spaCy 全量词频 |
|
||||
| `voc_report.py` | 聚类/词频/原文 → 单页 HTML 报告 |
|
||||
| `prompts/` | 可编辑 prompt 与 `schema.yaml`（见 `prompts/README.md`） |
|
||||
| `main_voc分析_jieba.py` | 可选变体入口：词频走 `词频_jieba.py`（jieba 分词），其余步骤与主流程类似 |
|
||||
|
||||
### main 内主要函数
|
||||
|
||||
| 函数 | 作用 |
|
||||
|------|------|
|
||||
| `run_voc_analysis` / `run_pipeline` | 核心流水线 |
|
||||
| `_should_run` | 根据 `--from-step` / `--only-step` 判断是否执行某步 |
|
||||
| `_latest_job_meta` | 从结构化库读最新 `job_id`、`industry`、`product_name` |
|
||||
| `_resolve_industry_product` | CLI 下 product 自动补全 |
|
||||
| `_clean_intermediates` | `--clean-intermediates` 时删除中间文件 |
|
||||
| `_report_html_path` | 计算报告 HTML 路径 |
|
||||
|
||||
---
|
||||
|
||||
## 6. SQLite 与路径约定
|
||||
|
||||
| 库文件 | 写入步骤 | 主要内容 |
|
||||
|--------|----------|----------|
|
||||
| `voc_structured.sqlite` | 3 | `analysis_jobs`、`comment_extractions` |
|
||||
| `voc_embeddings.sqlite` | 4 | `embedding_items`（向量 BLOB + 实体元数据） |
|
||||
| `voc_clustering.sqlite` | 5 | 各 `stage` 簇标签、调参日志、过滤元数据 |
|
||||
|
||||
**默认清理策略**（未加 `--keep-db`）：
|
||||
|
||||
- 步骤 3 运行 `run_analysis(clean_databases=True)` 时清理三个 sqlite（结构化写入逻辑见 `结构化_server.py`）；
|
||||
- 步骤 4 若未重跑步骤 3，main 会单独删除 `voc_embeddings.sqlite` 与 `voc_clustering.sqlite` 再向量化；
|
||||
- 步骤 5/6 的 `reset_db` 与 `clean_databases` 联动。
|
||||
|
||||
**行号溯源**：结构化与向量化的 `source_row` 为 CSV **首条数据行为 1**；对应 `merged_reviews_cleaned.csv` 物理行号 = `source_row + 1`（第 1 行为表头）。
|
||||
|
||||
---
|
||||
|
||||
## 7. 依赖与环境
|
||||
|
||||
```bash
|
||||
cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化"
|
||||
python3 -m venv 310py && source 310py/bin/activate # 可选，与 .gitignore 一致
|
||||
pip install -r requirements.txt
|
||||
python -m spacy download en_core_web_sm # 词频步骤需要
|
||||
export DASHSCOPE_API_KEY="sk-xxx" # 或配置 .dashscope_key（勿提交）
|
||||
```
|
||||
|
||||
`requirements.txt` 中与数学/ NLP 相关的主要包：`numpy`、`umap-learn`、`hdbscan`、`scikit-learn`、`spacy`、`openai`、`pyyaml`。
|
||||
|
||||
---
|
||||
|
||||
## 8. 维护说明
|
||||
|
||||
- 修改 LLM 话术：编辑 `prompts/` 下对应 `.md`，**勿改** `schema.yaml` 中 `report.markers` 四段标记名（见 `prompts/README.md`）。
|
||||
- 修改主流程步骤顺序或默认路径：改 `main_voc分析.py` 后请同步更新**本文档**。
|
||||
- 验收 prompt 加载：`python3 prompts/smoke.py`（无需 Key）；联调模型：`python3 prompts/smoke.py --live`。
|
||||
|
||||
---
|
||||
|
||||
*文档版本：与仓库 `main_voc分析.py` 七步流程一致。*
|
||||
393
main_voc分析.py
Normal file
393
main_voc分析.py
Normal file
|
|
@ -0,0 +1,393 @@
|
|||
"""
|
||||
VOC 全流程：合并 → 清洗 → 结构化 → 向量化 →（聚类 ∥ 词频）→ 分析报告 HTML。
|
||||
|
||||
用法::
|
||||
|
||||
# 全流程（--industry 默认 Pet Supplies；写入 sqlite 前默认清理旧库，keep-db不清理）
|
||||
python3 main_voc分析.py --input-dir 目录 --product "产品名" --keep-db
|
||||
|
||||
# 断点续跑（步骤 4 起可省略 --product，自动读 voc_structured.sqlite）
|
||||
python3 main_voc分析.py --from-step 5
|
||||
python3 main_voc分析.py --from-step 6 --skip-wordfreq-llm # 仅重跑词频
|
||||
python3 main_voc分析.py --only-step 7 # 仅生成报告
|
||||
|
||||
# 保留已有 voc_structured / voc_embeddings / voc_clustering.sqlite，不清理覆盖
|
||||
python3 main_voc分析.py --from-step 4 --keep-db
|
||||
|
||||
报告 HTML：output/{product_name}/{product_name}_voc_report.html
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from content清洗 import process_reviews, save_cleaned_reviews
|
||||
from 合并评论数据 import merge_csv_directory
|
||||
from 向量化 import run_embed
|
||||
from 结构化_server import run_analysis
|
||||
from 聚类 import run_clustering
|
||||
from voc_report import DEFAULT_CLUSTER_MIN_REVIEW_RATIO, generate_report
|
||||
from 词频 import run as run_wordfreq
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("voc_analysis")
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
OUTPUT_DIR = PROJECT_ROOT / "output"
|
||||
|
||||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||||
EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite"
|
||||
CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite"
|
||||
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
|
||||
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"
|
||||
|
||||
DEFAULT_MERGED = PROJECT_ROOT / "merged_reviews.csv"
|
||||
DEFAULT_CLEANED = PROJECT_ROOT / "merged_reviews_cleaned.csv"
|
||||
DEFAULT_INDUSTRY = "Pet Supplies"
|
||||
|
||||
|
||||
def _safe_product_dir_name(product_name: str) -> str:
|
||||
"""用于 output 子目录名；去除路径非法字符。"""
|
||||
name = (product_name or "product").strip()
|
||||
name = re.sub(r'[<>:"/\\|?*]', "_", name)
|
||||
return name or "product"
|
||||
|
||||
|
||||
def _product_output_dir(product_name: str) -> Path:
|
||||
return OUTPUT_DIR / _safe_product_dir_name(product_name)
|
||||
|
||||
|
||||
def _report_html_path(product_name: str) -> Path:
|
||||
slug = _safe_product_dir_name(product_name)
|
||||
return _product_output_dir(product_name) / f"{slug}_voc_report.html"
|
||||
|
||||
|
||||
def _latest_job_meta(structured_db: Path) -> tuple[int, str, str]:
|
||||
conn = sqlite3.connect(structured_db)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, industry, product_name
|
||||
FROM analysis_jobs ORDER BY id DESC LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise RuntimeError("无结构化任务，请先执行步骤 3")
|
||||
return int(row[0]), str(row[1]), str(row[2])
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _should_run(step: int, from_step: int, only_step: int | None) -> bool:
|
||||
if only_step is not None:
|
||||
return step == only_step
|
||||
return step >= from_step
|
||||
|
||||
|
||||
def _clean_intermediates(
|
||||
*,
|
||||
merged: Path,
|
||||
cleaned: Path,
|
||||
structured: Path,
|
||||
embed: Path,
|
||||
cluster: Path,
|
||||
terms: Path,
|
||||
word_freq: Path,
|
||||
) -> None:
|
||||
for p in (merged, cleaned, structured, embed, cluster, terms, word_freq):
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
logger.info("已删除中间文件 %s", p)
|
||||
|
||||
|
||||
def run_voc_analysis(
|
||||
*,
|
||||
input_dir: Path,
|
||||
industry: str,
|
||||
product_name: str,
|
||||
merged_csv: Path,
|
||||
cleaned_csv: Path,
|
||||
from_step: int = 1,
|
||||
only_step: int | None = None,
|
||||
clean_intermediates: bool = False,
|
||||
clean_databases: bool = True,
|
||||
skip_wordfreq_llm: bool = False,
|
||||
min_cluster_review_ratio: float | None = None,
|
||||
save_llm_raw: bool = False,
|
||||
) -> dict:
|
||||
result: dict = {}
|
||||
|
||||
if _should_run(1, from_step, only_step):
|
||||
logger.info("步骤 1/7：合并 CSV")
|
||||
merge_csv_directory(input_dir, merged_csv)
|
||||
result["merged_csv"] = str(merged_csv)
|
||||
|
||||
if _should_run(2, from_step, only_step):
|
||||
logger.info("步骤 2/7：清洗评论")
|
||||
if not merged_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {merged_csv}，请先执行合并")
|
||||
df = process_reviews(merged_csv)
|
||||
save_cleaned_reviews(df, cleaned_csv)
|
||||
result["cleaned_csv"] = str(cleaned_csv)
|
||||
|
||||
if _should_run(3, from_step, only_step):
|
||||
logger.info("步骤 3/7：结构化分析")
|
||||
if not cleaned_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {cleaned_csv}，请先执行清洗")
|
||||
ar = run_analysis(
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
file_path=str(cleaned_csv),
|
||||
clean_databases=clean_databases,
|
||||
)
|
||||
result["structured_job_id"] = ar.get("job_id")
|
||||
result["structured_db"] = str(STRUCTURED_DB)
|
||||
|
||||
job_id: int | None = None
|
||||
ind, prod = industry, product_name
|
||||
if STRUCTURED_DB.is_file():
|
||||
try:
|
||||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
if _should_run(4, from_step, only_step):
|
||||
if clean_databases and not _should_run(3, from_step, only_step):
|
||||
logger.info("写入向量化库前清理 voc_embeddings.sqlite / voc_clustering.sqlite")
|
||||
for p in (EMBED_DB, CLUSTER_DB):
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
logger.info("已清理 SQLite: %s", p.name)
|
||||
logger.info("步骤 4/7：向量化")
|
||||
er = run_embed(
|
||||
job_id=job_id,
|
||||
csv_path=cleaned_csv,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
reset_db=clean_databases,
|
||||
)
|
||||
result["embed"] = er
|
||||
job_id = int(er.get("job_id", job_id or 0))
|
||||
|
||||
if job_id is None and STRUCTURED_DB.is_file():
|
||||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
|
||||
run_cluster = _should_run(5, from_step, only_step)
|
||||
run_wf = _should_run(6, from_step, only_step)
|
||||
|
||||
if run_cluster and run_wf:
|
||||
logger.info("步骤 5–6/7：聚类与词频（并行）")
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
f_cluster = pool.submit(
|
||||
run_clustering,
|
||||
job_id=job_id,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
cluster_db=CLUSTER_DB,
|
||||
reset_db=clean_databases,
|
||||
)
|
||||
f_wf = pool.submit(run_wordfreq, skip_llm=skip_wordfreq_llm)
|
||||
result["clustering"] = f_cluster.result()
|
||||
result["wordfreq"] = f_wf.result()
|
||||
else:
|
||||
if run_cluster:
|
||||
logger.info("步骤 5/7：聚类")
|
||||
result["clustering"] = run_clustering(
|
||||
job_id=job_id,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
cluster_db=CLUSTER_DB,
|
||||
reset_db=clean_databases,
|
||||
)
|
||||
if run_wf:
|
||||
logger.info("步骤 6/7：词频")
|
||||
result["wordfreq"] = run_wordfreq(skip_llm=skip_wordfreq_llm)
|
||||
|
||||
if _should_run(7, from_step, only_step):
|
||||
logger.info("步骤 7/7：生成分析报告")
|
||||
if not CLUSTER_DB.is_file():
|
||||
raise FileNotFoundError(f"缺少 {CLUSTER_DB}")
|
||||
if not WORD_FREQ_CSV.is_file():
|
||||
raise FileNotFoundError(f"缺少 {WORD_FREQ_CSV}")
|
||||
if not cleaned_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {cleaned_csv}")
|
||||
if STRUCTURED_DB.is_file():
|
||||
_, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
report_html = _report_html_path(prod)
|
||||
result["report_html"] = str(report_html)
|
||||
result["report"] = generate_report(
|
||||
product_name=prod,
|
||||
industry=ind,
|
||||
cleaned_csv=cleaned_csv,
|
||||
cluster_db=CLUSTER_DB,
|
||||
embed_db=EMBED_DB,
|
||||
word_freq_csv=WORD_FREQ_CSV,
|
||||
output_html=report_html,
|
||||
structured_db=STRUCTURED_DB,
|
||||
min_cluster_review_ratio=min_cluster_review_ratio,
|
||||
save_llm_raw=save_llm_raw,
|
||||
)
|
||||
|
||||
if clean_intermediates and _should_run(7, from_step, only_step):
|
||||
_clean_intermediates(
|
||||
merged=merged_csv,
|
||||
cleaned=cleaned_csv,
|
||||
structured=STRUCTURED_DB,
|
||||
embed=EMBED_DB,
|
||||
cluster=CLUSTER_DB,
|
||||
terms=TERMS_JSON,
|
||||
word_freq=WORD_FREQ_CSV,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# 兼容旧名
|
||||
run_pipeline = run_voc_analysis
|
||||
|
||||
|
||||
def _resolve_industry_product(
|
||||
*,
|
||||
industry: str | None,
|
||||
product_name: str | None,
|
||||
from_step: int,
|
||||
only_step: int | None,
|
||||
) -> tuple[str, str]:
|
||||
"""步骤 4 起可从 structured 库自动读取 product；industry 默认 Pet Supplies。"""
|
||||
ind = (industry or DEFAULT_INDUSTRY).strip() or DEFAULT_INDUSTRY
|
||||
if product_name:
|
||||
return ind, product_name
|
||||
start_step = only_step if only_step is not None else from_step
|
||||
if start_step >= 4:
|
||||
if not STRUCTURED_DB.is_file():
|
||||
raise SystemExit(
|
||||
f"缺少 {STRUCTURED_DB.name}，无法自动读取 --product"
|
||||
)
|
||||
_, db_ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
logger.info("未指定 product，已从 structured 库读取: %r（industry=%r）", prod, db_ind)
|
||||
return db_ind, prod
|
||||
raise SystemExit(
|
||||
f"步骤 1–3 需要 --product；--industry 可省略（默认 {DEFAULT_INDUSTRY}）；"
|
||||
"从步骤 4 起 product 也可省略（自动读 voc_structured.sqlite）"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="VOC 分析全流程")
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="原始 VOC CSV 目录（步骤 1 合并输入；续跑后续步骤可不传）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--industry",
|
||||
default=DEFAULT_INDUSTRY,
|
||||
help=f"行业（默认 {DEFAULT_INDUSTRY}）",
|
||||
)
|
||||
parser.add_argument("--product", default=None, help="产品名（步骤 4 起可省略，自动读库）")
|
||||
parser.add_argument(
|
||||
"--merged-csv",
|
||||
type=Path,
|
||||
default=DEFAULT_MERGED,
|
||||
help=f"合并输出（默认 {DEFAULT_MERGED.name}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cleaned-csv",
|
||||
type=Path,
|
||||
default=DEFAULT_CLEANED,
|
||||
help=f"清洗输出（默认 {DEFAULT_CLEANED.name}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from-step",
|
||||
type=int,
|
||||
default=1,
|
||||
choices=range(1, 8),
|
||||
metavar="N",
|
||||
help="从第 N 步开始执行（1–7，默认 1）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only-step",
|
||||
type=int,
|
||||
default=None,
|
||||
choices=range(1, 8),
|
||||
metavar="N",
|
||||
help="仅执行第 N 步（需已有前置产物）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-db",
|
||||
action="store_true",
|
||||
help="保留已有 voc_structured / voc_embeddings / voc_clustering.sqlite，不覆盖清理（默认写入前会清理）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean-intermediates",
|
||||
action="store_true",
|
||||
help="报告生成成功后删除中间 sqlite/csv（仅保留各产品子目录下的报告 HTML）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-wordfreq-llm",
|
||||
action="store_true",
|
||||
help="词频步骤复用已有 voc_terms.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filter-small-clusters",
|
||||
action="store_true",
|
||||
help=(
|
||||
"报告仅纳入本 stage 内去重评论占比 ≥ "
|
||||
f"{DEFAULT_CLUSTER_MIN_REVIEW_RATIO * 100:.0f}%% 的簇；默认不筛选、全部簇参与"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-llm-raw",
|
||||
action="store_true",
|
||||
help="步骤 7 将报告 LLM 完整原文写入报告同目录下的 report_llm_raw.txt（默认不保存）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
need_input = args.only_step in (None, 1) and args.from_step <= 1
|
||||
if need_input:
|
||||
if not args.input_dir:
|
||||
raise SystemExit("步骤 1 需要 --input-dir")
|
||||
if not args.input_dir.is_dir():
|
||||
raise SystemExit(f"输入目录不存在: {args.input_dir}")
|
||||
input_dir = (args.input_dir or DEFAULT_MERGED.parent).resolve()
|
||||
industry, product_name = _resolve_industry_product(
|
||||
industry=args.industry,
|
||||
product_name=args.product,
|
||||
from_step=args.from_step,
|
||||
only_step=args.only_step,
|
||||
)
|
||||
|
||||
out = run_voc_analysis(
|
||||
input_dir=input_dir,
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
merged_csv=args.merged_csv.resolve(),
|
||||
cleaned_csv=args.cleaned_csv.resolve(),
|
||||
from_step=args.from_step,
|
||||
only_step=args.only_step,
|
||||
clean_intermediates=args.clean_intermediates,
|
||||
clean_databases=not args.keep_db,
|
||||
skip_wordfreq_llm=args.skip_wordfreq_llm,
|
||||
min_cluster_review_ratio=(
|
||||
DEFAULT_CLUSTER_MIN_REVIEW_RATIO
|
||||
if args.filter_small_clusters
|
||||
else None
|
||||
),
|
||||
save_llm_raw=args.save_llm_raw,
|
||||
)
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
342
main_voc分析_jieba.py
Normal file
342
main_voc分析_jieba.py
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
"""
|
||||
VOC 全流程（jieba 分词版）：与 main_voc分析.py 相同，步骤 6 词频改用 jieba。
|
||||
|
||||
用法::
|
||||
|
||||
python3 main_voc分析_jieba.py --input-dir reviews_export --industry "Pet supplements" --product "Turkey tail mushroom for dogs"
|
||||
|
||||
python3 main_voc分析_jieba.py --from-step 6 # 仅重跑 jieba 词频（可省略 --industry/--product）
|
||||
python3 main_voc分析_jieba.py --only-step 7 # 仅生成报告
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from content清洗 import process_reviews, save_cleaned_reviews
|
||||
from 合并评论数据 import merge_csv_directory
|
||||
from 向量化 import run_embed
|
||||
from 结构化_server import run_analysis
|
||||
from 聚类 import run_clustering
|
||||
from voc_report import DEFAULT_CLUSTER_MIN_REVIEW_RATIO, generate_report
|
||||
from 词频_jieba import run as run_wordfreq
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("voc_analysis_jieba")
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
OUTPUT_DIR = PROJECT_ROOT / "output"
|
||||
REPORT_HTML = OUTPUT_DIR / "voc_report.html"
|
||||
|
||||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||||
EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite"
|
||||
CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite"
|
||||
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
|
||||
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"
|
||||
|
||||
DEFAULT_MERGED = PROJECT_ROOT / "merged_reviews.csv"
|
||||
DEFAULT_CLEANED = PROJECT_ROOT / "merged_reviews_cleaned.csv"
|
||||
|
||||
|
||||
def _latest_job_meta(structured_db: Path) -> tuple[int, str, str]:
|
||||
conn = sqlite3.connect(structured_db)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, industry, product_name
|
||||
FROM analysis_jobs ORDER BY id DESC LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise RuntimeError("无结构化任务，请先执行步骤 3")
|
||||
return int(row[0]), str(row[1]), str(row[2])
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _should_run(step: int, from_step: int, only_step: int | None) -> bool:
|
||||
if only_step is not None:
|
||||
return step == only_step
|
||||
return step >= from_step
|
||||
|
||||
|
||||
def _clean_intermediates(
|
||||
*,
|
||||
merged: Path,
|
||||
cleaned: Path,
|
||||
structured: Path,
|
||||
embed: Path,
|
||||
cluster: Path,
|
||||
terms: Path,
|
||||
word_freq: Path,
|
||||
) -> None:
|
||||
for p in (merged, cleaned, structured, embed, cluster, terms, word_freq):
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
logger.info("已删除中间文件 %s", p)
|
||||
|
||||
|
||||
def run_voc_analysis(
|
||||
*,
|
||||
input_dir: Path,
|
||||
industry: str,
|
||||
product_name: str,
|
||||
merged_csv: Path,
|
||||
cleaned_csv: Path,
|
||||
from_step: int = 1,
|
||||
only_step: int | None = None,
|
||||
clean_intermediates: bool = False,
|
||||
skip_wordfreq_llm: bool = False,
|
||||
min_cluster_review_ratio: float | None = None,
|
||||
save_llm_raw: bool = False,
|
||||
) -> dict:
|
||||
result: dict = {"report_html": str(REPORT_HTML), "tokenizer": "jieba"}
|
||||
|
||||
if _should_run(1, from_step, only_step):
|
||||
logger.info("步骤 1/7：合并 CSV")
|
||||
merge_csv_directory(input_dir, merged_csv)
|
||||
result["merged_csv"] = str(merged_csv)
|
||||
|
||||
if _should_run(2, from_step, only_step):
|
||||
logger.info("步骤 2/7：清洗评论")
|
||||
if not merged_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {merged_csv}，请先执行合并")
|
||||
df = process_reviews(merged_csv)
|
||||
save_cleaned_reviews(df, cleaned_csv)
|
||||
result["cleaned_csv"] = str(cleaned_csv)
|
||||
|
||||
if _should_run(3, from_step, only_step):
|
||||
logger.info("步骤 3/7：结构化分析")
|
||||
if not cleaned_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {cleaned_csv}，请先执行清洗")
|
||||
ar = run_analysis(
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
file_path=str(cleaned_csv),
|
||||
)
|
||||
result["structured_job_id"] = ar.get("job_id")
|
||||
result["structured_db"] = str(STRUCTURED_DB)
|
||||
|
||||
job_id: int | None = None
|
||||
ind, prod = industry, product_name
|
||||
if STRUCTURED_DB.is_file():
|
||||
try:
|
||||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
if _should_run(4, from_step, only_step):
|
||||
logger.info("步骤 4/7：向量化")
|
||||
er = run_embed(
|
||||
job_id=job_id,
|
||||
csv_path=cleaned_csv,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
)
|
||||
result["embed"] = er
|
||||
job_id = int(er.get("job_id", job_id or 0))
|
||||
|
||||
if job_id is None and STRUCTURED_DB.is_file():
|
||||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
|
||||
run_cluster = _should_run(5, from_step, only_step)
|
||||
run_wf = _should_run(6, from_step, only_step)
|
||||
|
||||
if run_cluster and run_wf:
|
||||
logger.info("步骤 5–6/7：聚类与词频（jieba，并行）")
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
f_cluster = pool.submit(
|
||||
run_clustering,
|
||||
job_id=job_id,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
cluster_db=CLUSTER_DB,
|
||||
)
|
||||
f_wf = pool.submit(run_wordfreq, skip_llm=skip_wordfreq_llm)
|
||||
result["clustering"] = f_cluster.result()
|
||||
result["wordfreq"] = f_wf.result()
|
||||
else:
|
||||
if run_cluster:
|
||||
logger.info("步骤 5/7：聚类")
|
||||
result["clustering"] = run_clustering(
|
||||
job_id=job_id,
|
||||
structured_db=STRUCTURED_DB,
|
||||
embed_db=EMBED_DB,
|
||||
cluster_db=CLUSTER_DB,
|
||||
)
|
||||
if run_wf:
|
||||
logger.info("步骤 6/7：词频（jieba）")
|
||||
result["wordfreq"] = run_wordfreq(skip_llm=skip_wordfreq_llm)
|
||||
|
||||
if _should_run(7, from_step, only_step):
|
||||
logger.info("步骤 7/7：生成分析报告")
|
||||
if not CLUSTER_DB.is_file():
|
||||
raise FileNotFoundError(f"缺少 {CLUSTER_DB}")
|
||||
if not WORD_FREQ_CSV.is_file():
|
||||
raise FileNotFoundError(f"缺少 {WORD_FREQ_CSV}")
|
||||
if not cleaned_csv.is_file():
|
||||
raise FileNotFoundError(f"缺少 {cleaned_csv}")
|
||||
if STRUCTURED_DB.is_file():
|
||||
_, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
result["report"] = generate_report(
|
||||
product_name=prod,
|
||||
industry=ind,
|
||||
cleaned_csv=cleaned_csv,
|
||||
cluster_db=CLUSTER_DB,
|
||||
embed_db=EMBED_DB,
|
||||
word_freq_csv=WORD_FREQ_CSV,
|
||||
output_html=REPORT_HTML,
|
||||
structured_db=STRUCTURED_DB,
|
||||
min_cluster_review_ratio=min_cluster_review_ratio,
|
||||
save_llm_raw=save_llm_raw,
|
||||
)
|
||||
|
||||
if clean_intermediates and _should_run(7, from_step, only_step):
|
||||
_clean_intermediates(
|
||||
merged=merged_csv,
|
||||
cleaned=cleaned_csv,
|
||||
structured=STRUCTURED_DB,
|
||||
embed=EMBED_DB,
|
||||
cluster=CLUSTER_DB,
|
||||
terms=TERMS_JSON,
|
||||
word_freq=WORD_FREQ_CSV,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
run_pipeline = run_voc_analysis
|
||||
|
||||
|
||||
def _resolve_industry_product(
|
||||
*,
|
||||
industry: str | None,
|
||||
product_name: str | None,
|
||||
from_step: int,
|
||||
only_step: int | None,
|
||||
) -> tuple[str, str]:
|
||||
"""步骤 4 起可从 structured 库自动读取；步骤 1–3 须 CLI 传入。"""
|
||||
if industry and product_name:
|
||||
return industry, product_name
|
||||
start_step = only_step if only_step is not None else from_step
|
||||
if start_step >= 4:
|
||||
if not STRUCTURED_DB.is_file():
|
||||
raise SystemExit(
|
||||
f"缺少 {STRUCTURED_DB.name}，无法自动读取 --industry / --product"
|
||||
)
|
||||
_, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||||
logger.info("未指定 industry/product，已从 structured 库读取: %r / %r", ind, prod)
|
||||
return ind, prod
|
||||
raise SystemExit(
|
||||
"步骤 1–3 需要 --industry 与 --product；从步骤 4 起可省略（自动读 voc_structured.sqlite）"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="VOC 分析全流程（jieba 分词）")
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="原始 VOC CSV 目录（步骤 1 合并输入；续跑后续步骤可不传）",
|
||||
)
|
||||
parser.add_argument("--industry", default=None, help="行业（步骤 4 起可省略，自动读库）")
|
||||
parser.add_argument("--product", default=None, help="产品名（步骤 4 起可省略，自动读库）")
|
||||
parser.add_argument(
|
||||
"--merged-csv",
|
||||
type=Path,
|
||||
default=DEFAULT_MERGED,
|
||||
help=f"合并输出（默认 {DEFAULT_MERGED.name}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cleaned-csv",
|
||||
type=Path,
|
||||
default=DEFAULT_CLEANED,
|
||||
help=f"清洗输出（默认 {DEFAULT_CLEANED.name}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from-step",
|
||||
type=int,
|
||||
default=1,
|
||||
choices=range(1, 8),
|
||||
metavar="N",
|
||||
help="从第 N 步开始执行（1–7，默认 1）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only-step",
|
||||
type=int,
|
||||
default=None,
|
||||
choices=range(1, 8),
|
||||
metavar="N",
|
||||
help="仅执行第 N 步（需已有前置产物）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean-intermediates",
|
||||
action="store_true",
|
||||
help="报告生成成功后删除中间 sqlite/csv（仅保留 voc_report.html）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-wordfreq-llm",
|
||||
action="store_true",
|
||||
help="词频步骤复用已有 voc_terms.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filter-small-clusters",
|
||||
action="store_true",
|
||||
help=(
|
||||
"报告仅纳入本 stage 内去重评论占比 ≥ "
|
||||
f"{DEFAULT_CLUSTER_MIN_REVIEW_RATIO * 100:.0f}%% 的簇；默认不筛选、全部簇参与"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-llm-raw",
|
||||
action="store_true",
|
||||
help="步骤 7 将报告 LLM 完整原文写入 output/report_llm_raw.txt（默认不保存）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
need_input = args.only_step in (None, 1) and args.from_step <= 1
|
||||
if need_input:
|
||||
if not args.input_dir:
|
||||
raise SystemExit("步骤 1 需要 --input-dir")
|
||||
if not args.input_dir.is_dir():
|
||||
raise SystemExit(f"输入目录不存在: {args.input_dir}")
|
||||
input_dir = (args.input_dir or DEFAULT_MERGED.parent).resolve()
|
||||
industry, product_name = _resolve_industry_product(
|
||||
industry=args.industry,
|
||||
product_name=args.product,
|
||||
from_step=args.from_step,
|
||||
only_step=args.only_step,
|
||||
)
|
||||
|
||||
out = run_voc_analysis(
|
||||
input_dir=input_dir,
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
merged_csv=args.merged_csv.resolve(),
|
||||
cleaned_csv=args.cleaned_csv.resolve(),
|
||||
from_step=args.from_step,
|
||||
only_step=args.only_step,
|
||||
clean_intermediates=args.clean_intermediates,
|
||||
skip_wordfreq_llm=args.skip_wordfreq_llm,
|
||||
min_cluster_review_ratio=(
|
||||
DEFAULT_CLUSTER_MIN_REVIEW_RATIO
|
||||
if args.filter_small_clusters
|
||||
else None
|
||||
),
|
||||
save_llm_raw=args.save_llm_raw,
|
||||
)
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
33
prompts/README.md
Normal file
33
prompts/README.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Prompt 编辑说明（产品 / 运营 / 开发）
|
||||
|
||||
修改后**下次运行脚本自动生效**，无需重启服务。
|
||||
|
||||
## 可编辑文件
|
||||
|
||||
| 路径 | 谁改 | 内容 |
|
||||
|------|------|------|
|
||||
| `config.yaml` | 产品/运营 | Top N 数值、中文词频六类名称 |
|
||||
| `extraction/*.md` | 产品/运营 | 结构化提取规则、语气 |
|
||||
| `extraction/examples.yaml` | 产品/运营 | Few-shot “示例”（output 用 YAML 对象，勿手写 JSON 字符串） |
|
||||
| `report/*.md` | 产品/运营 | 报告分析要求、输出格式、修正提示 |
|
||||
| `word_freq/*.md` | 产品/运营 | 词频分类与解读 prompt（含 `category_definitions.md` 各类定义） |
|
||||
| `schema.yaml` | **仅开发** | JSON 字段名、英文 category、四段输出标记 |
|
||||
|
||||
## 占位符
|
||||
|
||||
`.md` 文件使用 Python `format` 语法，例如 `{product_name}`、`{industry}`。
|
||||
正文里需要字面量花括号时写双花括号：`{{"audience": "unknown"}}`。
|
||||
|
||||
## 改完怎么验收
|
||||
|
||||
```bash
|
||||
cd "/Users/onesvmwhoops/Cursor_Project/VOC_LLM结构化"
|
||||
python3 prompts/smoke.py # 检查文件能否加载（无需 API Key）
|
||||
python3 prompts/smoke.py --live # 用 3 条样例评论调模型（需 DASHSCOPE_API_KEY）
|
||||
```
|
||||
|
||||
## 注意
|
||||
|
||||
- 不要改 `schema.yaml` 里的 `report.markers` 四段标记名，否则报告解析会失败。
|
||||
- `examples.yaml` 里 `sentiment` / `category` 须与 `schema.yaml` 枚举一致。
|
||||
- 词频分类若增删类别，需同步 `config.yaml` 的 `word_categories` 与 `voc_report.py` 内 `CATEGORY_COLORS` 配色。
|
||||
45
prompts/__init__.py
Normal file
45
prompts/__init__.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""Prompt 包：外置 .md / .yaml 加载与组装。"""
|
||||
|
||||
from prompts.loader import (
|
||||
build_batch_extraction_system,
|
||||
build_batch_extraction_user,
|
||||
build_category_analysis_prompts,
|
||||
build_report_analysis_requirements,
|
||||
build_report_correction_message,
|
||||
build_report_json_markers,
|
||||
build_report_output_format,
|
||||
build_report_system,
|
||||
build_single_extraction_template,
|
||||
build_word_assign_prompts,
|
||||
format_examples,
|
||||
get_config,
|
||||
get_schema,
|
||||
load_extraction_examples,
|
||||
product_feedback_categories,
|
||||
report_markers,
|
||||
sync_voc_report_constants,
|
||||
validate_prompt_files,
|
||||
word_categories,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"build_batch_extraction_system",
|
||||
"build_batch_extraction_user",
|
||||
"build_category_analysis_prompts",
|
||||
"build_report_analysis_requirements",
|
||||
"build_report_correction_message",
|
||||
"build_report_json_markers",
|
||||
"build_report_output_format",
|
||||
"build_report_system",
|
||||
"build_single_extraction_template",
|
||||
"build_word_assign_prompts",
|
||||
"format_examples",
|
||||
"get_config",
|
||||
"get_schema",
|
||||
"load_extraction_examples",
|
||||
"product_feedback_categories",
|
||||
"report_markers",
|
||||
"sync_voc_report_constants",
|
||||
"validate_prompt_files",
|
||||
"word_categories",
|
||||
]
|
||||
26
prompts/config.yaml
Normal file
26
prompts/config.yaml
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# 产品/运营可修改：数值与中文词频分类枚举
|
||||
# 修改后下次运行脚本自动生效（无需重启）
|
||||
|
||||
word_freq:
|
||||
wordcloud_top_n: 180
|
||||
word_freq_table_n: 180
|
||||
word_category_classify_n: 180
|
||||
word_freq_page_size: 30
|
||||
word_freq_pages: 6
|
||||
word_category_assign_batch_size: 40
|
||||
word_category_analysis_max_words: 25
|
||||
|
||||
report:
|
||||
outlier_label_zh: "未归类"
|
||||
max_phrases_per_cluster: 15
|
||||
|
||||
# 词频分类（中文展示名；须与程序词云配色顺序一致）
|
||||
word_categories:
|
||||
- "成分/原料"
|
||||
- "剂型"
|
||||
- "受众/使用对象"
|
||||
- "功效/功能"
|
||||
- "需求/场景"
|
||||
- "品质/体验"
|
||||
- "价格/价值"
|
||||
- "物流/包装"
|
||||
7
prompts/extraction/batch_output_format.md
Normal file
7
prompts/extraction/batch_output_format.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
## 批量输出格式（必须严格遵守）:
|
||||
本批共 {n_keys} 条评论，用户消息中每条评论以 [C1]、[C2]… 前缀标识。
|
||||
- 只输出一个 JSON 对象；顶层键必须且仅能是：{keys_literal}
|
||||
- 每个顶层键对应一条同前缀评论，不得遗漏、不得新增其他顶层键
|
||||
- 每个键的值是单条结构化对象，仅含 audience、pain_points、product_feedback 三个字段
|
||||
- 不要用 JSON 数组作为顶层；不要把多条评论合并进一个对象；不要用 results、data 等包裹层
|
||||
- 不要输出 markdown 代码围栏或任何解释文字
|
||||
69
prompts/extraction/examples.yaml
Normal file
69
prompts/extraction/examples.yaml
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
# Few-shot 示例：output 为 YAML 对象，程序自动转为 JSON 字符串
|
||||
- instruction: "示例 1 教学：展示正常长评论如何标准提取，如何准确分类产品优点、缺陷以及物流问题。"
|
||||
review: "Bought this for my 12yo lab who struggles with stairs. It is very soft and helps her get onto the bed easily. But the zipper broke after a week and the shipping box was damaged."
|
||||
output:
|
||||
audience: "12yo lab"
|
||||
pain_points:
|
||||
- "struggles with stairs"
|
||||
- "difficulty getting onto bed"
|
||||
product_feedback:
|
||||
- aspect: "fabric"
|
||||
opinion: "very soft"
|
||||
sentiment: Positive
|
||||
category: Quality
|
||||
- aspect: "zipper"
|
||||
opinion: "broke after a week"
|
||||
sentiment: Negative
|
||||
category: Quality
|
||||
- aspect: "shipping box"
|
||||
opinion: "damaged"
|
||||
sentiment: Negative
|
||||
category: Logistics
|
||||
|
||||
- instruction: "示例 2 教学：防混淆与隐性属性提炼。什么是「产品缺陷（如讲太快）」，它属于 product_feedback，绝不是 pain_points（用户需求）！并提炼出 teaching speed 这个隐性反馈对象。"
|
||||
review: "The course content is good, the teacher speaks too fast, can't keep up."
|
||||
output:
|
||||
audience: "student"
|
||||
pain_points: []
|
||||
product_feedback:
|
||||
- aspect: "course content"
|
||||
opinion: "good"
|
||||
sentiment: Positive
|
||||
category: Quality
|
||||
- aspect: "teaching speed"
|
||||
opinion: "too fast to keep up"
|
||||
sentiment: Negative
|
||||
category: Function
|
||||
|
||||
- instruction: "示例 3 教学：深度上下文推理。如何通过动作（抓跳蚤）推理出使用者（pet），如何通过「全家能睡觉」推理出用户需求（失眠），以及如何将「Worth every penny」准确归类为对 Price（价格）的产品反馈。"
|
||||
review: "Finally found something that stops the midnight flea scratching! The chemical smell is a bit strong initially, but it fades. Worth every penny since our family can finally sleep."
|
||||
output:
|
||||
audience: "pet"
|
||||
pain_points:
|
||||
- "midnight flea scratching"
|
||||
- "sleep deprivation"
|
||||
product_feedback:
|
||||
- aspect: "chemical smell"
|
||||
opinion: "strong initially but fades"
|
||||
sentiment: Neutral
|
||||
category: Quality
|
||||
- aspect: "price"
|
||||
opinion: "worth every penny"
|
||||
sentiment: Positive
|
||||
category: Price
|
||||
|
||||
- instruction: "示例 4 教学：保留关键成分与症状等核心具体词汇。绝不能将带有具体成分/病症的词泛化提取。例如遇到「chicken flavor」（鸡肉风味）或「joint pain」（关节疼痛）时，必须保留核心修饰词，绝不能错误缩减提取为「flavor」或「pain」。"
|
||||
review: "My elderly cat suffers from severe joint pain. But he is very attracted by the chicken flavor of this supplement! It really helps him walk better."
|
||||
output:
|
||||
audience: "elderly cat"
|
||||
pain_points:
|
||||
- "severe joint pain"
|
||||
product_feedback:
|
||||
- aspect: "chicken flavor"
|
||||
opinion: "very attracted"
|
||||
sentiment: Positive
|
||||
category: Quality
|
||||
- aspect: "mobility improvement"
|
||||
opinion: "helps walk better"
|
||||
sentiment: Positive
|
||||
category: Function
|
||||
1
prompts/extraction/examples_footer_batch.md
Normal file
1
prompts/extraction/examples_footer_batch.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
(以上为单条示例；你对用户消息中的每一条带前缀评论分别做同样的结构化提取。)
|
||||
3
prompts/extraction/examples_section.md
Normal file
3
prompts/extraction/examples_section.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
## 分析示例 (请学习以下示例中的推理逻辑):
|
||||
在分析{product_name}时，你需要参考以下跨行业示例的逻辑，
|
||||
{examples_str}
|
||||
18
prompts/extraction/field_rules.md
Normal file
18
prompts/extraction/field_rules.md
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
## 分析要求:
|
||||
|
||||
1. audience (为谁购买):
|
||||
- 提取出实际的使用者，使用简短的英文名词。若无明确提及，请根据上下文推理；若完全无法推理则输出 'unknown'。
|
||||
|
||||
2. pain_points (用户需求):
|
||||
- 仅限提取用户在购买前遇到的外部困扰、疾病、或具体场景（购买前尚未被本产品解决的需求）。
|
||||
- **注意保留具体病症/需求核心词**（如 "joint pain" 不能缩减为 "pain"）。
|
||||
- **注意防重**：不要把「产品本身的缺陷或优点」当做用户需求提取。若无提及，输出空列表 []。
|
||||
|
||||
3. product_feedback (产品反馈):
|
||||
- 将用户对产品的反馈拆解为具体对象与反馈内容，包含以下子字段：
|
||||
- aspect (对象): 提炼成准确的英文简短名词。**务必保留具体的成分、材质或特定属性词**（例如 "chicken flavor" 不能泛化为 "flavor"）。
|
||||
- opinion (反馈内容): 必须是英文简短词组。
|
||||
- sentiment (情感): 仅限 {sentiments_literal}。
|
||||
- category (类别): 仅限 {categories_literal}（禁止 Value、Cost 等自创词；性价比高/物有所值 归入 Price）。
|
||||
- 每条 product_feedback 必须同时包含 aspect、opinion、sentiment、category 四个子字段。
|
||||
- 若无提及 product_feedback，输出空列表 []
|
||||
3
prompts/extraction/filter_irrelevant.md
Normal file
3
prompts/extraction/filter_irrelevant.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
5. 无关评论过滤：
|
||||
- 若某条评论明显与{product_name}无关（其他品类、其他 SKU 或完全跑题），该条输出：{{"audience": "unknown", "pain_points": [], "product_feedback": []}}。
|
||||
- 不得将无关内容填入 audience、pain_points 或 product_feedback。
|
||||
3
prompts/extraction/format_constraints_batch.md
Normal file
3
prompts/extraction/format_constraints_batch.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
4. 单条评论对象内的格式与字段约束：
|
||||
- 每个评论对象只能包含 `audience`, `pain_points`, `product_feedback` 这 3 个字段。
|
||||
- **绝对不要**在 JSON 中输出 `instruction`、`教学说明` 或其他任何多余字段。
|
||||
4
prompts/extraction/format_constraints_single.md
Normal file
4
prompts/extraction/format_constraints_single.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
4. 格式与字段约束：
|
||||
- **你的 JSON 输出只能包含 `audience`, `pain_points`, `product_feedback` 这 3 个根字段。**
|
||||
- **绝对不要**在 JSON 中输出 `instruction`、`教学说明` 或其他任何多余字段。
|
||||
- 必须以纯 JSON 格式输出结果，不要包含任何 markdown 标记（如 ```json ）或其他解释性文字。
|
||||
4
prompts/extraction/system_intro.md
Normal file
4
prompts/extraction/system_intro.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
你是一个专业的亚马逊电商评论分析专家，当前正在分析【{industry}】领域的【{product_name}】产品。
|
||||
请精准、简短地提炼评论中的核心信息，需要你具备深入的上下文推理能力（不仅仅是提取字面词汇，需结合{product_name}的使用语境）。
|
||||
|
||||
为了避免后续词频统计重复，请严格遵守各维度的定义，**同一概念或词汇绝对不能在不同字段中重复提取**。
|
||||
3
prompts/extraction/user_batch.md
Normal file
3
prompts/extraction/user_batch.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
以下为 {n_keys} 段带前缀的英文评论，每段互相独立。请严格按 system 中的「批量输出格式」返回 JSON，顶层键为 {keys_literal}。
|
||||
|
||||
{tagged_input}
|
||||
3
prompts/extraction/user_single_tail.md
Normal file
3
prompts/extraction/user_single_tail.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
## 请分析以下评论:
|
||||
[输入评论]: "{{review_content}}"
|
||||
[输出 JSON]:
|
||||
320
prompts/loader.py
Normal file
320
prompts/loader.py
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
"""
|
||||
外置 Prompt 加载器：每次调用均从磁盘重读 .md / .yaml（无缓存）。
|
||||
|
||||
编辑入口：
|
||||
prompts/config.yaml — 数值、中文词频分类（产品/运营可改）
|
||||
prompts/schema.yaml — 锁定 JSON 字段与标记（改前需开发确认）
|
||||
prompts/extraction/ — 结构化提取规则与示例
|
||||
prompts/report/ — 报告静态规则
|
||||
prompts/word_freq/ — 词频分类/解读
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
PROMPTS_ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def get_config() -> Dict[str, Any]:
|
||||
with (PROMPTS_ROOT / "config.yaml").open(encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
def get_schema() -> Dict[str, Any]:
|
||||
with (PROMPTS_ROOT / "schema.yaml").open(encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
def read_text(relative_path: str) -> str:
|
||||
return (PROMPTS_ROOT / relative_path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def render(relative_path: str, **kwargs: Any) -> str:
|
||||
return read_text(relative_path).format(**kwargs).strip()
|
||||
|
||||
|
||||
def word_categories(config: Dict[str, Any] | None = None) -> Tuple[str, ...]:
|
||||
cfg = config or get_config()
|
||||
return tuple(cfg.get("word_categories") or ())
|
||||
|
||||
|
||||
def report_markers(schema: Dict[str, Any] | None = None) -> Tuple[str, ...]:
|
||||
sch = schema or get_schema()
|
||||
return tuple(sch.get("report", {}).get("markers") or ())
|
||||
|
||||
|
||||
def product_feedback_categories(schema: Dict[str, Any] | None = None) -> frozenset[str]:
|
||||
sch = schema or get_schema()
|
||||
cats = sch.get("extraction", {}).get("product_feedback_categories") or []
|
||||
return frozenset(str(c) for c in cats)
|
||||
|
||||
|
||||
def _categories_literal(schema: Dict[str, Any]) -> str:
|
||||
cats = schema.get("extraction", {}).get("product_feedback_categories") or []
|
||||
return ", ".join(f"'{c}'" for c in cats)
|
||||
|
||||
|
||||
def _sentiments_literal(schema: Dict[str, Any]) -> str:
|
||||
sent = schema.get("extraction", {}).get("sentiments") or []
|
||||
return ", ".join(f"'{s}'" for s in sent)
|
||||
|
||||
|
||||
def load_extraction_examples() -> List[Dict[str, Any]]:
|
||||
with (PROMPTS_ROOT / "extraction/examples.yaml").open(encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or []
|
||||
|
||||
|
||||
def format_examples(examples: List[Dict[str, Any]] | None = None) -> str:
|
||||
items = examples if examples is not None else load_extraction_examples()
|
||||
lines: List[str] = []
|
||||
for i, ex in enumerate(items, 1):
|
||||
output_json = json.dumps(ex["output"], ensure_ascii=False, separators=(",", ":"))
|
||||
lines.append(f"=== 示例 {i} ===")
|
||||
lines.append(f"[教学说明]: {ex['instruction']}")
|
||||
lines.append(f"[输入评论]: \"{ex['review']}\"")
|
||||
lines.append(f"[输出 JSON]:\n{output_json}\n")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _extraction_vars(product_name: str, schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"product_name": product_name,
|
||||
"categories_literal": _categories_literal(schema),
|
||||
"sentiments_literal": _sentiments_literal(schema),
|
||||
"examples_str": format_examples(),
|
||||
}
|
||||
|
||||
|
||||
def build_batch_extraction_system(
|
||||
industry: str,
|
||||
product_name: str,
|
||||
*,
|
||||
n_keys: int,
|
||||
keys_literal: str,
|
||||
) -> str:
|
||||
schema = get_schema()
|
||||
common = _extraction_vars(product_name, schema)
|
||||
parts = [
|
||||
render("extraction/system_intro.md", industry=industry, product_name=product_name),
|
||||
render(
|
||||
"extraction/batch_output_format.md",
|
||||
n_keys=n_keys,
|
||||
keys_literal=keys_literal,
|
||||
),
|
||||
render("extraction/field_rules.md", **common),
|
||||
render("extraction/format_constraints_batch.md"),
|
||||
render("extraction/filter_irrelevant.md", product_name=product_name),
|
||||
render(
|
||||
"extraction/examples_section.md",
|
||||
product_name=product_name,
|
||||
examples_str=common["examples_str"],
|
||||
),
|
||||
render("extraction/examples_footer_batch.md"),
|
||||
]
|
||||
return "\n\n".join(p for p in parts if p.strip())
|
||||
|
||||
|
||||
def build_batch_extraction_user(
|
||||
*,
|
||||
n_keys: int,
|
||||
keys_literal: str,
|
||||
tagged_input: str,
|
||||
) -> str:
|
||||
return render(
|
||||
"extraction/user_batch.md",
|
||||
n_keys=n_keys,
|
||||
keys_literal=keys_literal,
|
||||
tagged_input=tagged_input,
|
||||
)
|
||||
|
||||
|
||||
def build_single_extraction_template(industry: str, product_name: str) -> str:
|
||||
schema = get_schema()
|
||||
common = _extraction_vars(product_name, schema)
|
||||
parts = [
|
||||
render("extraction/system_intro.md", industry=industry, product_name=product_name),
|
||||
render("extraction/field_rules.md", **common),
|
||||
render("extraction/format_constraints_single.md"),
|
||||
render("extraction/filter_irrelevant.md", product_name=product_name),
|
||||
render(
|
||||
"extraction/examples_section.md",
|
||||
product_name=product_name,
|
||||
examples_str=common["examples_str"],
|
||||
),
|
||||
render("extraction/user_single_tail.md"),
|
||||
]
|
||||
return "\n\n".join(p for p in parts if p.strip())
|
||||
|
||||
|
||||
def _report_marker_kwargs(
|
||||
config: Dict[str, Any],
|
||||
schema: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
markers = list(report_markers(schema))
|
||||
wf = config.get("word_freq", {})
|
||||
rp = config.get("report", {})
|
||||
return {
|
||||
"marker_word_zh": markers[0],
|
||||
"marker_word_category": markers[1],
|
||||
"marker_cluster_names": markers[2],
|
||||
"marker_report_html": markers[3],
|
||||
"wordcloud_top_n": wf.get("wordcloud_top_n", 180),
|
||||
"word_category_classify_n": wf.get("word_category_classify_n", 180),
|
||||
"outlier_label_zh": rp.get("outlier_label_zh", "未归类"),
|
||||
}
|
||||
|
||||
|
||||
def build_report_system() -> str:
|
||||
return render("report/system.md")
|
||||
|
||||
|
||||
def build_report_analysis_requirements(
|
||||
*,
|
||||
product_name: str,
|
||||
stage_1_audience: str,
|
||||
) -> str:
|
||||
return render(
|
||||
"report/analysis_requirements.md",
|
||||
product_name=product_name,
|
||||
stage_1_audience=stage_1_audience,
|
||||
)
|
||||
|
||||
|
||||
def build_report_output_format() -> str:
|
||||
return render("report/output_format.md", **_report_marker_kwargs(get_config(), get_schema()))
|
||||
|
||||
|
||||
def _word_category_definitions() -> str:
|
||||
return read_text("word_freq/category_definitions.md").strip()
|
||||
|
||||
|
||||
def build_report_json_markers(*, cats_literal: str) -> str:
|
||||
return render(
|
||||
"report/json_markers.md",
|
||||
cats_literal=cats_literal,
|
||||
category_definitions=_word_category_definitions(),
|
||||
**_report_marker_kwargs(get_config(), get_schema()),
|
||||
)
|
||||
|
||||
|
||||
def build_report_correction_message(*, err_block: str, raw_body: str, cats_literal: str) -> str:
|
||||
return render(
|
||||
"report/correction.md",
|
||||
err_block=err_block,
|
||||
raw_body=raw_body,
|
||||
cats_literal=cats_literal,
|
||||
**_report_marker_kwargs(get_config(), get_schema()),
|
||||
)
|
||||
|
||||
|
||||
def build_word_assign_prompts(
|
||||
*,
|
||||
industry: str,
|
||||
product_name: str,
|
||||
word_lines: str,
|
||||
cats_literal: str,
|
||||
) -> Tuple[str, str]:
|
||||
system = render(
|
||||
"word_freq/assign_system.md",
|
||||
cats_literal=cats_literal,
|
||||
category_definitions=_word_category_definitions(),
|
||||
)
|
||||
user = render(
|
||||
"word_freq/assign_user.md",
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
word_lines=word_lines,
|
||||
cats_literal=cats_literal,
|
||||
)
|
||||
return system, user
|
||||
|
||||
|
||||
def build_category_analysis_prompts(
|
||||
*,
|
||||
industry: str,
|
||||
product_name: str,
|
||||
category_lines: str,
|
||||
cats_literal: str,
|
||||
) -> Tuple[str, str]:
|
||||
system = render("word_freq/analysis_system.md")
|
||||
user = render(
|
||||
"word_freq/analysis_user.md",
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
category_lines=category_lines,
|
||||
cats_literal=cats_literal,
|
||||
)
|
||||
return system, user
|
||||
|
||||
|
||||
def sync_voc_report_constants(voc_report: Any) -> None:
|
||||
"""从 config.yaml / schema.yaml 同步 voc_report 模块级常量（每次 generate_report 前调用）。"""
|
||||
cfg = get_config()
|
||||
sch = get_schema()
|
||||
wf = cfg.get("word_freq", {})
|
||||
rp = cfg.get("report", {})
|
||||
|
||||
voc_report.WORDCLOUD_TOP_N = int(wf.get("wordcloud_top_n", 180))
|
||||
voc_report.WORD_FREQ_TABLE_N = int(wf.get("word_freq_table_n", 180))
|
||||
voc_report.WORD_CATEGORY_CLASSIFY_N = int(wf.get("word_category_classify_n", 180))
|
||||
voc_report.WORD_FREQ_PAGE_SIZE = int(wf.get("word_freq_page_size", 30))
|
||||
voc_report.WORD_FREQ_PAGES = int(wf.get("word_freq_pages", 6))
|
||||
voc_report.WORD_CATEGORY_ASSIGN_BATCH_SIZE = int(
|
||||
wf.get("word_category_assign_batch_size", 40)
|
||||
)
|
||||
voc_report.WORD_CATEGORY_ANALYSIS_MAX_WORDS = int(
|
||||
wf.get("word_category_analysis_max_words", 25)
|
||||
)
|
||||
voc_report.OUTLIER_LABEL_ZH = str(rp.get("outlier_label_zh", "未归类"))
|
||||
voc_report.MAX_PHRASES_PER_CLUSTER = int(rp.get("max_phrases_per_cluster", 15))
|
||||
voc_report.WORD_CATEGORIES = word_categories(cfg)
|
||||
voc_report.REPORT_MARKERS = report_markers(sch)
|
||||
|
||||
|
||||
def validate_prompt_files() -> List[str]:
|
||||
"""检查外置文件能否正常加载与渲染；返回错误列表（空=通过）。"""
|
||||
errors: List[str] = []
|
||||
try:
|
||||
cfg = get_config()
|
||||
sch = get_schema()
|
||||
if not word_categories(cfg):
|
||||
errors.append("config.yaml: word_categories 为空")
|
||||
if len(report_markers(sch)) != 4:
|
||||
errors.append("schema.yaml: report.markers 须为 4 项")
|
||||
if not product_feedback_categories(sch):
|
||||
errors.append("schema.yaml: extraction.product_feedback_categories 为空")
|
||||
format_examples()
|
||||
build_batch_extraction_system(
|
||||
"Test Industry",
|
||||
"Test Product",
|
||||
n_keys=2,
|
||||
keys_literal='"C1", "C2"',
|
||||
)
|
||||
build_batch_extraction_user(n_keys=2, keys_literal='"C1", "C2"', tagged_input="[C1] hi")
|
||||
build_single_extraction_template("Test", "Product")
|
||||
build_report_system()
|
||||
build_report_analysis_requirements(
|
||||
product_name="P",
|
||||
stage_1_audience="1_audience",
|
||||
)
|
||||
build_report_output_format()
|
||||
build_report_json_markers(cats_literal="成分/原料")
|
||||
build_word_assign_prompts(
|
||||
industry="I",
|
||||
product_name="P",
|
||||
word_lines="dog\t10",
|
||||
cats_literal="成分/原料",
|
||||
)
|
||||
build_category_analysis_prompts(
|
||||
industry="I",
|
||||
product_name="P",
|
||||
category_lines="- 成分/原料：a(1)",
|
||||
cats_literal="成分/原料",
|
||||
)
|
||||
except Exception as e:
|
||||
errors.append(f"渲染失败: {e}")
|
||||
return errors
|
||||
8
prompts/report/analysis_requirements.md
Normal file
8
prompts/report/analysis_requirements.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
**分析要求：**
|
||||
1. **一、摘要**：四条——核心受众与场景、最核心的用户需求、显著的产品反馈特征（正/负/客观）、提炼核心改进建议与机会（一句话）。
|
||||
2. **二、{product_name}评论分析**：严格按 HTML 模版层级填写。
|
||||
- **（一）受众画像与分析**：1.受众画像 —（1）受众群体特征、（2）受众的主要需求（仅用 {stage_1_audience} 各簇）；
|
||||
2.分受众分析 — 对 top2 受众分别写（1）用户需求（2）正面（3）负面（4）客观（仅用对应 2a/2b stage）。
|
||||
- **（二）全部用户需求与产品反馈**：1.全部受众需求（3a）、2.正面、3.负面、4.客观（3b 三档）；
|
||||
每个 <ul> 只收纳对应 stage 的簇；占比=结构化短语数÷a；同 ul 内按短语数降序。正文禁用「观点 / 评价 / 痛点」。
|
||||
3. **三、改进建议与机会**：保持模版 <h2> 与四条 <ol> 结构；建议须可执行，第 4 条单独写可放大的产品/市场机会；覆盖未满足需求、负面反馈与客观描述中的风险，勿复述本条款文字。
|
||||
15
prompts/report/correction.md
Normal file
15
prompts/report/correction.md
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
你上一次输出未通过程序自动校验，请修正后**重新输出完整的四段标记内容**（四段都要给出，即使某段上次已正确也请原样附上）。
|
||||
|
||||
【校验错误】
|
||||
{err_block}
|
||||
|
||||
【修正要点】
|
||||
- 四个标记必须按顺序出现：{marker_word_zh}、{marker_word_category}、{marker_cluster_names}、{marker_report_html}（JSON 在前，HTML 最后）
|
||||
- JSON 段必须是严格合法 JSON：键与字符串均用英文双引号；例如 "words":["dog","cat"]，禁止写成 "words:["dog"] 或 words:[
|
||||
- 不要用 Markdown 代码围栏包裹 JSON；WORD_CATEGORY_JSON 的键仅限：{cats_literal}
|
||||
- WORD_CATEGORY_JSON 必须是**一个** JSON 对象，words 仅来自 Top{word_category_classify_n}；逐词尽量归类、不强制每类凑满；禁止「其他」键；禁止多段 `{{"某类":[...]}}, "另一类":[...]`
|
||||
- 务必输出完整的 {marker_cluster_names} 段
|
||||
- REPORT_HTML 为完整 <html lang="zh-CN">...</html> 文档，放在最后一段
|
||||
|
||||
【你上一次的完整输出（请对照修改）】
|
||||
{raw_body}
|
||||
24
prompts/report/json_markers.md
Normal file
24
prompts/report/json_markers.md
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
请严格按以下四段标记输出（标记外不要有任何文字；**顺序不可调换**）：
|
||||
|
||||
{marker_word_zh}
|
||||
一行合法 JSON 对象：Top{wordcloud_top_n} 英文词 -> 中文，如 {{"dog":"狗","cat":"猫"}}。
|
||||
|
||||
{marker_word_category}
|
||||
**仅一个** JSON 对象（禁止拆成多段 `{{"某类":[...]}}, "另一类":[...]`）。
|
||||
键仅限：{cats_literal}；每类值为 {{"words":["英文词",...],"analysis":"中文解读"}}
|
||||
|
||||
{category_definitions}
|
||||
|
||||
要求：
|
||||
- 对「词频分类词表 Top{word_category_classify_n}」**逐词**判断是否可归类；仅语义明确贴合时才写入对应类 words，一词可多类
|
||||
- **不强制**每类都有词、**不强制**覆盖全部词；无法判断或仅为噪声的词**不要**硬塞进任何类（程序会对未分类词二次尝试）
|
||||
- words 必须全部来自 Top{word_category_classify_n} 列表；每类按词频降序；每类须写 analysis（基于已归入的词总结）；禁止「其他」键
|
||||
|
||||
{marker_cluster_names}
|
||||
一行合法 JSON 对象：键为聚类原始 id（必须与上文 id 严格一致），值为中文簇名；整段仅一对最外层花括号 `{{...}}`，勿多写结尾 `}}`。
|
||||
簇标签 -1 统一命名为「{outlier_label_zh}」。
|
||||
命名规则：1_audience、3a_pain_global、3b_aspect_opinion_* 写完整业务簇名；
|
||||
2a/2b 只写子主题（程序拼接受众名）；2b 子主题禁止含情感词。
|
||||
|
||||
{marker_report_html}
|
||||
（最后输出）完整 HTML 文档，结构遵循上文模版；勿用 Markdown 代码围栏包裹。
|
||||
8
prompts/report/output_format.md
Normal file
8
prompts/report/output_format.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
**输出格式（必须严格遵守）：**
|
||||
1. 只允许输出标准 HTML，不允许 Markdown。
|
||||
2. {marker_report_html} 内必须是完整 HTML 文档，包含 <html>、<head>、<body>；可保留模版中的 <!-- 映射注释 -->，浏览器不会显示。
|
||||
3. 允许标签：<h1><h2><h3><h4><p><ul><ol><li><strong>；列表项格式须为 <strong>纯中文簇名</strong> (XX.X%)：洞察。
|
||||
4. 禁止 Markdown（#、**、``` 等）及 <table> 等未列出的标签。
|
||||
5. 【红线】正文可见文字中禁止出现 1_audience|2、2a_pain_audience_c2|0、stage=、audience_c 等任何机器标识；仅 CLUSTER_NAMES JSON 键可保留原始 id。
|
||||
6. 不要输出解释性前后缀；REPORT_HTML 段内只放 HTML 文档本体。
|
||||
7. 四段标记顺序固定：先三个 JSON 段（WORD_ZH → WORD_CATEGORY → CLUSTER_NAMES），**最后**输出 REPORT_HTML，避免长 HTML 导致 JSON 被截断。
|
||||
4
prompts/report/system.md
Normal file
4
prompts/report/system.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
你是一位非常资深且专业的亚马逊美国站运营师，正在撰写面向业务高管的 VOC 改进报告。
|
||||
报告必须专业、可读，正文中绝对禁止出现原始聚类 ID、stage 代码名或簇标签编号。
|
||||
展示用语统一：用户需求、产品反馈、产品客观描述；禁止观点、评价、痛点等旧称。
|
||||
你必须严格遵守 HTML 输出规范，禁止 Markdown。
|
||||
31
prompts/schema.yaml
Normal file
31
prompts/schema.yaml
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# 🔒 程序锁定区：修改前请与开发确认，否则可能导致 JSON 解析失败
|
||||
# 字段名、英文 category、输出标记等与下游 SQLite / 聚类 / 报告解析绑定
|
||||
|
||||
extraction:
|
||||
root_fields:
|
||||
- audience
|
||||
- pain_points
|
||||
- product_feedback
|
||||
product_feedback_fields:
|
||||
- aspect
|
||||
- opinion
|
||||
- sentiment
|
||||
- category
|
||||
product_feedback_categories:
|
||||
- Quality
|
||||
- Function
|
||||
- Appearance
|
||||
- Logistics
|
||||
- Customer Service
|
||||
- Price
|
||||
sentiments:
|
||||
- Positive
|
||||
- Negative
|
||||
- Neutral
|
||||
|
||||
report:
|
||||
markers:
|
||||
- "===WORD_ZH_JSON==="
|
||||
- "===WORD_CATEGORY_JSON==="
|
||||
- "===CLUSTER_NAMES_JSON==="
|
||||
- "===REPORT_HTML==="
|
||||
139
prompts/smoke.py
Normal file
139
prompts/smoke.py
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Prompt 验收脚本（修改 prompts/ 后运行）。
|
||||
|
||||
python3 prompts/smoke.py # 仅校验文件加载与渲染（无需 API Key）
|
||||
python3 prompts/smoke.py --live # 调用模型跑 smoke/reviews.yaml（需 DASHSCOPE_API_KEY）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
import yaml
|
||||
|
||||
from prompts.loader import (
|
||||
build_batch_extraction_system,
|
||||
build_batch_extraction_user,
|
||||
get_schema,
|
||||
product_feedback_categories,
|
||||
validate_prompt_files,
|
||||
)
|
||||
|
||||
|
||||
def _load_smoke_reviews() -> dict:
|
||||
path = Path(__file__).resolve().parent / "smoke/reviews.yaml"
|
||||
with path.open(encoding="utf-8") as f:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
def run_dry() -> int:
|
||||
errors = validate_prompt_files()
|
||||
if errors:
|
||||
print("❌ Prompt 文件校验失败：")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
return 1
|
||||
smoke = _load_smoke_reviews()
|
||||
industry = smoke.get("industry", "Test")
|
||||
product = smoke.get("product_name", "Test")
|
||||
reviews = smoke.get("reviews") or []
|
||||
keys = [str(r["id"]) for r in reviews]
|
||||
tagged = "\n".join(f"[{r['id']}] {r['text']}" for r in reviews)
|
||||
keys_literal = ", ".join(json.dumps(k) for k in keys)
|
||||
system = build_batch_extraction_system(
|
||||
industry,
|
||||
product,
|
||||
n_keys=len(keys),
|
||||
keys_literal=keys_literal,
|
||||
)
|
||||
user = build_batch_extraction_user(
|
||||
n_keys=len(keys),
|
||||
keys_literal=keys_literal,
|
||||
tagged_input=tagged,
|
||||
)
|
||||
print("✅ 外置 Prompt 加载与渲染通过")
|
||||
print(f" 示例评论数: {len(reviews)}")
|
||||
print(f" system 长度: {len(system)} 字符")
|
||||
print(f" user 长度: {len(user)} 字符")
|
||||
return 0
|
||||
|
||||
|
||||
def run_live() -> int:
|
||||
rc = run_dry()
|
||||
if rc != 0:
|
||||
return rc
|
||||
try:
|
||||
from 结构化_server import _call_dashscope_chat, _validate_extraction_strict
|
||||
except ImportError as e:
|
||||
print(f"❌ 无法导入结构化_server: {e}")
|
||||
return 1
|
||||
|
||||
smoke = _load_smoke_reviews()
|
||||
industry = smoke["industry"]
|
||||
product = smoke["product_name"]
|
||||
reviews = smoke["reviews"]
|
||||
keys = [str(r["id"]) for r in reviews]
|
||||
tagged = "\n".join(f"[{r['id']}] {r['text']}" for r in reviews)
|
||||
keys_literal = ", ".join(json.dumps(k) for k in keys)
|
||||
system = build_batch_extraction_system(
|
||||
industry,
|
||||
product,
|
||||
n_keys=len(keys),
|
||||
keys_literal=keys_literal,
|
||||
)
|
||||
user = build_batch_extraction_user(
|
||||
n_keys=len(keys),
|
||||
keys_literal=keys_literal,
|
||||
tagged_input=tagged,
|
||||
)
|
||||
print("⏳ 调用模型进行 smoke 结构化…")
|
||||
try:
|
||||
raw = _call_dashscope_chat(
|
||||
[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||||
max_tokens=8192,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"❌ 模型调用失败: {e}")
|
||||
return 1
|
||||
|
||||
if not isinstance(raw, dict):
|
||||
print(f"❌ 期望 JSON 对象，得到: {type(raw).__name__}")
|
||||
return 1
|
||||
|
||||
ok = 0
|
||||
for k in keys:
|
||||
if k not in raw:
|
||||
print(f"❌ 缺少键 {k}")
|
||||
continue
|
||||
try:
|
||||
_validate_extraction_strict(raw[k], k)
|
||||
ok += 1
|
||||
print(f"✅ {k} 校验通过")
|
||||
except ValueError as e:
|
||||
print(f"❌ {k} 校验失败: {e}")
|
||||
|
||||
schema_cats = product_feedback_categories(get_schema())
|
||||
print(f" 锁定 category 枚举: {', '.join(sorted(schema_cats))}")
|
||||
return 0 if ok == len(keys) else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Prompt smoke 验收")
|
||||
parser.add_argument(
|
||||
"--live",
|
||||
action="store_true",
|
||||
help="调用 DashScope 跑 smoke 评论（需 API Key）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return run_live() if args.live else run_dry()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
12
prompts/smoke/reviews.yaml
Normal file
12
prompts/smoke/reviews.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Smoke 验收用评论（修改 prompt 后运行 python prompts/smoke.py 验证）
|
||||
|
||||
industry: "Pet supplements"
|
||||
product_name: "Turkey tail mushroom for dogs"
|
||||
|
||||
reviews:
|
||||
- id: C1
|
||||
text: "My senior dog has weak immunity. This turkey tail powder mixes easily and he eats it without fuss. Shipping was fast."
|
||||
- id: C2
|
||||
text: "Good quality mushrooms but the bag seal broke and powder spilled everywhere. Too expensive for the amount."
|
||||
- id: C3
|
||||
text: "Random spam about phone cases, not related to dog supplements at all."
|
||||
1
prompts/word_freq/analysis_system.md
Normal file
1
prompts/word_freq/analysis_system.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
你是亚马逊 VOC 词频分析专家。根据每类已归入的英文高频词，写一两句简洁的中文解读（说明该类词反映的用户关注点，勿罗列词表）。
|
||||
9
prompts/word_freq/analysis_user.md
Normal file
9
prompts/word_freq/analysis_user.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
行业：{industry}
|
||||
产品：{product_name}
|
||||
|
||||
【须撰写 analysis 的类别及已归入词（word(count)）】
|
||||
{category_lines}
|
||||
|
||||
请输出**仅一个** JSON 对象：键为类别名（仅限：{cats_literal}），值为 1～2 句中文解读字符串。
|
||||
- 只输出上述列出的类别；不要 Markdown 围栏
|
||||
- 示例：{{"成分/原料":"用户高度关注火鸡尾等药用真菌原料…","受众/使用对象":"以老年犬与患病犬为主…"}}
|
||||
4
prompts/word_freq/assign_system.md
Normal file
4
prompts/word_freq/assign_system.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
你是亚马逊 VOC 词频分类专家。对给定英文词**逐一**判断是否属于以下类别之一或多个：{cats_literal}。
|
||||
拿不准、仅为噪声或与品类无关的词返回空数组 []，不要硬分类。
|
||||
|
||||
{category_definitions}
|
||||
12
prompts/word_freq/assign_user.md
Normal file
12
prompts/word_freq/assign_user.md
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
行业：{industry}
|
||||
产品：{product_name}
|
||||
|
||||
【本批待分类词（word\tcount，拼写须原样作为 JSON 键）】
|
||||
{word_lines}
|
||||
|
||||
请输出**仅一个** JSON 对象：键为本批每个英文词（与上表拼写完全一致），值为类别名数组。
|
||||
- 值仅限：{cats_literal} 中的 0～多个类别；一词可多类
|
||||
- 无法明确归类时值为 []，不要编造类别
|
||||
- 不要输出「其他」；不要 Markdown 围栏
|
||||
|
||||
示例：{{"dog":["受众/使用对象"],"turmeric":["成分/原料","功效/功能"],"powder":["剂型"],"xyz":[]}}
|
||||
10
prompts/word_freq/category_definitions.md
Normal file
10
prompts/word_freq/category_definitions.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
## 词频各类定义（分类时严格遵守）
|
||||
|
||||
- **成分/原料**：核心成分、原料、主材名词（如 turmeric、mushroom、beta-glucan）；定语或形容词禁止写入此类（如natural、organic 等）。
|
||||
- **剂型**：产品的物理形态与给药/食用形式（如 powder、capsule、chew、tablet、soft chew、liquid、treat、granule、sachet、drop）。与成分本身、功效描述区分；定语或形容词不归此类。
|
||||
- **受众/使用对象**：使用者或购买对象（dog、puppy、senior cat…）；定语或形容词不归此类。
|
||||
- **功效/功能**：作用、效果、功能（immunity、digestion、joint support…），不是成分名或剂型。
|
||||
- **需求/场景**：使用场景、未满足需求（allergy、anxiety、appetite…）。
|
||||
- **品质/体验**：口感、气味、质地、纯度等体验词（taste、smell、organic、natural…）。
|
||||
- **价格/价值**：价格、性价比相关（price、worth、expensive…）。
|
||||
- **物流/包装**：包装、运输、到货（packaging、shipping、box…）。
|
||||
10
pyproject.toml
Normal file
10
pyproject.toml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
[project]
|
||||
name = "voc-llm-structured"
|
||||
version = "0.1.0"
|
||||
description = "Structured VOC review extraction via DashScope qwen3.6-flash"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"mcp>=1.2.0",
|
||||
"openai>=1.40.0",
|
||||
"pyyaml>=6.0",
|
||||
]
|
||||
8
requirements.txt
Normal file
8
requirements.txt
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
openai>=1.40.0
|
||||
numpy>=1.24.0
|
||||
umap-learn>=0.5.5
|
||||
hdbscan>=0.8.33
|
||||
scikit-learn>=1.3.0
|
||||
spacy>=3.7.0
|
||||
pyyaml>=6.0
|
||||
jieba>=0.42.1
|
||||
2415
voc_report.py
Normal file
2415
voc_report.py
Normal file
File diff suppressed because it is too large
Load diff
103
合并评论数据.py
Normal file
103
合并评论数据.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""
|
||||
合并指定目录下所有表头一致的 CSV 文件为一个 CSV。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
DEFAULT_INPUT_DIR = _PROJECT_ROOT / "turkey tail mushroom-voc"
|
||||
DEFAULT_OUTPUT_PATH = _PROJECT_ROOT / "merged_reviews.csv"
|
||||
|
||||
|
||||
def merge_csv_directory(
|
||||
input_dir: str | Path,
|
||||
output_path: str | Path,
|
||||
*,
|
||||
pattern: str = "*.csv",
|
||||
recursive: bool = False,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
读取目录内所有匹配的 CSV（表头须一致），纵向合并后写入 output_path。
|
||||
|
||||
:param input_dir: 含多个 CSV 的目录
|
||||
:param output_path: 合并结果输出路径
|
||||
:param pattern: 文件名 glob，默认 *.csv
|
||||
:param recursive: 是否包含子目录中的 CSV
|
||||
:return: 合并后的 DataFrame
|
||||
"""
|
||||
input_dir = Path(input_dir).resolve()
|
||||
output_path = Path(output_path).resolve()
|
||||
|
||||
if not input_dir.is_dir():
|
||||
raise NotADirectoryError(f"目录不存在: {input_dir}")
|
||||
|
||||
globber = input_dir.rglob if recursive else input_dir.glob
|
||||
csv_files = sorted(
|
||||
p for p in globber(pattern) if p.is_file() and p.resolve() != output_path
|
||||
)
|
||||
|
||||
if not csv_files:
|
||||
raise FileNotFoundError(f"未在 {input_dir} 找到匹配 {pattern!r} 的 CSV 文件")
|
||||
|
||||
frames: list[pd.DataFrame] = []
|
||||
expected_columns: list[str] | None = None
|
||||
|
||||
for path in csv_files:
|
||||
df = pd.read_csv(path, dtype=str, keep_default_na=False)
|
||||
if expected_columns is None:
|
||||
expected_columns = list(df.columns)
|
||||
elif list(df.columns) != expected_columns:
|
||||
raise ValueError(
|
||||
f"表头不一致: {path.name}\n"
|
||||
f" 期望: {expected_columns}\n"
|
||||
f" 实际: {list(df.columns)}"
|
||||
)
|
||||
frames.append(df)
|
||||
|
||||
merged = pd.concat(frames, ignore_index=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
merged.to_csv(output_path, index=False, encoding="utf-8-sig")
|
||||
|
||||
print(f"已合并 {len(csv_files)} 个文件，共 {len(merged)} 行 -> {output_path}")
|
||||
for path in csv_files:
|
||||
print(f" - {path.name}")
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="合并目录内表头相同的 CSV 文件")
|
||||
parser.add_argument(
|
||||
"-i",
|
||||
"--input",
|
||||
type=Path,
|
||||
default=DEFAULT_INPUT_DIR,
|
||||
metavar="DIR",
|
||||
help=f"输入目录（默认: {DEFAULT_INPUT_DIR}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DEFAULT_OUTPUT_PATH,
|
||||
metavar="FILE",
|
||||
help=f"输出 CSV 路径（默认: {DEFAULT_OUTPUT_PATH}）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-r",
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
help="是否递归搜索子目录",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
merge_csv_directory(args.input, args.output, recursive=args.recursive)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
534
向量化.py
Normal file
534
向量化.py
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
"""
|
||||
从 voc_structured.sqlite 最新 job 展开 audience / pain_points / aspect / opinion /
|
||||
aspect_opinion，调用 DashScope text-embedding-v4（256 维）写入 voc_embeddings.sqlite。
|
||||
|
||||
溯源：source_row 与结构化时一致（CSV 第 1 条数据行=1）；对应 merged_reviews_cleaned.csv
|
||||
物理行号 = source_row + 1（第 1 行为表头），content 取自该数据行。
|
||||
|
||||
用法（项目根目录）::
|
||||
|
||||
python3 向量化.py
|
||||
python3 向量化.py --workers 8
|
||||
python3 向量化.py --job-id 4 --csv merged_reviews_cleaned.csv
|
||||
|
||||
说明：DashScope text-embedding-v4 单次请求最多 10 条文本；脚本按批调用 API，
|
||||
默认多线程并行多批（--workers），并非逐条请求。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import struct
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Sequence, Tuple
|
||||
|
||||
from csv import DictReader
|
||||
from openai import OpenAI
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("voc_embed")
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||||
EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite"
|
||||
DEFAULT_CSV = PROJECT_ROOT / "merged_reviews_cleaned.csv"
|
||||
DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
EMBEDDING_MODEL = "text-embedding-v4"
|
||||
EMBEDDING_DIMENSIONS = 256
|
||||
EMBED_BATCH_SIZE = 10 # DashScope text-embedding-v4 单请求 input 数组上限 10
|
||||
EMBED_DEFAULT_WORKERS = 6 # 并行批次数（每批最多 10 条）
|
||||
ENTITY_TYPES = (
|
||||
"audience",
|
||||
"pain_point",
|
||||
"aspect",
|
||||
"opinion",
|
||||
"aspect_opinion",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbedTask:
|
||||
job_id: int
|
||||
extraction_id: int
|
||||
source_row: int
|
||||
entity_type: str
|
||||
entity_index: int
|
||||
embed_text: str
|
||||
audience: str
|
||||
aspect: str | None
|
||||
opinion: str | None
|
||||
category: str | None
|
||||
sentiment: str | None
|
||||
content: str
|
||||
|
||||
|
||||
def _resolve_api_key() -> str:
|
||||
v = os.environ.get("DASHSCOPE_API_KEY", "").strip()
|
||||
if v:
|
||||
return v
|
||||
fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip()
|
||||
if fp:
|
||||
p = Path(fp).expanduser()
|
||||
if p.is_file():
|
||||
return p.read_text(encoding="utf-8").strip().strip('"').strip("'")
|
||||
local = PROJECT_ROOT / ".dashscope_key"
|
||||
if local.is_file():
|
||||
return local.read_text(encoding="utf-8").strip().strip('"').strip("'")
|
||||
return ""
|
||||
|
||||
|
||||
def _latest_job_id(conn: sqlite3.Connection) -> int:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise RuntimeError("analysis_jobs 为空，请先运行结构化")
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def _load_content_by_source_row(csv_path: Path) -> Dict[int, str]:
|
||||
"""source_row（1 起）-> content；与结构化_server._load_csv_reviews 行号一致。"""
|
||||
with csv_path.open(encoding="utf-8-sig", newline="") as f:
|
||||
reader = DictReader(f)
|
||||
if not reader.fieldnames:
|
||||
raise ValueError(f"CSV 无表头: {csv_path}")
|
||||
headers = [h.strip() for h in reader.fieldnames]
|
||||
lower_map = {h.lower(): h for h in headers}
|
||||
col = None
|
||||
for name in ("content", "review", "评论"):
|
||||
if name.lower() in lower_map:
|
||||
col = lower_map[name.lower()]
|
||||
break
|
||||
if col is None:
|
||||
if len(headers) == 1:
|
||||
col = headers[0]
|
||||
else:
|
||||
raise ValueError(f"CSV 缺少 content 列: {headers}")
|
||||
out: Dict[int, str] = {}
|
||||
for idx, row in enumerate(reader, start=1):
|
||||
cell = row.get(col)
|
||||
t = str(cell or "").strip()
|
||||
if t:
|
||||
out[idx] = t
|
||||
return out
|
||||
|
||||
|
||||
def _pack_embedding(vec: Sequence[float]) -> bytes:
|
||||
return struct.pack(f"{len(vec)}f", *vec)
|
||||
|
||||
|
||||
def _init_embed_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS embedding_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_id INTEGER NOT NULL,
|
||||
extraction_id INTEGER NOT NULL,
|
||||
source_row INTEGER NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_index INTEGER NOT NULL DEFAULT 0,
|
||||
embed_text TEXT NOT NULL,
|
||||
audience TEXT,
|
||||
aspect TEXT,
|
||||
opinion TEXT,
|
||||
category TEXT,
|
||||
sentiment TEXT,
|
||||
content TEXT NOT NULL,
|
||||
dimensions INTEGER NOT NULL,
|
||||
embedding BLOB NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_embed_job_type
|
||||
ON embedding_items(job_id, entity_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_embed_source_row
|
||||
ON embedding_items(job_id, source_row);
|
||||
CREATE INDEX IF NOT EXISTS idx_embed_extraction
|
||||
ON embedding_items(extraction_id, entity_type, entity_index);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _should_skip_extraction(audience: str, pain_points: List[Any], feedback: List[Any]) -> bool:
|
||||
aud = (audience or "").strip().lower()
|
||||
return aud == "unknown" and not pain_points and not feedback
|
||||
|
||||
|
||||
def _build_tasks(
|
||||
job_id: int,
|
||||
rows: List[sqlite3.Row],
|
||||
content_map: Dict[int, str],
|
||||
) -> List[EmbedTask]:
|
||||
tasks: List[EmbedTask] = []
|
||||
for row in rows:
|
||||
ext_id = int(row["id"])
|
||||
source_row = int(row["source_row"])
|
||||
try:
|
||||
data = json.loads(row["extraction_json"])
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("跳过无效 JSON extraction_id=%s: %s", ext_id, e)
|
||||
continue
|
||||
|
||||
audience = str(data.get("audience", "unknown")).strip() or "unknown"
|
||||
pain_points = data.get("pain_points") or []
|
||||
feedback = data.get("product_feedback") or []
|
||||
if not isinstance(pain_points, list):
|
||||
pain_points = []
|
||||
if not isinstance(feedback, list):
|
||||
feedback = []
|
||||
|
||||
if _should_skip_extraction(audience, pain_points, feedback):
|
||||
continue
|
||||
|
||||
content = content_map.get(source_row, "")
|
||||
if not content:
|
||||
logger.warning(
|
||||
"source_row=%s 在 CSV 中无 content，仍写入向量但 content 为空",
|
||||
source_row,
|
||||
)
|
||||
|
||||
aud_norm = audience.strip()
|
||||
if aud_norm.lower() != "unknown":
|
||||
tasks.append(
|
||||
EmbedTask(
|
||||
job_id=job_id,
|
||||
extraction_id=ext_id,
|
||||
source_row=source_row,
|
||||
entity_type="audience",
|
||||
entity_index=0,
|
||||
embed_text=aud_norm,
|
||||
audience=aud_norm,
|
||||
aspect=None,
|
||||
opinion=None,
|
||||
category=None,
|
||||
sentiment=None,
|
||||
content=content,
|
||||
)
|
||||
)
|
||||
|
||||
for i, p in enumerate(pain_points):
|
||||
text = str(p).strip()
|
||||
if not text:
|
||||
continue
|
||||
tasks.append(
|
||||
EmbedTask(
|
||||
job_id=job_id,
|
||||
extraction_id=ext_id,
|
||||
source_row=source_row,
|
||||
entity_type="pain_point",
|
||||
entity_index=i,
|
||||
embed_text=text,
|
||||
audience=aud_norm,
|
||||
aspect=None,
|
||||
opinion=None,
|
||||
category=None,
|
||||
sentiment=None,
|
||||
content=content,
|
||||
)
|
||||
)
|
||||
|
||||
for i, item in enumerate(feedback):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
aspect = str(item.get("aspect", "")).strip()
|
||||
opinion = str(item.get("opinion", "")).strip()
|
||||
category = str(item.get("category", "")).strip() or None
|
||||
sentiment = str(item.get("sentiment", "")).strip() or None
|
||||
if not aspect and not opinion:
|
||||
continue
|
||||
if aspect:
|
||||
tasks.append(
|
||||
EmbedTask(
|
||||
job_id=job_id,
|
||||
extraction_id=ext_id,
|
||||
source_row=source_row,
|
||||
entity_type="aspect",
|
||||
entity_index=i,
|
||||
embed_text=aspect,
|
||||
audience=aud_norm,
|
||||
aspect=aspect,
|
||||
opinion=opinion or None,
|
||||
category=category,
|
||||
sentiment=sentiment,
|
||||
content=content,
|
||||
)
|
||||
)
|
||||
if opinion:
|
||||
tasks.append(
|
||||
EmbedTask(
|
||||
job_id=job_id,
|
||||
extraction_id=ext_id,
|
||||
source_row=source_row,
|
||||
entity_type="opinion",
|
||||
entity_index=i,
|
||||
embed_text=opinion,
|
||||
audience=aud_norm,
|
||||
aspect=aspect or None,
|
||||
opinion=opinion,
|
||||
category=category,
|
||||
sentiment=sentiment,
|
||||
content=content,
|
||||
)
|
||||
)
|
||||
if aspect and opinion:
|
||||
merged = f"{aspect}, {opinion}"
|
||||
tasks.append(
|
||||
EmbedTask(
|
||||
job_id=job_id,
|
||||
extraction_id=ext_id,
|
||||
source_row=source_row,
|
||||
entity_type="aspect_opinion",
|
||||
entity_index=i,
|
||||
embed_text=merged,
|
||||
audience=aud_norm,
|
||||
aspect=aspect,
|
||||
opinion=opinion,
|
||||
category=category,
|
||||
sentiment=sentiment,
|
||||
content=content,
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
|
||||
|
||||
def _embed_one_api_batch(api_key: str, texts: List[str]) -> List[bytes]:
|
||||
"""单次 API 调用（最多 batch_size 条文本）。"""
|
||||
client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL)
|
||||
resp = client.embeddings.create(
|
||||
model=EMBEDDING_MODEL,
|
||||
input=texts,
|
||||
dimensions=EMBEDDING_DIMENSIONS,
|
||||
)
|
||||
if len(resp.data) != len(texts):
|
||||
raise RuntimeError(
|
||||
f"embedding 返回条数 {len(resp.data)} != 请求 {len(texts)}"
|
||||
)
|
||||
ordered = sorted(resp.data, key=lambda d: d.index)
|
||||
blobs: List[bytes] = []
|
||||
for item in ordered:
|
||||
vec = item.embedding
|
||||
if len(vec) != EMBEDDING_DIMENSIONS:
|
||||
raise RuntimeError(f"维度 {len(vec)} != 期望 {EMBEDDING_DIMENSIONS}")
|
||||
blobs.append(_pack_embedding(vec))
|
||||
return blobs
|
||||
|
||||
|
||||
def _embed_batches(
|
||||
api_key: str,
|
||||
tasks: List[EmbedTask],
|
||||
*,
|
||||
batch_size: int = EMBED_BATCH_SIZE,
|
||||
workers: int = EMBED_DEFAULT_WORKERS,
|
||||
) -> List[bytes]:
|
||||
"""
|
||||
将任务切成每批最多 batch_size 条，并行请求 DashScope。
|
||||
返回与 tasks 顺序一致的 embedding BLOB 列表。
|
||||
"""
|
||||
if batch_size < 1 or batch_size > 10:
|
||||
raise ValueError("batch_size 须在 1–10 之间（DashScope 单请求上限 10）")
|
||||
workers = max(1, workers)
|
||||
|
||||
chunks: List[List[EmbedTask]] = [
|
||||
tasks[i : i + batch_size] for i in range(0, len(tasks), batch_size)
|
||||
]
|
||||
n_chunks = len(chunks)
|
||||
if n_chunks == 0:
|
||||
return []
|
||||
|
||||
logger.info(
|
||||
"共 %s 条文本，%s 批（每批≤%s 条），并行 workers=%s",
|
||||
len(tasks),
|
||||
n_chunks,
|
||||
batch_size,
|
||||
min(workers, n_chunks),
|
||||
)
|
||||
|
||||
# 单线程：逻辑简单，便于限流环境
|
||||
if workers == 1:
|
||||
all_blobs: List[bytes] = []
|
||||
for i, chunk in enumerate(chunks, start=1):
|
||||
blobs = _embed_one_api_batch(api_key, [t.embed_text for t in chunk])
|
||||
all_blobs.extend(blobs)
|
||||
if i == n_chunks or i % 20 == 0:
|
||||
logger.info("Embedding 进度 %s/%s 批", i, n_chunks)
|
||||
return all_blobs
|
||||
|
||||
results: List[List[bytes] | None] = [None] * n_chunks
|
||||
done = 0
|
||||
with ThreadPoolExecutor(max_workers=min(workers, n_chunks)) as pool:
|
||||
future_map = {
|
||||
pool.submit(
|
||||
_embed_one_api_batch,
|
||||
api_key,
|
||||
[t.embed_text for t in chunk],
|
||||
): idx
|
||||
for idx, chunk in enumerate(chunks)
|
||||
}
|
||||
for fut in as_completed(future_map):
|
||||
idx = future_map[fut]
|
||||
results[idx] = fut.result()
|
||||
done += 1
|
||||
if done == n_chunks or done % 20 == 0:
|
||||
logger.info("Embedding 进度 %s/%s 批", done, n_chunks)
|
||||
|
||||
return [blob for batch in results for blob in batch] # type: ignore[union-attr]
|
||||
|
||||
|
||||
def run_embed(
|
||||
*,
|
||||
job_id: int | None = None,
|
||||
csv_path: Path = DEFAULT_CSV,
|
||||
structured_db: Path = STRUCTURED_DB,
|
||||
embed_db: Path = EMBED_DB,
|
||||
batch_size: int = EMBED_BATCH_SIZE,
|
||||
workers: int = EMBED_DEFAULT_WORKERS,
|
||||
reset_db: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
api_key = _resolve_api_key()
|
||||
if not api_key:
|
||||
raise RuntimeError(
|
||||
"缺少 DASHSCOPE_API_KEY 或项目根 .dashscope_key"
|
||||
)
|
||||
|
||||
csv_path = csv_path.expanduser().resolve()
|
||||
if not csv_path.is_file():
|
||||
raise FileNotFoundError(csv_path)
|
||||
if not structured_db.is_file():
|
||||
raise FileNotFoundError(structured_db)
|
||||
|
||||
content_map = _load_content_by_source_row(csv_path)
|
||||
sconn = sqlite3.connect(structured_db)
|
||||
sconn.row_factory = sqlite3.Row
|
||||
try:
|
||||
jid = job_id if job_id is not None else _latest_job_id(sconn)
|
||||
ext_rows = sconn.execute(
|
||||
"""
|
||||
SELECT id, source_row, extraction_json
|
||||
FROM comment_extractions
|
||||
WHERE job_id = ?
|
||||
ORDER BY source_row
|
||||
""",
|
||||
(jid,),
|
||||
).fetchall()
|
||||
if not ext_rows:
|
||||
raise RuntimeError(f"job_id={jid} 无 comment_extractions 记录")
|
||||
finally:
|
||||
sconn.close()
|
||||
|
||||
tasks = _build_tasks(jid, ext_rows, content_map)
|
||||
if not tasks:
|
||||
raise RuntimeError("没有可向量化的条目（检查过滤规则与结构化结果）")
|
||||
|
||||
by_type: Dict[str, int] = {}
|
||||
for t in tasks:
|
||||
by_type[t.entity_type] = by_type.get(t.entity_type, 0) + 1
|
||||
logger.info(
|
||||
"job_id=%s：%s 条评论结构化记录 -> %s 条向量任务 %s",
|
||||
jid,
|
||||
len(ext_rows),
|
||||
len(tasks),
|
||||
by_type,
|
||||
)
|
||||
|
||||
blobs = _embed_batches(
|
||||
api_key,
|
||||
tasks,
|
||||
batch_size=batch_size,
|
||||
workers=workers,
|
||||
)
|
||||
|
||||
econn = sqlite3.connect(embed_db)
|
||||
try:
|
||||
_init_embed_schema(econn)
|
||||
if reset_db:
|
||||
econn.execute("DELETE FROM embedding_items")
|
||||
else:
|
||||
econn.execute("DELETE FROM embedding_items WHERE job_id = ?", (jid,))
|
||||
econn.commit()
|
||||
insert_sql = """
|
||||
INSERT INTO embedding_items (
|
||||
job_id, extraction_id, source_row, entity_type, entity_index,
|
||||
embed_text, audience, aspect, opinion, category, sentiment,
|
||||
content, dimensions, embedding
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
"""
|
||||
rows = [
|
||||
(
|
||||
task.job_id,
|
||||
task.extraction_id,
|
||||
task.source_row,
|
||||
task.entity_type,
|
||||
task.entity_index,
|
||||
task.embed_text,
|
||||
task.audience,
|
||||
task.aspect,
|
||||
task.opinion,
|
||||
task.category,
|
||||
task.sentiment,
|
||||
task.content,
|
||||
EMBEDDING_DIMENSIONS,
|
||||
blob,
|
||||
)
|
||||
for task, blob in zip(tasks, blobs)
|
||||
]
|
||||
econn.executemany(insert_sql, rows)
|
||||
econn.commit()
|
||||
total = econn.execute("SELECT COUNT(*) FROM embedding_items").fetchone()[0]
|
||||
finally:
|
||||
econn.close()
|
||||
|
||||
summary = {
|
||||
"job_id": jid,
|
||||
"structured_db": str(structured_db),
|
||||
"embed_db": str(embed_db),
|
||||
"csv": str(csv_path),
|
||||
"model": EMBEDDING_MODEL,
|
||||
"dimensions": EMBEDDING_DIMENSIONS,
|
||||
"extractions": len(ext_rows),
|
||||
"vectors": total,
|
||||
"by_entity_type": by_type,
|
||||
}
|
||||
logger.info("完成：%s", summary)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="VOC 结构化结果向量化")
|
||||
parser.add_argument("--job-id", type=int, default=None, help="默认最新 job")
|
||||
parser.add_argument("--csv", type=Path, default=DEFAULT_CSV)
|
||||
parser.add_argument("--structured-db", type=Path, default=STRUCTURED_DB)
|
||||
parser.add_argument("--embed-db", type=Path, default=EMBED_DB)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=EMBED_BATCH_SIZE,
|
||||
help="每批 API 请求条数，最大 10（DashScope 限制）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
default=EMBED_DEFAULT_WORKERS,
|
||||
help="并行批次数；设为 1 则串行",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
summary = run_embed(
|
||||
job_id=args.job_id,
|
||||
csv_path=args.csv,
|
||||
structured_db=args.structured_db,
|
||||
embed_db=args.embed_db,
|
||||
batch_size=args.batch_size,
|
||||
workers=args.workers,
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
59
结构化_Prompt.py
Normal file
59
结构化_Prompt.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""
|
||||
结构化提取 Prompt 入口（外置 prompts/ 目录，每次调用重读磁盘）。
|
||||
|
||||
对外 API 保持不变，供 结构化_server.py 动态加载。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import List, Tuple
|
||||
|
||||
from prompts.loader import (
|
||||
build_batch_extraction_system,
|
||||
build_batch_extraction_user,
|
||||
build_single_extraction_template,
|
||||
format_examples,
|
||||
)
|
||||
|
||||
|
||||
def build_batch_review_analysis_prompts(
|
||||
industry: str,
|
||||
product_name: str,
|
||||
keys: List[str],
|
||||
tagged_input: str,
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
生成批量评论分析用的 (system_prompt, user_prompt)。
|
||||
tagged_input 形如多行 "[C1] ...\\n[C2] ..."；模型须输出仅含这些键的 JSON 对象。
|
||||
"""
|
||||
keys_literal = ", ".join(json.dumps(k) for k in keys)
|
||||
n_keys = len(keys)
|
||||
system_prompt = build_batch_extraction_system(
|
||||
industry,
|
||||
product_name,
|
||||
n_keys=n_keys,
|
||||
keys_literal=keys_literal,
|
||||
)
|
||||
user_prompt = build_batch_extraction_user(
|
||||
n_keys=n_keys,
|
||||
keys_literal=keys_literal,
|
||||
tagged_input=tagged_input,
|
||||
)
|
||||
return system_prompt, user_prompt
|
||||
|
||||
|
||||
def generate_extraction_prompt_template(industry: str = "行业", product_name: str = "产品") -> str:
|
||||
"""
|
||||
根据行业和产品名，动态生成适合大模型结构化提取的 Prompt 模版。
|
||||
返回的字符串中包含 {review_content} 占位符，供后续批量处理时填入真实评论。
|
||||
"""
|
||||
return build_single_extraction_template(industry, product_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
general_prompt = generate_extraction_prompt_template(
|
||||
industry="线上教育及宠物用品",
|
||||
product_name="综合商品",
|
||||
)
|
||||
print("============== Prompt 预览 ==============\n")
|
||||
print(general_prompt)
|
||||
1082
结构化_server.py
Normal file
1082
结构化_server.py
Normal file
File diff suppressed because it is too large
Load diff
608
词频.py
Normal file
608
词频.py
Normal file
|
|
@ -0,0 +1,608 @@
|
|||
"""
|
||||
词频统计：从最新结构化任务读取产品与 CSV，两步连跑。
|
||||
|
||||
1. 随机 25 条 content → LLM 归纳「产品专有名词」与「Amazon/产品专属停用词」
|
||||
2. spaCy 全量 content 分词 + 词频 → output/word_freq.csv（专有名词按完整短语统计，不拆词）
|
||||
|
||||
用法::
|
||||
|
||||
python3 词频词云.py
|
||||
python3 词频词云.py --skip-llm # 复用 output/voc_terms.json，仅跑第 2 步
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Sequence, Set, Tuple
|
||||
|
||||
from openai import OpenAI
|
||||
from spacy.lang.en.stop_words import STOP_WORDS as EN_STOP_WORDS
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("voc_wordfreq")
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||||
OUTPUT_DIR = PROJECT_ROOT / "output"
|
||||
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
|
||||
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"
|
||||
|
||||
DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
MODEL_NAME = os.environ.get("DASHSCOPE_MODEL", "qwen3.6-flash").strip()
|
||||
|
||||
SAMPLE_SIZE = 25
|
||||
SAMPLE_SEED = 42
|
||||
|
||||
|
||||
def _resolve_api_key() -> str:
|
||||
v = os.environ.get("DASHSCOPE_API_KEY", "").strip()
|
||||
if v:
|
||||
return v
|
||||
fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip()
|
||||
if fp:
|
||||
p = Path(fp).expanduser()
|
||||
if p.is_file():
|
||||
return p.read_text(encoding="utf-8").strip().strip('"').strip("'")
|
||||
local = PROJECT_ROOT / ".dashscope_key"
|
||||
if local.is_file():
|
||||
return local.read_text(encoding="utf-8").strip().strip('"').strip("'")
|
||||
return ""
|
||||
|
||||
|
||||
def _strip_think(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
text = re.sub(
|
||||
r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
|
||||
)
|
||||
text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _latest_job(conn: sqlite3.Connection) -> Tuple[int, str, str, str]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id, industry, product_name, source_file
|
||||
FROM analysis_jobs
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise RuntimeError(f"{STRUCTURED_DB} 中无 analysis_jobs 记录，请先跑结构化")
|
||||
return int(row[0]), str(row[1]), str(row[2]), str(row[3])
|
||||
|
||||
|
||||
def _resolve_source_path(source_file: str) -> Path:
|
||||
p = Path(source_file)
|
||||
if p.is_file():
|
||||
return p.resolve()
|
||||
candidate = PROJECT_ROOT / source_file
|
||||
if candidate.is_file():
|
||||
return candidate.resolve()
|
||||
raise FileNotFoundError(f"找不到评论 CSV: {source_file}")
|
||||
|
||||
|
||||
def _load_contents(csv_path: Path) -> List[Tuple[int, str]]:
|
||||
rows: List[Tuple[int, str]] = []
|
||||
with csv_path.open(encoding="utf-8-sig", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
if not reader.fieldnames or "content" not in reader.fieldnames:
|
||||
raise ValueError(f"{csv_path} 缺少 content 列")
|
||||
for i, row in enumerate(reader, start=1):
|
||||
text = (row.get("content") or "").strip()
|
||||
if text:
|
||||
rows.append((i, text))
|
||||
if not rows:
|
||||
raise ValueError(f"{csv_path} 无有效 content")
|
||||
return rows
|
||||
|
||||
|
||||
def _sample_reviews(
|
||||
rows: Sequence[Tuple[int, str]], *, n: int, seed: int
|
||||
) -> List[Tuple[int, str]]:
|
||||
rng = random.Random(seed)
|
||||
k = min(n, len(rows))
|
||||
return rng.sample(list(rows), k)
|
||||
|
||||
|
||||
def _build_terms_prompt(
|
||||
industry: str, product_name: str, samples: Sequence[Tuple[int, str]]
|
||||
) -> Tuple[str, str]:
|
||||
blocks = []
|
||||
for row_id, text in samples:
|
||||
blocks.append(f"[R{row_id}]\n{text}")
|
||||
joined = "\n\n---\n\n".join(blocks)
|
||||
system = (
|
||||
"你是亚马逊美国站英文评论与电商文本分析专家。"
|
||||
"根据样本评论，列出两类词表，供后续英文分词与词频统计使用。"
|
||||
)
|
||||
user = f"""行业：{industry}
|
||||
产品：{product_name}
|
||||
|
||||
以下为该产品 {len(samples)} 条随机评论（仅正文 content）：
|
||||
|
||||
{joined}
|
||||
|
||||
请输出两部分（可用 Markdown 标题、编号列表或逗号分隔，不必是 JSON）：
|
||||
|
||||
## 产品专有名词（product_terms）
|
||||
列出该产品评论中可能出现的**全部**专有表达，尽量合理扩展，包括但不限于：
|
||||
成分/品类、宠物品种与病种、剂型包装、用法场景、Amazon 物流售后相关但**属于本产品语境**的词等。
|
||||
多词短语请保留完整形式（如 turkey tail mushroom powder）；后续词频**只统计完整短语**，不会拆成单词分别计数。
|
||||
**不要**把下面「专属停用词」里的词放进本列表。
|
||||
|
||||
## 专属停用词（stopwords）
|
||||
列出 Amazon 美国站电商通用词，以及与本产品强相关、词频高但**分析价值低**的词（如泛化评价词、过泛产品描述词等）。
|
||||
多词短语可保留。
|
||||
注意：不要把产品名「{product_name}」**整句**列为停用词；也不要把其中的**实词/品类成分**（如 turkey、tail、mushroom）列为停用词。
|
||||
产品名拆词里**无分析价值的虚词/功能词**（如 for、the、a、of、and）可以且应列入停用词。
|
||||
|
||||
每条一行或逗号分隔即可。"""
|
||||
return system, user
|
||||
|
||||
|
||||
def _call_llm(system: str, user: str, api_key: str) -> str:
|
||||
client = OpenAI(api_key=api_key, base_url=DASHSCOPE_BASE_URL)
|
||||
extra_body: Dict[str, Any] = {}
|
||||
if MODEL_NAME.lower().startswith(("qwen3.6", "qwen3.5", "qwen3")):
|
||||
extra_body["enable_thinking"] = False
|
||||
resp = client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
temperature=0.3,
|
||||
**({"extra_body": extra_body} if extra_body else {}),
|
||||
)
|
||||
msg = resp.choices[0].message
|
||||
text = msg.content or getattr(msg, "reasoning_content", None) or ""
|
||||
if not text.strip():
|
||||
raise RuntimeError("LLM 返回为空")
|
||||
return _strip_think(text)
|
||||
|
||||
|
||||
def _normalize_term(s: str) -> str:
|
||||
s = re.sub(r"\s+", " ", (s or "").strip().lower())
|
||||
return s.strip(".,;:\"'""''`-–—()[]{}")
|
||||
|
||||
|
||||
def _split_list_items(line: str) -> List[str]:
|
||||
line = re.sub(r"^[\s\d\.\)\-•\*]+", "", line.strip())
|
||||
if not line or line.startswith("#"):
|
||||
return []
|
||||
parts = re.split(r"[,，;；、|]", line)
|
||||
return [_normalize_term(p) for p in parts if _normalize_term(p)]
|
||||
|
||||
|
||||
def _try_parse_json_terms(text: str) -> Tuple[List[str], List[str]] | None:
|
||||
text = text.replace("```json", "").replace("```", "").strip()
|
||||
start, end = text.find("{"), text.rfind("}")
|
||||
if start == -1 or end <= start:
|
||||
return None
|
||||
try:
|
||||
obj = json.loads(text[start : end + 1])
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(obj, dict):
|
||||
return None
|
||||
|
||||
def pick(keys: Sequence[str]) -> List[str]:
|
||||
for k in keys:
|
||||
v = obj.get(k)
|
||||
if isinstance(v, list):
|
||||
return [_normalize_term(str(x)) for x in v if _normalize_term(str(x))]
|
||||
if isinstance(v, str):
|
||||
return _split_list_items(v)
|
||||
return []
|
||||
|
||||
terms = pick(("product_terms", "专有名词", "terms", "keywords"))
|
||||
stops = pick(("stopwords", "stop_words", "停用词", "custom_stopwords"))
|
||||
if terms or stops:
|
||||
return terms, stops
|
||||
return None
|
||||
|
||||
|
||||
def _parse_section_lines(text: str, mode: str) -> List[str]:
|
||||
"""mode: product | stop"""
|
||||
lines = text.splitlines()
|
||||
collecting = False
|
||||
items: List[str] = []
|
||||
product_hdr = re.compile(
|
||||
r"(专有名词|product[_\s-]*terms?|product\s+terms)", re.I
|
||||
)
|
||||
stop_hdr = re.compile(
|
||||
r"(专属停用|停用词|stop[_\s-]*words?|custom\s+stop)", re.I
|
||||
)
|
||||
other_hdr = re.compile(r"^#{1,3}\s+")
|
||||
|
||||
for line in lines:
|
||||
raw = line.strip()
|
||||
if not raw:
|
||||
if collecting and items:
|
||||
collecting = False
|
||||
continue
|
||||
if mode == "product" and product_hdr.search(raw):
|
||||
collecting = True
|
||||
inline = re.split(r"[:：]", raw, maxsplit=1)
|
||||
if len(inline) > 1:
|
||||
items.extend(_split_list_items(inline[1]))
|
||||
continue
|
||||
if mode == "stop" and stop_hdr.search(raw):
|
||||
collecting = True
|
||||
inline = re.split(r"[:：]", raw, maxsplit=1)
|
||||
if len(inline) > 1:
|
||||
items.extend(_split_list_items(inline[1]))
|
||||
continue
|
||||
if collecting and (other_hdr.match(raw) or (
|
||||
mode == "product" and stop_hdr.search(raw)
|
||||
) or (mode == "stop" and product_hdr.search(raw))):
|
||||
collecting = False
|
||||
continue
|
||||
if collecting:
|
||||
items.extend(_split_list_items(raw))
|
||||
return items
|
||||
|
||||
|
||||
def _parse_llm_terms(text: str) -> Tuple[List[str], List[str]]:
|
||||
parsed = _try_parse_json_terms(text)
|
||||
if parsed:
|
||||
return parsed
|
||||
product = _parse_section_lines(text, "product")
|
||||
stop = _parse_section_lines(text, "stop")
|
||||
if not product and not stop:
|
||||
# 兜底：按行提取非空短语
|
||||
for line in text.splitlines():
|
||||
items = _split_list_items(line)
|
||||
if items and len(items) <= 8:
|
||||
product.extend(items)
|
||||
return _dedupe_terms(product), _dedupe_terms(stop)
|
||||
|
||||
|
||||
def _dedupe_terms(items: Iterable[str]) -> List[str]:
|
||||
seen: Set[str] = set()
|
||||
out: List[str] = []
|
||||
for x in items:
|
||||
if x and x not in seen:
|
||||
seen.add(x)
|
||||
out.append(x)
|
||||
return out
|
||||
|
||||
|
||||
def _product_name_tokens(product_name: str) -> Set[str]:
|
||||
return {
|
||||
t
|
||||
for t in re.findall(r"[a-z0-9]+", product_name.lower())
|
||||
if t
|
||||
}
|
||||
|
||||
|
||||
# 产品标题拆词中可视为「不重要」、应参与停用的功能词（含 spaCy 英文停用词交集）
|
||||
_PRODUCT_NAME_FILLER = frozenset(
|
||||
{
|
||||
"a", "an", "the", "and", "or", "but", "for", "nor", "so", "yet",
|
||||
"at", "by", "in", "of", "on", "to", "up", "as", "is", "it", "be",
|
||||
"with", "from", "into", "via", "per", "vs", "vs.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _product_name_filler_tokens(product_name: str) -> Set[str]:
|
||||
"""产品名拆词中的虚词/功能词 → 应停用。"""
|
||||
tokens = _product_name_tokens(product_name)
|
||||
return {t for t in tokens if t in EN_STOP_WORDS or t in _PRODUCT_NAME_FILLER}
|
||||
|
||||
|
||||
def _product_name_core_tokens(product_name: str) -> Set[str]:
|
||||
"""产品名中有分析价值的实词 → 不停用。"""
|
||||
fillers = _product_name_filler_tokens(product_name)
|
||||
return _product_name_tokens(product_name) - fillers
|
||||
|
||||
|
||||
def _is_pure_number(s: str) -> bool:
|
||||
return bool(s) and s.isdigit()
|
||||
|
||||
|
||||
def _finalize_term_lists(
|
||||
product_terms: List[str],
|
||||
stopwords: List[str],
|
||||
product_name: str,
|
||||
) -> Tuple[List[str], Set[str]]:
|
||||
stop_set = set(EN_STOP_WORDS)
|
||||
stop_set.update(_dedupe_terms(stopwords))
|
||||
# 产品名虚词（for/the/a 等）纳入停用；实词成分保持可统计
|
||||
stop_set.update(_product_name_filler_tokens(product_name))
|
||||
for tok in _product_name_core_tokens(product_name):
|
||||
stop_set.discard(tok)
|
||||
pn_lower = _normalize_term(product_name)
|
||||
if pn_lower:
|
||||
stop_set.discard(pn_lower)
|
||||
|
||||
cleaned_terms: List[str] = []
|
||||
for t in _dedupe_terms(product_terms):
|
||||
if t in stop_set:
|
||||
continue
|
||||
cleaned_terms.append(t)
|
||||
# 长短语优先匹配
|
||||
cleaned_terms.sort(key=lambda x: (-len(x.split()), -len(x)))
|
||||
return cleaned_terms, stop_set
|
||||
|
||||
|
||||
def _load_spacy():
|
||||
import spacy
|
||||
|
||||
try:
|
||||
return spacy.load("en_core_web_sm", disable=["ner", "parser"])
|
||||
except OSError as e:
|
||||
raise RuntimeError(
|
||||
"未安装 spaCy 英文模型，请执行: python3 -m spacy download en_core_web_sm"
|
||||
) from e
|
||||
|
||||
|
||||
def _phrase_pattern(phrase: str) -> re.Pattern[str]:
|
||||
parts = [re.escape(p) for p in phrase.split()]
|
||||
body = r"\s+".join(parts)
|
||||
return re.compile(rf"(?<!\w){body}(?!\w)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _merge_spans(spans: List[Tuple[int, int]]) -> List[Tuple[int, int]]:
|
||||
if not spans:
|
||||
return []
|
||||
ordered = sorted(spans)
|
||||
merged: List[Tuple[int, int]] = [ordered[0]]
|
||||
for start, end in ordered[1:]:
|
||||
prev_s, prev_e = merged[-1]
|
||||
if start <= prev_e:
|
||||
merged[-1] = (prev_s, max(prev_e, end))
|
||||
else:
|
||||
merged.append((start, end))
|
||||
return merged
|
||||
|
||||
|
||||
def _overlaps_span(char_start: int, char_end: int, spans: Sequence[Tuple[int, int]]) -> bool:
|
||||
for s, e in spans:
|
||||
if char_start < e and char_end > s:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _apply_product_terms(
|
||||
lower: str,
|
||||
product_terms: List[str],
|
||||
counter: Counter[str],
|
||||
stop_set: Set[str],
|
||||
) -> List[Tuple[int, int]]:
|
||||
"""匹配专有名词短语：只计完整短语频次，并返回需屏蔽拆词统计的字符区间。"""
|
||||
spans: List[Tuple[int, int]] = []
|
||||
for phrase in product_terms:
|
||||
if not phrase or phrase in stop_set or _is_pure_number(phrase):
|
||||
continue
|
||||
pat = _phrase_pattern(phrase)
|
||||
hits = 0
|
||||
for m in pat.finditer(lower):
|
||||
spans.append((m.start(), m.end()))
|
||||
hits += 1
|
||||
if hits:
|
||||
counter[phrase] += hits
|
||||
return _merge_spans(spans)
|
||||
|
||||
|
||||
def _tokenize_doc(
|
||||
nlp, text: str, protected_spans: Sequence[Tuple[int, int]]
|
||||
) -> List[str]:
|
||||
doc = nlp(text)
|
||||
tokens: List[str] = []
|
||||
for tok in doc:
|
||||
if tok.is_space or tok.is_punct:
|
||||
continue
|
||||
char_start = tok.idx
|
||||
char_end = tok.idx + len(tok.text)
|
||||
if _overlaps_span(char_start, char_end, protected_spans):
|
||||
continue
|
||||
lemma = (tok.lemma_ or tok.text).lower().strip()
|
||||
if not lemma or _is_pure_number(lemma):
|
||||
continue
|
||||
if not re.search(r"[a-z]", lemma, re.I):
|
||||
continue
|
||||
tokens.append(lemma)
|
||||
return tokens
|
||||
|
||||
|
||||
_SINGLE_EN_WORD = re.compile(r"^[a-z]+$")
|
||||
|
||||
|
||||
def _should_lemma_normalize(word: str) -> bool:
|
||||
"""仅对单个英文词做词形还原；多词短语、连字符短语保持原样。"""
|
||||
w = word.strip().lower()
|
||||
if not w or " " in w or "-" in w or "'" in w:
|
||||
return False
|
||||
return bool(_SINGLE_EN_WORD.match(w))
|
||||
|
||||
|
||||
def _lemma_form(word: str, nlp) -> str:
|
||||
w = word.strip().lower()
|
||||
if not _should_lemma_normalize(w):
|
||||
return w
|
||||
doc = nlp(w)
|
||||
if not doc:
|
||||
return w
|
||||
tok = doc[0]
|
||||
if tok.is_space or tok.is_punct:
|
||||
return w
|
||||
lemma = (tok.lemma_ or tok.text).lower().strip()
|
||||
if not lemma or lemma == "-":
|
||||
return w
|
||||
return lemma
|
||||
|
||||
|
||||
def _merge_word_forms(counter: Counter[str], nlp) -> Counter[str]:
|
||||
"""写入 CSV 前合并单复数/时态等词形（如 dogs→dog, bought→buy）。"""
|
||||
merged: Counter[str] = Counter()
|
||||
for word, count in counter.items():
|
||||
merged[_lemma_form(word, nlp)] += count
|
||||
return merged
|
||||
|
||||
|
||||
def _build_word_freq(
|
||||
rows: Sequence[Tuple[int, str]],
|
||||
product_terms: List[str],
|
||||
stop_set: Set[str],
|
||||
) -> Counter[str]:
|
||||
nlp = _load_spacy()
|
||||
counter: Counter[str] = Counter()
|
||||
for _, text in rows:
|
||||
if not text.strip():
|
||||
continue
|
||||
lower = text.lower()
|
||||
protected = _apply_product_terms(lower, product_terms, counter, stop_set)
|
||||
for tok in _tokenize_doc(nlp, text, protected):
|
||||
if tok in stop_set:
|
||||
continue
|
||||
counter[tok] += 1
|
||||
counter = Counter({k: v for k, v in counter.items() if not _is_pure_number(k)})
|
||||
return _merge_word_forms(counter, nlp)
|
||||
|
||||
|
||||
def _save_word_freq(counter: Counter[str], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["word", "count"])
|
||||
for word, count in counter.most_common():
|
||||
w.writerow([word, count])
|
||||
|
||||
def _step1_extract_terms(
|
||||
*,
|
||||
job_id: int,
|
||||
industry: str,
|
||||
product_name: str,
|
||||
samples: List[Tuple[int, str]],
|
||||
api_key: str,
|
||||
) -> dict:
|
||||
system, user = _build_terms_prompt(industry, product_name, samples)
|
||||
logger.info("第 1 步：调用 %s 提取专有名词与停用词（样本 %s 条）", MODEL_NAME, len(samples))
|
||||
raw = _call_llm(system, user, api_key)
|
||||
product_raw, stop_raw = _parse_llm_terms(raw)
|
||||
product_terms, _stop_set = _finalize_term_lists(
|
||||
product_raw, stop_raw, product_name
|
||||
)
|
||||
logger.info(
|
||||
"解析得到专有名词 %s 条、专属停用词 %s 条",
|
||||
len(product_terms),
|
||||
len(stop_raw),
|
||||
)
|
||||
payload = {
|
||||
"job_id": job_id,
|
||||
"industry": industry,
|
||||
"product_name": product_name,
|
||||
"sample_size": len(samples),
|
||||
"sample_seed": SAMPLE_SEED,
|
||||
"sample_row_ids": [r for r, _ in samples],
|
||||
"model": MODEL_NAME,
|
||||
"product_terms": product_terms,
|
||||
"stopwords_custom": _dedupe_terms(stop_raw),
|
||||
"llm_raw": raw,
|
||||
}
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
TERMS_JSON.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
logger.info("已保存 %s", TERMS_JSON)
|
||||
return payload
|
||||
|
||||
|
||||
def _load_terms_json() -> dict:
|
||||
if not TERMS_JSON.is_file():
|
||||
raise FileNotFoundError(f"缺少 {TERMS_JSON}，请先运行完整流程或去掉 --skip-llm")
|
||||
return json.loads(TERMS_JSON.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def run(*, skip_llm: bool = False) -> dict:
|
||||
if not STRUCTURED_DB.is_file():
|
||||
raise FileNotFoundError(f"缺少 {STRUCTURED_DB}")
|
||||
|
||||
conn = sqlite3.connect(STRUCTURED_DB)
|
||||
try:
|
||||
job_id, industry, product_name, source_file = _latest_job(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
csv_path = _resolve_source_path(source_file)
|
||||
rows = _load_contents(csv_path)
|
||||
logger.info(
|
||||
"job_id=%s product=%r 评论 %s 条，来源 %s",
|
||||
job_id,
|
||||
product_name,
|
||||
len(rows),
|
||||
csv_path.name,
|
||||
)
|
||||
|
||||
if skip_llm:
|
||||
meta = _load_terms_json()
|
||||
product_terms, stop_set = _finalize_term_lists(
|
||||
list(meta.get("product_terms") or []),
|
||||
list(meta.get("stopwords_custom") or []),
|
||||
product_name,
|
||||
)
|
||||
logger.info("第 1 步跳过，复用 %s", TERMS_JSON)
|
||||
else:
|
||||
api_key = _resolve_api_key()
|
||||
if not api_key:
|
||||
raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key")
|
||||
samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED)
|
||||
meta = _step1_extract_terms(
|
||||
job_id=job_id,
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
samples=samples,
|
||||
api_key=api_key,
|
||||
)
|
||||
product_terms, stop_set = _finalize_term_lists(
|
||||
meta["product_terms"],
|
||||
meta.get("stopwords_custom") or [],
|
||||
product_name,
|
||||
)
|
||||
|
||||
logger.info("第 2 步：spaCy 全量分词与词频统计")
|
||||
counter = _build_word_freq(rows, product_terms, stop_set)
|
||||
_save_word_freq(counter, WORD_FREQ_CSV)
|
||||
logger.info("已写入 %s", WORD_FREQ_CSV)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"product_name": product_name,
|
||||
"terms_json": str(TERMS_JSON),
|
||||
"word_freq_csv": str(WORD_FREQ_CSV),
|
||||
"unique_words": len(counter),
|
||||
"total_tokens": sum(counter.values()),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="VOC 词频统计")
|
||||
parser.add_argument(
|
||||
"--skip-llm",
|
||||
action="store_true",
|
||||
help="跳过第 1 步，使用 output/voc_terms.json",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
result = run(skip_llm=args.skip_llm)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
182
词频_jieba.py
Normal file
182
词频_jieba.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
"""
|
||||
词频统计（jieba 分词版）：流程与 词频.py 相同，第 2 步改用 jieba 分词。
|
||||
|
||||
1. 随机样本 → LLM 归纳专有名词与停用词（与 spaCy 版共用 voc_terms.json）
|
||||
2. jieba 全量 content 分词 + 词频 → output/word_freq.csv
|
||||
|
||||
用法::
|
||||
|
||||
python3 词频_jieba.py
|
||||
python3 词频_jieba.py --skip-llm
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import List, Sequence, Set, Tuple
|
||||
|
||||
from 词频 import (
|
||||
OUTPUT_DIR,
|
||||
SAMPLE_SEED,
|
||||
SAMPLE_SIZE,
|
||||
STRUCTURED_DB,
|
||||
TERMS_JSON,
|
||||
WORD_FREQ_CSV,
|
||||
_apply_product_terms,
|
||||
_finalize_term_lists,
|
||||
_is_pure_number,
|
||||
_latest_job,
|
||||
_load_contents,
|
||||
_load_spacy,
|
||||
_load_terms_json,
|
||||
_merge_word_forms,
|
||||
_overlaps_span,
|
||||
_resolve_api_key,
|
||||
_resolve_source_path,
|
||||
_sample_reviews,
|
||||
_save_word_freq,
|
||||
_step1_extract_terms,
|
||||
)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
logger = logging.getLogger("voc_wordfreq_jieba")
|
||||
|
||||
# 英文词或中文词均保留
|
||||
_TOKEN_OK = re.compile(r"[a-z\u4e00-\u9fff]", re.I)
|
||||
|
||||
|
||||
def _prepare_jieba(product_terms: Sequence[str]) -> None:
|
||||
import jieba
|
||||
|
||||
for term in product_terms:
|
||||
t = (term or "").strip()
|
||||
if t:
|
||||
jieba.add_word(t)
|
||||
|
||||
|
||||
def _tokenize_jieba(
|
||||
text: str,
|
||||
protected_spans: Sequence[Tuple[int, int]],
|
||||
stop_set: Set[str],
|
||||
) -> List[str]:
|
||||
import jieba
|
||||
|
||||
tokens: List[str] = []
|
||||
for word, start, end in jieba.tokenize(text):
|
||||
if _overlaps_span(start, end, protected_spans):
|
||||
continue
|
||||
w = word.strip().lower()
|
||||
if not w or _is_pure_number(w) or w in stop_set:
|
||||
continue
|
||||
if not _TOKEN_OK.search(w):
|
||||
continue
|
||||
tokens.append(w)
|
||||
return tokens
|
||||
|
||||
|
||||
def _build_word_freq(
|
||||
rows: Sequence[Tuple[int, str]],
|
||||
product_terms: List[str],
|
||||
stop_set: Set[str],
|
||||
) -> Counter[str]:
|
||||
_prepare_jieba(product_terms)
|
||||
counter: Counter[str] = Counter()
|
||||
for _, text in rows:
|
||||
if not text.strip():
|
||||
continue
|
||||
lower = text.lower()
|
||||
protected = _apply_product_terms(lower, product_terms, counter, stop_set)
|
||||
for tok in _tokenize_jieba(text, protected, stop_set):
|
||||
counter[tok] += 1
|
||||
counter = Counter({k: v for k, v in counter.items() if not _is_pure_number(k)})
|
||||
nlp = _load_spacy()
|
||||
return _merge_word_forms(counter, nlp)
|
||||
|
||||
|
||||
def run(*, skip_llm: bool = False) -> dict:
|
||||
if not STRUCTURED_DB.is_file():
|
||||
raise FileNotFoundError(f"缺少 {STRUCTURED_DB}")
|
||||
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(STRUCTURED_DB)
|
||||
try:
|
||||
job_id, industry, product_name, source_file = _latest_job(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
csv_path = _resolve_source_path(source_file)
|
||||
rows = _load_contents(csv_path)
|
||||
logger.info(
|
||||
"job_id=%s product=%r 评论 %s 条，来源 %s（jieba 分词）",
|
||||
job_id,
|
||||
product_name,
|
||||
len(rows),
|
||||
csv_path.name,
|
||||
)
|
||||
|
||||
if skip_llm:
|
||||
meta = _load_terms_json()
|
||||
product_terms, stop_set = _finalize_term_lists(
|
||||
list(meta.get("product_terms") or []),
|
||||
list(meta.get("stopwords_custom") or []),
|
||||
product_name,
|
||||
)
|
||||
logger.info("第 1 步跳过，复用 %s", TERMS_JSON)
|
||||
else:
|
||||
api_key = _resolve_api_key()
|
||||
if not api_key:
|
||||
raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key")
|
||||
samples = _sample_reviews(rows, n=SAMPLE_SIZE, seed=SAMPLE_SEED)
|
||||
meta = _step1_extract_terms(
|
||||
job_id=job_id,
|
||||
industry=industry,
|
||||
product_name=product_name,
|
||||
samples=samples,
|
||||
api_key=api_key,
|
||||
)
|
||||
product_terms, stop_set = _finalize_term_lists(
|
||||
meta["product_terms"],
|
||||
meta.get("stopwords_custom") or [],
|
||||
product_name,
|
||||
)
|
||||
|
||||
logger.info("第 2 步：jieba 全量分词与词频统计")
|
||||
counter = _build_word_freq(rows, product_terms, stop_set)
|
||||
_save_word_freq(counter, WORD_FREQ_CSV)
|
||||
logger.info("已写入 %s", WORD_FREQ_CSV)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"product_name": product_name,
|
||||
"tokenizer": "jieba",
|
||||
"terms_json": str(TERMS_JSON),
|
||||
"word_freq_csv": str(WORD_FREQ_CSV),
|
||||
"unique_words": len(counter),
|
||||
"total_tokens": sum(counter.values()),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="VOC 词频统计（jieba 分词）")
|
||||
parser.add_argument(
|
||||
"--skip-llm",
|
||||
action="store_true",
|
||||
help="跳过第 1 步，使用 output/voc_terms.json",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
result = run(skip_llm=args.skip_llm)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue