91e6c47fc0
统一 voc_llm 密钥解析与默认模型；向量化改为本地 mlx 模型；更新 README、gitignore 与流水线文档。 Co-authored-by: Cursor <cursoragent@cursor.com>
362 lines
12 KiB
Python
362 lines
12 KiB
Python
"""
|
||
VOC 全流程（jieba 分词版）：与 main_voc分析.py 相同，步骤 6 词频改用 jieba。
|
||
|
||
用法::
|
||
|
||
./310py/bin/python main_voc分析_jieba.py --input-dir reviews_export --industry "Pet supplements" --product "Turkey tail mushroom for dogs"
|
||
|
||
./310py/bin/python main_voc分析_jieba.py --from-step 6 # 仅重跑 jieba 词频（可省略 --industry/--product）
|
||
./310py/bin/python main_voc分析_jieba.py --only-step 7 # 仅生成报告
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import logging
|
||
import sqlite3
|
||
import sys
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
|
||
from content清洗 import process_reviews, save_cleaned_reviews
|
||
from 合并评论数据 import merge_csv_directory
|
||
from 向量化 import EMBED_DEFAULT_WORKERS, run_embed
|
||
from 结构化_server import STRUCT_DEFAULT_WORKERS, run_analysis
|
||
from 聚类 import run_clustering
|
||
from voc_report import DEFAULT_CLUSTER_MIN_REVIEW_RATIO, generate_report
|
||
from 词频_jieba import run as run_wordfreq
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||
stream=sys.stderr,
|
||
)
|
||
logger = logging.getLogger("voc_analysis_jieba")
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||
OUTPUT_DIR = PROJECT_ROOT / "output"
|
||
REPORT_HTML = OUTPUT_DIR / "voc_report.html"
|
||
|
||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||
EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite"
|
||
CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite"
|
||
TERMS_JSON = OUTPUT_DIR / "voc_terms.json"
|
||
WORD_FREQ_CSV = OUTPUT_DIR / "word_freq.csv"
|
||
|
||
DEFAULT_MERGED = PROJECT_ROOT / "merged_reviews.csv"
|
||
DEFAULT_CLEANED = PROJECT_ROOT / "merged_reviews_cleaned.csv"
|
||
|
||
|
||
def _latest_job_meta(structured_db: Path) -> tuple[int, str, str]:
|
||
conn = sqlite3.connect(structured_db)
|
||
try:
|
||
row = conn.execute(
|
||
"""
|
||
SELECT id, industry, product_name
|
||
FROM analysis_jobs ORDER BY id DESC LIMIT 1
|
||
"""
|
||
).fetchone()
|
||
if not row:
|
||
raise RuntimeError("无结构化任务，请先执行步骤 3")
|
||
return int(row[0]), str(row[1]), str(row[2])
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _should_run(step: int, from_step: int, only_step: int | None) -> bool:
|
||
if only_step is not None:
|
||
return step == only_step
|
||
return step >= from_step
|
||
|
||
|
||
def _clean_intermediates(
|
||
*,
|
||
merged: Path,
|
||
cleaned: Path,
|
||
structured: Path,
|
||
embed: Path,
|
||
cluster: Path,
|
||
terms: Path,
|
||
word_freq: Path,
|
||
) -> None:
|
||
for p in (merged, cleaned, structured, embed, cluster, terms, word_freq):
|
||
if p.is_file():
|
||
p.unlink()
|
||
logger.info("已删除中间文件 %s", p)
|
||
|
||
|
||
def run_voc_analysis(
|
||
*,
|
||
input_dir: Path,
|
||
industry: str,
|
||
product_name: str,
|
||
merged_csv: Path,
|
||
cleaned_csv: Path,
|
||
from_step: int = 1,
|
||
only_step: int | None = None,
|
||
clean_intermediates: bool = False,
|
||
skip_wordfreq_llm: bool = False,
|
||
min_cluster_review_ratio: float | None = None,
|
||
save_llm_raw: bool = False,
|
||
struct_workers: int | None = None,
|
||
embed_workers: int | None = None,
|
||
) -> dict:
|
||
result: dict = {"report_html": str(REPORT_HTML), "tokenizer": "jieba"}
|
||
|
||
if _should_run(1, from_step, only_step):
|
||
logger.info("步骤 1/7：合并 CSV")
|
||
merge_csv_directory(input_dir, merged_csv)
|
||
result["merged_csv"] = str(merged_csv)
|
||
|
||
if _should_run(2, from_step, only_step):
|
||
logger.info("步骤 2/7：清洗评论")
|
||
if not merged_csv.is_file():
|
||
raise FileNotFoundError(f"缺少 {merged_csv}，请先执行合并")
|
||
df = process_reviews(merged_csv)
|
||
save_cleaned_reviews(df, cleaned_csv)
|
||
result["cleaned_csv"] = str(cleaned_csv)
|
||
|
||
if _should_run(3, from_step, only_step):
|
||
logger.info("步骤 3/7：结构化分析")
|
||
if not cleaned_csv.is_file():
|
||
raise FileNotFoundError(f"缺少 {cleaned_csv}，请先执行清洗")
|
||
ar = run_analysis(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
file_path=str(cleaned_csv),
|
||
workers=struct_workers,
|
||
)
|
||
result["structured_job_id"] = ar.get("job_id")
|
||
result["structured_db"] = str(STRUCTURED_DB)
|
||
|
||
job_id: int | None = None
|
||
ind, prod = industry, product_name
|
||
if STRUCTURED_DB.is_file():
|
||
try:
|
||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||
except RuntimeError:
|
||
pass
|
||
|
||
if _should_run(4, from_step, only_step):
|
||
logger.info("步骤 4/7：向量化")
|
||
er = run_embed(
|
||
job_id=job_id,
|
||
csv_path=cleaned_csv,
|
||
structured_db=STRUCTURED_DB,
|
||
embed_db=EMBED_DB,
|
||
workers=embed_workers,
|
||
)
|
||
result["embed"] = er
|
||
job_id = int(er.get("job_id", job_id or 0))
|
||
|
||
if job_id is None and STRUCTURED_DB.is_file():
|
||
job_id, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||
|
||
run_cluster = _should_run(5, from_step, only_step)
|
||
run_wf = _should_run(6, from_step, only_step)
|
||
|
||
if run_cluster and run_wf:
|
||
logger.info("步骤 5–6/7：聚类与词频（jieba，并行）")
|
||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
f_cluster = pool.submit(
|
||
run_clustering,
|
||
job_id=job_id,
|
||
structured_db=STRUCTURED_DB,
|
||
embed_db=EMBED_DB,
|
||
cluster_db=CLUSTER_DB,
|
||
)
|
||
f_wf = pool.submit(run_wordfreq, skip_llm=skip_wordfreq_llm)
|
||
result["clustering"] = f_cluster.result()
|
||
result["wordfreq"] = f_wf.result()
|
||
else:
|
||
if run_cluster:
|
||
logger.info("步骤 5/7：聚类")
|
||
result["clustering"] = run_clustering(
|
||
job_id=job_id,
|
||
structured_db=STRUCTURED_DB,
|
||
embed_db=EMBED_DB,
|
||
cluster_db=CLUSTER_DB,
|
||
)
|
||
if run_wf:
|
||
logger.info("步骤 6/7：词频（jieba）")
|
||
result["wordfreq"] = run_wordfreq(skip_llm=skip_wordfreq_llm)
|
||
|
||
if _should_run(7, from_step, only_step):
|
||
logger.info("步骤 7/7：生成分析报告")
|
||
if not CLUSTER_DB.is_file():
|
||
raise FileNotFoundError(f"缺少 {CLUSTER_DB}")
|
||
if not WORD_FREQ_CSV.is_file():
|
||
raise FileNotFoundError(f"缺少 {WORD_FREQ_CSV}")
|
||
if not cleaned_csv.is_file():
|
||
raise FileNotFoundError(f"缺少 {cleaned_csv}")
|
||
if STRUCTURED_DB.is_file():
|
||
_, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||
result["report"] = generate_report(
|
||
product_name=prod,
|
||
industry=ind,
|
||
cleaned_csv=cleaned_csv,
|
||
cluster_db=CLUSTER_DB,
|
||
embed_db=EMBED_DB,
|
||
word_freq_csv=WORD_FREQ_CSV,
|
||
output_html=REPORT_HTML,
|
||
structured_db=STRUCTURED_DB,
|
||
min_cluster_review_ratio=min_cluster_review_ratio,
|
||
save_llm_raw=save_llm_raw,
|
||
)
|
||
|
||
if clean_intermediates and _should_run(7, from_step, only_step):
|
||
_clean_intermediates(
|
||
merged=merged_csv,
|
||
cleaned=cleaned_csv,
|
||
structured=STRUCTURED_DB,
|
||
embed=EMBED_DB,
|
||
cluster=CLUSTER_DB,
|
||
terms=TERMS_JSON,
|
||
word_freq=WORD_FREQ_CSV,
|
||
)
|
||
|
||
return result
|
||
|
||
|
||
run_pipeline = run_voc_analysis
|
||
|
||
|
||
def _resolve_industry_product(
|
||
*,
|
||
industry: str | None,
|
||
product_name: str | None,
|
||
from_step: int,
|
||
only_step: int | None,
|
||
) -> tuple[str, str]:
|
||
"""步骤 4 起可从 structured 库自动读取；步骤 1–3 须 CLI 传入。"""
|
||
if industry and product_name:
|
||
return industry, product_name
|
||
start_step = only_step if only_step is not None else from_step
|
||
if start_step >= 4:
|
||
if not STRUCTURED_DB.is_file():
|
||
raise SystemExit(
|
||
f"缺少 {STRUCTURED_DB.name}，无法自动读取 --industry / --product"
|
||
)
|
||
_, ind, prod = _latest_job_meta(STRUCTURED_DB)
|
||
logger.info("未指定 industry/product，已从 structured 库读取: %r / %r", ind, prod)
|
||
return ind, prod
|
||
raise SystemExit(
|
||
"步骤 1–3 需要 --industry 与 --product；从步骤 4 起可省略（自动读 voc_structured.sqlite）"
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="VOC 分析全流程（jieba 分词）")
|
||
parser.add_argument(
|
||
"--input-dir",
|
||
type=Path,
|
||
default=None,
|
||
help="原始 VOC CSV 目录（步骤 1 合并输入；续跑后续步骤可不传）",
|
||
)
|
||
parser.add_argument("--industry", default=None, help="行业（步骤 4 起可省略，自动读库）")
|
||
parser.add_argument("--product", default=None, help="产品名（步骤 4 起可省略，自动读库）")
|
||
parser.add_argument(
|
||
"--merged-csv",
|
||
type=Path,
|
||
default=DEFAULT_MERGED,
|
||
help=f"合并输出（默认 {DEFAULT_MERGED.name}）",
|
||
)
|
||
parser.add_argument(
|
||
"--cleaned-csv",
|
||
type=Path,
|
||
default=DEFAULT_CLEANED,
|
||
help=f"清洗输出（默认 {DEFAULT_CLEANED.name}）",
|
||
)
|
||
parser.add_argument(
|
||
"--from-step",
|
||
type=int,
|
||
default=1,
|
||
choices=range(1, 8),
|
||
metavar="N",
|
||
help="从第 N 步开始执行（1–7，默认 1）",
|
||
)
|
||
parser.add_argument(
|
||
"--only-step",
|
||
type=int,
|
||
default=None,
|
||
choices=range(1, 8),
|
||
metavar="N",
|
||
help="仅执行第 N 步（需已有前置产物）",
|
||
)
|
||
parser.add_argument(
|
||
"--clean-intermediates",
|
||
action="store_true",
|
||
help="报告生成成功后删除中间 sqlite/csv（仅保留 voc_report.html）",
|
||
)
|
||
parser.add_argument(
|
||
"--skip-wordfreq-llm",
|
||
action="store_true",
|
||
help="词频步骤复用已有 voc_terms.json",
|
||
)
|
||
parser.add_argument(
|
||
"--filter-small-clusters",
|
||
action="store_true",
|
||
help=(
|
||
"报告仅纳入本 stage 内去重评论占比 ≥ "
|
||
f"{DEFAULT_CLUSTER_MIN_REVIEW_RATIO * 100:.0f}%% 的簇；默认不筛选、全部簇参与"
|
||
),
|
||
)
|
||
parser.add_argument(
|
||
"--save-llm-raw",
|
||
action="store_true",
|
||
help="步骤 7 将报告 LLM 完整原文写入 output/report_llm_raw.txt（默认不保存）",
|
||
)
|
||
parser.add_argument(
|
||
"--struct-workers",
|
||
type=int,
|
||
default=None,
|
||
metavar="N",
|
||
help=f"步骤 3 结构化批间并行数（默认 {STRUCT_DEFAULT_WORKERS}）",
|
||
)
|
||
parser.add_argument(
|
||
"--embed-workers",
|
||
type=int,
|
||
default=None,
|
||
metavar="N",
|
||
help=f"步骤 4 向量化批间并行数（默认 {EMBED_DEFAULT_WORKERS}）",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
need_input = args.only_step in (None, 1) and args.from_step <= 1
|
||
if need_input:
|
||
if not args.input_dir:
|
||
raise SystemExit("步骤 1 需要 --input-dir")
|
||
if not args.input_dir.is_dir():
|
||
raise SystemExit(f"输入目录不存在: {args.input_dir}")
|
||
input_dir = (args.input_dir or DEFAULT_MERGED.parent).resolve()
|
||
industry, product_name = _resolve_industry_product(
|
||
industry=args.industry,
|
||
product_name=args.product,
|
||
from_step=args.from_step,
|
||
only_step=args.only_step,
|
||
)
|
||
|
||
out = run_voc_analysis(
|
||
input_dir=input_dir,
|
||
industry=industry,
|
||
product_name=product_name,
|
||
merged_csv=args.merged_csv.resolve(),
|
||
cleaned_csv=args.cleaned_csv.resolve(),
|
||
from_step=args.from_step,
|
||
only_step=args.only_step,
|
||
clean_intermediates=args.clean_intermediates,
|
||
skip_wordfreq_llm=args.skip_wordfreq_llm,
|
||
min_cluster_review_ratio=(
|
||
DEFAULT_CLUSTER_MIN_REVIEW_RATIO
|
||
if args.filter_small_clusters
|
||
else None
|
||
),
|
||
save_llm_raw=args.save_llm_raw,
|
||
struct_workers=args.struct_workers,
|
||
embed_workers=args.embed_workers,
|
||
)
|
||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|