c91e7f8c20
移除结构化 audience 字段，强化 voc_业务_2 源评论归因匹配与 Persona 引用展示，更新 README 与流水线默认清理 SQLite。 Co-authored-by: Cursor <cursoragent@cursor.com>
2784 lines
100 KiB
Python
2784 lines
100 KiB
Python
"""
|
||
基于聚类结果与词频 CSV 生成 output/voc_report.html（词云 + 词频 + AI 报告 + 各簇表述）。
|
||
|
||
用法::
|
||
./310py/bin/python main_voc分析.py
|
||
./310py/bin/python main_voc分析.py --only-step 7 --industry "..." --product "..."
|
||
# 仅纳入本 stage 内评论占比 ≥10% 的簇：
|
||
./310py/bin/python main_voc分析.py --only-step 7 --filter-small-clusters ...
|
||
# 调试：保存报告 LLM 完整原文到 output/report_llm_raw.txt
|
||
./310py/bin/python main_voc分析.py --only-step 7 --save-llm-raw --industry "..." --product "..."
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import html
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Sequence, Set, Tuple
|
||
|
||
from prompts.loader import (
|
||
build_category_analysis_prompts,
|
||
build_report_analysis_requirements,
|
||
build_report_correction_message,
|
||
build_report_json_markers,
|
||
build_report_output_format,
|
||
build_report_system,
|
||
build_word_assign_prompts,
|
||
sync_voc_report_constants,
|
||
)
|
||
from voc_llm import CHAT_MODEL, chat_extra_body, create_chat_client, require_chat_api_key
|
||
|
||
logger = logging.getLogger("voc_report")
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parent
|
||
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
|
||
STRUCTURED_APPENDIX_SAMPLE_N = 15
|
||
MODEL_NAME = CHAT_MODEL
|
||
# 步骤 7 主报告（HTML + JSON 标记块）；其它报告内 LLM 仍用 MODEL_NAME
|
||
REPORT_MODEL = os.environ.get("DEEPSEEK_REPORT_MODEL", "deepseek-v4-pro").strip()
|
||
REPORT_REASONING_EFFORT = os.environ.get(
|
||
"DEEPSEEK_REPORT_REASONING_EFFORT", "max"
|
||
).strip() or "max"
|
||
|
||
WORDCLOUD_TOP_N = 180
|
||
WORD_FREQ_TABLE_N = 180 # 页面词频表、排名展示上限
|
||
WORD_CATEGORY_CLASSIFY_N = 180 # LLM 词频分类词表范围（≥ 词表展示数）
|
||
WORD_FREQ_PAGE_SIZE = 30
|
||
WORD_FREQ_PAGES = 6
|
||
# 词云字号映射：0.5 次幂缓解「超高频词过大、长尾过小」
|
||
WORDCLOUD_SIZE_POWER = 0.5
|
||
WORDCLOUD_SIZE_MIN = 14
|
||
WORDCLOUD_SIZE_MAX = 96
|
||
MAX_PHRASES_PER_CLUSTER = 15
|
||
# 启用簇筛选时的默认阈值（本 stage 内去重评论占比）
|
||
DEFAULT_CLUSTER_MIN_REVIEW_RATIO = 0.10
|
||
OUTLIER_LABEL_ZH = "未归类"
|
||
|
||
_STAGE_3B_TITLES: Dict[str, str] = {
|
||
"3b_aspect_opinion_positive": "全量正面产品反馈",
|
||
"3b_aspect_opinion_negative": "全量负面产品反馈",
|
||
"3b_aspect_opinion_neutral": "全量产品客观描述",
|
||
}
|
||
_STAGE_2B_BLOCK_TITLE: Dict[str, str] = {
|
||
"positive": "正面产品反馈",
|
||
"negative": "负面产品反馈",
|
||
"neutral": "产品客观描述",
|
||
}
|
||
# 词频分类展示名迁移（兼容旧 LLM 输出）
|
||
_WORD_CATEGORY_DISPLAY_ALIASES: Dict[str, str] = {
|
||
"痛点/场景": "需求/场景",
|
||
}
|
||
_STAGE_SORT_KEY: Dict[str, int] = {
|
||
"3a_pain_global": 0,
|
||
"3b_aspect_opinion_positive": 1,
|
||
"3b_aspect_opinion_negative": 2,
|
||
"3b_aspect_opinion_neutral": 3,
|
||
}
|
||
# 与 聚类.py 流程一致的 stage 前缀（用于模版与映射说明）
|
||
STAGE_1_AUDIENCE = "1_audience"
|
||
STAGE_2A_PAIN_PREFIX = "2a_pain_audience_c"
|
||
STAGE_2B_AO_PREFIX = "2b_aspect_opinion_"
|
||
STAGE_3A_PAIN = "3a_pain_global"
|
||
STAGE_3B_NEGATIVE = "3b_aspect_opinion_negative"
|
||
STAGE_3B_AO_PREFIX = "3b_aspect_opinion_"
|
||
# 底部「聚类效果验证」固定展示的 stage（写死，不含 -1 离群簇）
|
||
PHRASE_APPENDIX_STAGES: Tuple[str, ...] = (STAGE_3A_PAIN, STAGE_3B_NEGATIVE)
|
||
STEP2_TOP2_META_STAGE = "step2_filter"
|
||
_AUDIENCE_RANK_ZH: Tuple[str, ...] = ("第一", "第二")
|
||
|
||
WORD_CATEGORIES: Tuple[str, ...] = (
|
||
"成分/原料",
|
||
"剂型",
|
||
"受众/使用对象",
|
||
"功效/功能",
|
||
"需求/场景",
|
||
"品质/体验",
|
||
"价格/价值",
|
||
"物流/包装",
|
||
)
|
||
|
||
# 词云按分类配色（与 WORD_CATEGORIES 顺序一一对应）
|
||
CATEGORY_COLORS: Dict[str, str] = {
|
||
"成分/原料": "#dc2626",
|
||
"剂型": "#db2777",
|
||
"受众/使用对象": "#2563eb",
|
||
"功效/功能": "#16a34a",
|
||
"需求/场景": "#ea580c",
|
||
"品质/体验": "#7c3aed",
|
||
"价格/价值": "#ca8a04",
|
||
"物流/包装": "#0891b2",
|
||
}
|
||
WORDCLOUD_UNCATEGORIZED_COLOR = "#9ca3af"
|
||
|
||
TRANSLATE_CHARS_PER_REQUEST = 200_000
|
||
TRANSLATE_MAX_OUTPUT_TOKENS = 200000
|
||
REPORT_MAX_OUTPUT_TOKENS = 200000
|
||
TRANSLATE_LLM_TIMEOUT_SEC = 600.0
|
||
REPORT_LLM_TIMEOUT_SEC = 720.0
|
||
REPORT_PARSE_MAX_RETRIES = 2
|
||
# JSON 段放前、HTML 放后：输出被 max_tokens 截断时优先保留可解析的 JSON
|
||
REPORT_MARKERS: Tuple[str, ...] = (
|
||
"===WORD_ZH_JSON===",
|
||
"===WORD_CATEGORY_JSON===",
|
||
"===CLUSTER_NAMES_JSON===",
|
||
"===REPORT_HTML===",
|
||
)
|
||
WORD_CATEGORY_ASSIGN_BATCH_SIZE = 40
|
||
WORD_CATEGORY_ASSIGN_MAX_TOKENS = 200_000
|
||
WORD_CATEGORY_ANALYSIS_MAX_WORDS = 25 # 生成 analysis 时每类最多展示的词条数
|
||
WORD_CATEGORY_ANALYSIS_MAX_TOKENS = 200_000
|
||
|
||
|
||
@dataclass
|
||
class ClusterBundle:
|
||
section: str
|
||
stage: str
|
||
stage_title_zh: str
|
||
cluster_label: int
|
||
cluster_title_zh: str
|
||
anchor_id: str
|
||
review_count: int
|
||
phrase_count: int
|
||
ratio: float # 占本 stage 去重评论比（小簇过滤用）
|
||
ratio_global: float # 短语数 / 全部评论 a（报告洞察用）
|
||
embed_texts: List[str] = field(default_factory=list)
|
||
embed_texts_zh: List[str] = field(default_factory=list)
|
||
|
||
|
||
def _strip_think(text: str) -> str:
|
||
if not text:
|
||
return text
|
||
text = re.sub(
|
||
r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
|
||
)
|
||
text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
|
||
return text.strip()
|
||
|
||
|
||
def _call_llm_messages(
|
||
messages: Sequence[Dict[str, str]],
|
||
api_key: str,
|
||
*,
|
||
model: str | None = None,
|
||
temperature: float = 0.3,
|
||
max_tokens: int = REPORT_MAX_OUTPUT_TOKENS,
|
||
timeout: float = 300.0,
|
||
reasoning_effort: str | None = None,
|
||
extra_body: Dict[str, Any] | None = None,
|
||
report_thinking: bool = False,
|
||
) -> str:
|
||
_ = api_key
|
||
client = create_chat_client(timeout=timeout)
|
||
use_model = model or MODEL_NAME
|
||
create_kw: Dict[str, Any] = {
|
||
"model": use_model,
|
||
"messages": list(messages),
|
||
"max_tokens": max_tokens,
|
||
}
|
||
if report_thinking:
|
||
create_kw["reasoning_effort"] = reasoning_effort or REPORT_REASONING_EFFORT
|
||
create_kw["extra_body"] = extra_body or {"thinking": {"type": "enabled"}}
|
||
else:
|
||
create_kw["temperature"] = temperature
|
||
eb = extra_body if extra_body is not None else chat_extra_body(use_model)
|
||
if eb:
|
||
create_kw["extra_body"] = eb
|
||
|
||
resp = client.chat.completions.create(**create_kw)
|
||
msg = resp.choices[0].message
|
||
if report_thinking:
|
||
text = msg.content or ""
|
||
if not text.strip():
|
||
fr = getattr(resp.choices[0], "finish_reason", None)
|
||
raise RuntimeError(
|
||
f"报告 LLM content 为空（model={use_model}，finish_reason={fr!r}）；"
|
||
"思考模式下请检查 max_tokens 是否截断"
|
||
)
|
||
else:
|
||
text = msg.content or getattr(msg, "reasoning_content", None) or ""
|
||
if not text.strip():
|
||
raise RuntimeError("LLM 返回为空")
|
||
return _strip_think(text)
|
||
|
||
|
||
def _call_report_llm_messages(
|
||
messages: Sequence[Dict[str, str]],
|
||
api_key: str,
|
||
*,
|
||
max_tokens: int = REPORT_MAX_OUTPUT_TOKENS,
|
||
timeout: float = REPORT_LLM_TIMEOUT_SEC,
|
||
) -> str:
|
||
"""主分析报告：deepseek-v4-pro + 思考模式 max（仅此处启用）。"""
|
||
return _call_llm_messages(
|
||
messages,
|
||
api_key,
|
||
model=REPORT_MODEL,
|
||
max_tokens=max_tokens,
|
||
timeout=timeout,
|
||
report_thinking=True,
|
||
)
|
||
|
||
|
||
def _call_llm(
|
||
system: str,
|
||
user: str,
|
||
api_key: str,
|
||
*,
|
||
temperature: float = 0.3,
|
||
max_tokens: int = REPORT_MAX_OUTPUT_TOKENS,
|
||
timeout: float = 300.0,
|
||
) -> str:
|
||
return _call_llm_messages(
|
||
[
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
api_key,
|
||
temperature=temperature,
|
||
max_tokens=max_tokens,
|
||
timeout=timeout,
|
||
)
|
||
|
||
|
||
def _load_review_count(csv_path: Path) -> int:
|
||
n = 0
|
||
with csv_path.open(encoding="utf-8-sig", newline="") as f:
|
||
reader = csv.DictReader(f)
|
||
for row in reader:
|
||
if (row.get("content") or "").strip():
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def _load_review_texts_by_source_row(csv_path: Path) -> Dict[int, str]:
|
||
"""与 结构化_server.load_reviews_from_file 一致：第 1 条数据行 source_row=1。"""
|
||
out: Dict[int, str] = {}
|
||
with csv_path.open(encoding="utf-8-sig", newline="") as f:
|
||
reader = csv.DictReader(f)
|
||
for i, row in enumerate(reader, start=1):
|
||
text = (row.get("content") or "").strip()
|
||
if text:
|
||
out[i] = text
|
||
return out
|
||
|
||
|
||
def _latest_structured_job_id(conn: sqlite3.Connection) -> int:
|
||
row = conn.execute(
|
||
"SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1"
|
||
).fetchone()
|
||
if not row:
|
||
raise RuntimeError("voc_structured.sqlite 中无结构化任务")
|
||
return int(row[0])
|
||
|
||
|
||
def _load_structured_samples(
|
||
structured_db: Path,
|
||
*,
|
||
job_id: int | None = None,
|
||
limit: int = STRUCTURED_APPENDIX_SAMPLE_N,
|
||
) -> List[Tuple[int, Dict[str, Any]]]:
|
||
conn = sqlite3.connect(structured_db)
|
||
try:
|
||
jid = job_id if job_id is not None else _latest_structured_job_id(conn)
|
||
cur = conn.execute(
|
||
"""
|
||
SELECT source_row, extraction_json
|
||
FROM comment_extractions
|
||
WHERE job_id = ?
|
||
ORDER BY source_row
|
||
LIMIT ?
|
||
""",
|
||
(jid, limit),
|
||
)
|
||
return [(int(sr), json.loads(js)) for sr, js in cur.fetchall()]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _extraction_to_display_zh(ext: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""将库内英文字段转为报告展示用中文键（与业务阅读一致）。"""
|
||
feedback: List[Dict[str, str]] = []
|
||
for item in ext.get("product_feedback") or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
feedback.append(
|
||
{
|
||
"方面": str(item.get("aspect", "")).strip(),
|
||
"观点": str(item.get("opinion", "")).strip(),
|
||
"态度": str(item.get("sentiment", "")).strip(),
|
||
"类别标签": str(item.get("category", "")).strip(),
|
||
}
|
||
)
|
||
pains = ext.get("pain_points") or []
|
||
if not isinstance(pains, list):
|
||
pains = []
|
||
return {
|
||
"画像信号": [str(s).strip() for s in ext.get("persona_signals") or [] if str(s).strip()],
|
||
"需求/痛点": [str(p).strip() for p in pains if str(p).strip()],
|
||
"产品反馈": feedback,
|
||
}
|
||
|
||
|
||
def _build_structured_appendix_samples(
|
||
*,
|
||
structured_db: Path,
|
||
cleaned_csv: Path,
|
||
job_id: int | None = None,
|
||
limit: int = STRUCTURED_APPENDIX_SAMPLE_N,
|
||
) -> List[Tuple[int, str, Dict[str, Any]]]:
|
||
if not structured_db.is_file():
|
||
logger.warning("未找到结构化库 %s，跳过结构化提取效果验证", structured_db)
|
||
return []
|
||
texts = _load_review_texts_by_source_row(cleaned_csv)
|
||
try:
|
||
rows = _load_structured_samples(structured_db, job_id=job_id, limit=limit)
|
||
except (RuntimeError, sqlite3.Error) as e:
|
||
logger.warning("读取结构化样本失败: %s", e)
|
||
return []
|
||
out: List[Tuple[int, str, Dict[str, Any]]] = []
|
||
for sr, ext in rows:
|
||
content = texts.get(sr, "").strip() or "（原文缺失）"
|
||
out.append((sr, content, _extraction_to_display_zh(ext)))
|
||
return out
|
||
|
||
|
||
def _load_word_freq(path: Path, top_n: int) -> List[Tuple[str, int]]:
|
||
rows: List[Tuple[str, int]] = []
|
||
with path.open(encoding="utf-8", newline="") as f:
|
||
reader = csv.DictReader(f)
|
||
for row in reader:
|
||
w = (row.get("word") or "").strip()
|
||
if not w:
|
||
continue
|
||
rows.append((w, int(row.get("count") or 0)))
|
||
rows.sort(key=lambda x: x[1], reverse=True)
|
||
return rows[:top_n]
|
||
|
||
|
||
def _stage_section(stage: str) -> str:
|
||
if stage == "1_audience" or stage.startswith("2a_") or stage.startswith("2b_"):
|
||
return "part1"
|
||
return "part2"
|
||
|
||
|
||
def _audience_cluster_from_stage(stage: str) -> int | None:
|
||
if stage.startswith("2a_pain_audience_c"):
|
||
try:
|
||
return int(stage.split("_c")[-1])
|
||
except ValueError:
|
||
return None
|
||
m = re.match(
|
||
r"2b_aspect_opinion_(?:positive|negative|neutral)_audience_c(\d+)$",
|
||
stage,
|
||
)
|
||
if m:
|
||
return int(m.group(1))
|
||
return None
|
||
|
||
|
||
def _sentiment_suffix_from_2b_stage(stage: str) -> str | None:
|
||
m = re.match(
|
||
r"2b_aspect_opinion_(positive|negative|neutral)_audience_c\d+$",
|
||
stage,
|
||
)
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def _stage_title_zh(stage: str, audience_names: Dict[int, str]) -> str:
|
||
if stage == "1_audience":
|
||
return "受众画像聚类（阶段一）"
|
||
if stage in _STAGE_3B_TITLES:
|
||
return _STAGE_3B_TITLES[stage]
|
||
aud_c = _audience_cluster_from_stage(stage)
|
||
if aud_c is not None:
|
||
aud_name = audience_names.get(aud_c, f"受众{aud_c}")
|
||
if stage.startswith("2a_"):
|
||
return f"{aud_name} · 用户需求（阶段二）"
|
||
sent = _sentiment_suffix_from_2b_stage(stage)
|
||
if sent:
|
||
label = _STAGE_2B_BLOCK_TITLE.get(sent, sent)
|
||
return f"{aud_name} · {label}（阶段二）"
|
||
return f"{aud_name} · 产品反馈（阶段二）"
|
||
if stage == "3a_pain_global":
|
||
return "全量用户需求（阶段三）"
|
||
return stage
|
||
|
||
|
||
def _phrase_appendix_stage_title(stage: str) -> str:
|
||
if stage == STAGE_3A_PAIN:
|
||
return "全量用户需求（阶段三）"
|
||
if stage in _STAGE_3B_TITLES:
|
||
return _STAGE_3B_TITLES[stage]
|
||
return stage
|
||
|
||
|
||
def _bundles_for_phrase_appendix(bundles: List[ClusterBundle]) -> List[ClusterBundle]:
|
||
return [
|
||
b
|
||
for b in bundles
|
||
if b.stage in PHRASE_APPENDIX_STAGES and b.cluster_label != -1
|
||
]
|
||
|
||
|
||
def _bundle_sort_key(b: ClusterBundle) -> Tuple[int, int, str, int]:
|
||
sec = 0 if b.section == "part1" else 1
|
||
stage_ord = _STAGE_SORT_KEY.get(b.stage, 0)
|
||
lab = -1 if b.cluster_label == -1 else b.cluster_label
|
||
return (sec, stage_ord, b.stage, lab)
|
||
|
||
|
||
def _cluster_key(stage: str, label: int) -> str:
|
||
return f"{stage}|{label}"
|
||
|
||
|
||
def _anchor_id(stage: str, label: int) -> str:
|
||
safe = re.sub(r"[^a-zA-Z0-9_-]", "_", stage)
|
||
return f"cluster-{safe}-{label}"
|
||
|
||
|
||
def _latest_cluster_run(conn: sqlite3.Connection) -> Tuple[int, int]:
|
||
row = conn.execute(
|
||
"SELECT id, job_id FROM cluster_runs ORDER BY id DESC LIMIT 1"
|
||
).fetchone()
|
||
if not row:
|
||
raise RuntimeError("voc_clustering.sqlite 中无聚类记录")
|
||
return int(row[0]), int(row[1])
|
||
|
||
|
||
def _load_assignments(
|
||
cconn: sqlite3.Connection, run_id: int
|
||
) -> List[Tuple[str, int, int, str, int]]:
|
||
cur = cconn.execute(
|
||
"""
|
||
SELECT stage, cluster_label, source_row, embed_text, embedding_item_id
|
||
FROM cluster_assignments
|
||
WHERE run_id = ?
|
||
ORDER BY stage, cluster_label, source_row
|
||
""",
|
||
(run_id,),
|
||
)
|
||
return [
|
||
(str(s), int(lab), int(sr), str(et), int(eid))
|
||
for s, lab, sr, et, eid in cur.fetchall()
|
||
]
|
||
|
||
|
||
def build_cluster_bundles(
|
||
*,
|
||
cluster_db: Path,
|
||
cleaned_csv: Path,
|
||
embed_db: Path | None = None, # 保留参数兼容 VOC分析，不再使用
|
||
min_cluster_review_ratio: float | None = None,
|
||
) -> Tuple[int, List[ClusterBundle]]:
|
||
del embed_db
|
||
total_reviews = _load_review_count(cleaned_csv)
|
||
|
||
cconn = sqlite3.connect(cluster_db)
|
||
try:
|
||
run_id, _ = _latest_cluster_run(cconn)
|
||
raw = _load_assignments(cconn, run_id)
|
||
finally:
|
||
cconn.close()
|
||
|
||
grouped: Dict[Tuple[str, int], List[Tuple[int, str, int]]] = defaultdict(list)
|
||
texts_by_key: Dict[Tuple[str, int], List[str]] = defaultdict(list)
|
||
stage_source_rows: Dict[str, set[int]] = defaultdict(set)
|
||
|
||
for stage, lab, sr, et, _eid in raw:
|
||
key = (stage, lab)
|
||
grouped[key].append((sr, et, _eid))
|
||
texts_by_key[key].append(et)
|
||
stage_source_rows[stage].add(sr)
|
||
|
||
bundles: List[ClusterBundle] = []
|
||
for (stage, lab), items in sorted(grouped.items(), key=lambda x: (x[0][0], x[0][1])):
|
||
sec = _stage_section(stage)
|
||
if sec not in ("part1", "part2"):
|
||
continue
|
||
unique_srs = sorted({sr for sr, _, _ in items})
|
||
cnt = len(unique_srs)
|
||
stage_total = len(stage_source_rows.get(stage, set()))
|
||
phrase_cnt = len(items)
|
||
stage_ratio = cnt / stage_total if stage_total else 0.0
|
||
ratio = stage_ratio
|
||
ratio_global = phrase_cnt / total_reviews if total_reviews else 0.0
|
||
skip_filter = (
|
||
stage == STAGE_1_AUDIENCE
|
||
or stage.startswith(STAGE_2A_PAIN_PREFIX)
|
||
or stage == STAGE_3A_PAIN
|
||
)
|
||
if (
|
||
min_cluster_review_ratio is not None
|
||
and stage_total > 0
|
||
and stage_ratio < min_cluster_review_ratio
|
||
and not skip_filter
|
||
):
|
||
logger.info(
|
||
"报告跳过小簇 %s / %s：%s 条评论 (本步骤 %.1f%% < %.0f%%)",
|
||
stage,
|
||
lab,
|
||
cnt,
|
||
stage_ratio * 100,
|
||
min_cluster_review_ratio * 100,
|
||
)
|
||
continue
|
||
text_ctr = Counter(texts_by_key[(stage, lab)])
|
||
reps = [t for t, _ in text_ctr.most_common(MAX_PHRASES_PER_CLUSTER)]
|
||
|
||
bundles.append(
|
||
ClusterBundle(
|
||
section=sec,
|
||
stage=stage,
|
||
stage_title_zh=stage,
|
||
cluster_label=lab,
|
||
cluster_title_zh=str(lab),
|
||
anchor_id=_anchor_id(stage, lab),
|
||
review_count=cnt,
|
||
phrase_count=phrase_cnt,
|
||
ratio=ratio,
|
||
ratio_global=ratio_global,
|
||
embed_texts=reps,
|
||
)
|
||
)
|
||
|
||
bundles.sort(key=_bundle_sort_key)
|
||
return total_reviews, bundles
|
||
|
||
|
||
def _apply_cluster_names(
|
||
bundles: List[ClusterBundle], names: Dict[str, str]
|
||
) -> None:
|
||
audience_names: Dict[int, str] = {}
|
||
for b in bundles:
|
||
if b.stage != "1_audience":
|
||
continue
|
||
if b.cluster_label == -1:
|
||
audience_names[-1] = OUTLIER_LABEL_ZH
|
||
b.cluster_title_zh = OUTLIER_LABEL_ZH
|
||
continue
|
||
key = _cluster_key(b.stage, b.cluster_label)
|
||
title = names.get(key, "").strip() or f"受众群体 {b.cluster_label}"
|
||
b.cluster_title_zh = title
|
||
audience_names[b.cluster_label] = title
|
||
|
||
for b in bundles:
|
||
b.stage_title_zh = _stage_title_zh(b.stage, audience_names)
|
||
if b.stage == "1_audience":
|
||
continue
|
||
if b.cluster_label == -1:
|
||
aud_c = _audience_cluster_from_stage(b.stage)
|
||
if aud_c is not None:
|
||
aud = audience_names.get(aud_c, f"受众{aud_c}")
|
||
b.cluster_title_zh = f"{aud} · {OUTLIER_LABEL_ZH}"
|
||
else:
|
||
b.cluster_title_zh = OUTLIER_LABEL_ZH
|
||
continue
|
||
key = _cluster_key(b.stage, b.cluster_label)
|
||
sub = names.get(key, "").strip()
|
||
aud_c = _audience_cluster_from_stage(b.stage)
|
||
if aud_c is not None:
|
||
aud = audience_names.get(aud_c, f"受众{aud_c}")
|
||
b.cluster_title_zh = f"{aud} · {sub}" if sub else f"{aud} · 子簇 {b.cluster_label}"
|
||
else:
|
||
b.cluster_title_zh = sub or f"主题簇 {b.cluster_label}"
|
||
|
||
|
||
def _parse_numbered_translations(
|
||
raw: str, count: int, *, originals: Sequence[str]
|
||
) -> List[str]:
|
||
out: List[str] = []
|
||
for i in range(count):
|
||
m = re.search(rf"\[{i + 1}\]\s*([\s\S]*?)(?=\n\[{i + 2}\]|\Z)", raw)
|
||
out.append(m.group(1).strip() if m else originals[i])
|
||
while len(out) < count:
|
||
j = len(out)
|
||
out.append(originals[j] if j < len(originals) else "")
|
||
return out[:count]
|
||
|
||
|
||
def _chunk_texts_for_translation(texts: List[str]) -> List[List[str]]:
|
||
batches: List[List[str]] = []
|
||
current: List[str] = []
|
||
size = 0
|
||
for text in texts:
|
||
block_size = len(text) + 32
|
||
if current and size + block_size > TRANSLATE_CHARS_PER_REQUEST:
|
||
batches.append(current)
|
||
current = []
|
||
size = 0
|
||
current.append(text)
|
||
size += block_size
|
||
if current:
|
||
batches.append(current)
|
||
return batches
|
||
|
||
|
||
def _translate_unique_phrases(
|
||
phrases: List[str], api_key: str
|
||
) -> Dict[str, str]:
|
||
unique: List[str] = []
|
||
seen: set[str] = set()
|
||
for p in phrases:
|
||
t = p.strip()
|
||
if not t or t in seen:
|
||
continue
|
||
seen.add(t)
|
||
unique.append(t)
|
||
if not unique:
|
||
return {}
|
||
|
||
merged: Dict[str, str] = {}
|
||
batches = _chunk_texts_for_translation(unique)
|
||
for bi, batch in enumerate(batches, start=1):
|
||
logger.info(
|
||
"批量翻译短语（第 %s/%s 批）：%s 条",
|
||
bi,
|
||
len(batches),
|
||
len(batch),
|
||
)
|
||
body = "\n\n".join(f"[{i + 1}]\n{t}" for i, t in enumerate(batch))
|
||
user = (
|
||
f"将以下 {len(batch)} 条英文 VOC 结构化短语逐条译为流畅简体中文。\n"
|
||
"输出格式：仍用 [1]、[2]… 编号；不要解释、不要 Markdown。\n\n"
|
||
+ body
|
||
)
|
||
raw = _call_llm(
|
||
"你是专业英中翻译。按编号输出全部译文，不要遗漏。",
|
||
user,
|
||
api_key,
|
||
temperature=0.2,
|
||
max_tokens=TRANSLATE_MAX_OUTPUT_TOKENS,
|
||
timeout=TRANSLATE_LLM_TIMEOUT_SEC,
|
||
)
|
||
zh_list = _parse_numbered_translations(raw, len(batch), originals=batch)
|
||
for en, zh in zip(batch, zh_list):
|
||
merged[en] = zh
|
||
return merged
|
||
|
||
|
||
def _fill_phrase_translations(
|
||
bundles: List[ClusterBundle], zh_map: Dict[str, str]
|
||
) -> None:
|
||
for b in bundles:
|
||
b.embed_texts_zh = [zh_map.get(t, t) for t in b.embed_texts]
|
||
|
||
|
||
def _bundles_for_prompt(bundles: List[ClusterBundle], section: str) -> str:
|
||
"""按短语数降序，供 LLM 按规模分析各簇。"""
|
||
subset = [b for b in bundles if b.section == section]
|
||
subset.sort(key=lambda b: (-b.phrase_count, b.stage, b.cluster_label))
|
||
lines: List[str] = []
|
||
for b in subset:
|
||
pct_a = f"{b.ratio_global * 100:.1f}%"
|
||
pct_stage = f"{b.ratio * 100:.1f}%"
|
||
key = _cluster_key(b.stage, b.cluster_label)
|
||
lines.append(
|
||
f"- id={key}；阶段={b.stage}；簇标签={b.cluster_label}；"
|
||
f"结构化短语数={b.phrase_count}（占全部评论 a 的 {pct_a}）；"
|
||
f"去重评论数={b.review_count}（占本阶段 {pct_stage}）；"
|
||
f"代表短语：{' | '.join(b.embed_texts)}"
|
||
)
|
||
return "\n".join(lines) if lines else "（无）"
|
||
|
||
|
||
def _audience_cluster_ids_from_bundles(bundles: Sequence[ClusterBundle]) -> List[int]:
|
||
ids: set[int] = set()
|
||
for b in bundles:
|
||
aud = _audience_cluster_from_stage(b.stage)
|
||
if aud is not None:
|
||
ids.add(aud)
|
||
return sorted(ids)
|
||
|
||
|
||
def _load_step2_top2_audience(cluster_db: Path) -> List[int]:
|
||
"""读取聚类阶段二入选的前两个受众簇标签（与 聚类.py _top2_audience_clusters 一致）。"""
|
||
try:
|
||
cconn = sqlite3.connect(cluster_db)
|
||
try:
|
||
run_id, _ = _latest_cluster_run(cconn)
|
||
row = cconn.execute(
|
||
"""
|
||
SELECT meta_json FROM cluster_stage_meta
|
||
WHERE run_id = ? AND stage = ?
|
||
""",
|
||
(run_id, STEP2_TOP2_META_STAGE),
|
||
).fetchone()
|
||
finally:
|
||
cconn.close()
|
||
if not row:
|
||
return []
|
||
meta = json.loads(str(row[0]))
|
||
raw = meta.get("top2_audience_clusters")
|
||
if not isinstance(raw, list):
|
||
return []
|
||
return [int(x) for x in raw]
|
||
except (json.JSONDecodeError, TypeError, ValueError, sqlite3.Error) as e:
|
||
logger.warning("读取 step2 top2 受众簇失败: %s", e)
|
||
return []
|
||
|
||
|
||
def _ordered_step2_audiences(
|
||
bundles: Sequence[ClusterBundle], top2_from_db: Sequence[int]
|
||
) -> List[int]:
|
||
"""阶段二模版顺序：优先 DB 中 top2 排名，再补齐 bundles 里出现的其它受众簇。"""
|
||
in_bundles = set(_audience_cluster_ids_from_bundles(bundles))
|
||
ordered: List[int] = []
|
||
for aud in top2_from_db:
|
||
if aud in in_bundles and aud not in ordered:
|
||
ordered.append(aud)
|
||
for aud in sorted(in_bundles):
|
||
if aud not in ordered:
|
||
ordered.append(aud)
|
||
return ordered
|
||
|
||
|
||
def _audience_rank_title(rank_index: int, audience_cluster: int) -> str:
|
||
del audience_cluster # 仅用于调用方传入映射注释，勿出现在正文标题
|
||
rank_zh = (
|
||
_AUDIENCE_RANK_ZH[rank_index]
|
||
if rank_index < len(_AUDIENCE_RANK_ZH)
|
||
else f"第{rank_index + 1}"
|
||
)
|
||
return f"{rank_zh}大受众簇"
|
||
|
||
|
||
def _stages_in_bundles(bundles: Sequence[ClusterBundle]) -> List[str]:
|
||
return sorted({b.stage for b in bundles})
|
||
|
||
|
||
def _cluster_stage_mapping_guide(
|
||
bundles: Sequence[ClusterBundle],
|
||
*,
|
||
top2_audience_clusters: Sequence[int],
|
||
) -> str:
|
||
"""将本批次实际 stage 映射到 HTML 模版小节（与 聚类.py 五段流程一致）。"""
|
||
stages = _stages_in_bundles(bundles)
|
||
if not stages:
|
||
return "（本批次无聚类簇数据）"
|
||
aud_ids = _ordered_step2_audiences(bundles, top2_audience_clusters)
|
||
top2_txt = (
|
||
"、".join(str(a) for a in top2_audience_clusters)
|
||
if top2_audience_clusters
|
||
else "（见各 2a/2b stage 后缀）"
|
||
)
|
||
lines = [
|
||
"阶段二入选规则（聚类.py）：1_audience 完成后，按各受众簇「去重评论数」降序，"
|
||
f"仅对规模最大的前 2 个簇（本批次 top2 簇标签={top2_txt}）分别做：",
|
||
" · 2a：簇内 pain_point → 报告中称「用户需求」（每受众 1 个 stage）",
|
||
" · 2b：簇内 aspect_opinion → 正面/负面产品反馈、产品客观描述（每受众 3 个 stage）",
|
||
"聚类 stage → HTML 模版小节（仅分析下列已出现的 stage；无数据的小节可省略 <li>）：",
|
||
f"- {STAGE_1_AUDIENCE} → 二·（一）·1.受众画像 ·（1）受众群体特征 &（2）受众的主要需求",
|
||
]
|
||
for i, aud in enumerate(aud_ids):
|
||
rank = _audience_rank_title(i, aud)
|
||
lines.append(
|
||
f"- {STAGE_2A_PAIN_PREFIX}{aud} → 二·（一）·2.分受众分析 · {rank} ·（1）用户需求（2a）"
|
||
)
|
||
for suf, block_title in _STAGE_2B_BLOCK_TITLE.items():
|
||
st = f"{STAGE_2B_AO_PREFIX}{suf}_audience_c{aud}"
|
||
slot = {"positive": "（2）", "negative": "（3）", "neutral": "（4）"}[suf]
|
||
sent_en = {"positive": "Positive", "negative": "Negative", "neutral": "Neutral"}[
|
||
suf
|
||
]
|
||
lines.append(
|
||
f"- {st} → 二·（一）·2.分受众分析 · {rank} ·{slot}{block_title}（2b，sentiment={sent_en}）"
|
||
)
|
||
lines.append(f"- {STAGE_3A_PAIN} → 二·（二）·1.全部受众需求分析（3a）")
|
||
for slot, suf, title in (
|
||
("2.", "positive", "正面产品反馈"),
|
||
("3.", "negative", "负面产品反馈"),
|
||
("4.", "neutral", "产品客观描述"),
|
||
):
|
||
st = f"{STAGE_3B_AO_PREFIX}{suf}"
|
||
marker = "（本批次未出现可省略）" if st not in stages else ""
|
||
lines.append(f"- {st} → 二·（二）·{slot}{title}（3b）{marker}")
|
||
lines.append(
|
||
"- 占比口径：括号内 (XX.X%) = 该簇结构化短语数 ÷ a；"
|
||
"同一 <ul> 内各 <li> 按短语数从高到低排列。"
|
||
)
|
||
lines.append(
|
||
"⚠️ 【排版与格式极度重要警告】："
|
||
"这是面向高管的业务报告，直接暴露原始机器 ID（如 1_audience|2、2a_pain_audience_c2|0）"
|
||
"极其不专业且严重影响阅读！正文及任何列表标题中【绝对禁止】出现原始 ID、stage 名、"
|
||
"簇标签数字；必须且只能使用提炼后的「纯中文业务簇名」。"
|
||
)
|
||
lines.append(
|
||
"- 簇 -1 请统一命名并使用「未归类」或「离群反馈」；"
|
||
"2a/2b 的 CLUSTER_NAMES JSON 中只写子主题，但在 HTML 正文排版时请自行补全为通顺纯中文"
|
||
"（例如：犬类受众-消化不适需求、适口性），不得保留 audience_c2 等代码片段；"
|
||
"2b 子主题禁止带「好评/差评/正面/负面」等情感词（情感由 stage 区分）。"
|
||
)
|
||
lines.append(
|
||
"- 展示用语：全文只用「用户需求 / 产品反馈 / 产品客观描述」，禁止「观点 / 评价 / 痛点」等旧称。"
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _html_list_placeholder(n: int = 2) -> str:
|
||
items = "\n".join(
|
||
" <li><strong>[纯中文业务簇名，绝不许带原始ID]</strong> "
|
||
"(XX.X%)：结合代表短语写深入的业务洞察。</li>"
|
||
for _ in range(n)
|
||
)
|
||
return f" <ul>\n{items}\n </ul>"
|
||
|
||
|
||
def _html_step2_intro(top2_audience_clusters: Sequence[int]) -> str:
|
||
top2_txt = (
|
||
"、".join(str(a) for a in top2_audience_clusters[:2])
|
||
if top2_audience_clusters
|
||
else "见聚类库 step2_filter"
|
||
)
|
||
return (
|
||
f" <!-- 二·（一）·2.分受众分析 · top2 簇标签（禁止写入正文）：{top2_txt} -->\n"
|
||
" <p class=\"muted\">仅对评论量排名前 2 的受众进行分析。</p>"
|
||
)
|
||
|
||
|
||
def _html_audience_step2_blocks(
|
||
audience_ids: Sequence[int],
|
||
*,
|
||
top2_audience_clusters: Sequence[int],
|
||
) -> str:
|
||
if not audience_ids:
|
||
return (
|
||
_html_step2_intro(top2_audience_clusters)
|
||
+ "\n <p>（本批次无「2.分受众分析」聚类结果，请先运行 聚类.py）</p>\n"
|
||
+ _html_list_placeholder(1)
|
||
)
|
||
blocks: List[str] = [_html_step2_intro(top2_audience_clusters)]
|
||
for i, aud in enumerate(audience_ids):
|
||
rank_title = _audience_rank_title(i, aud)
|
||
blocks.append(
|
||
f""" <!-- 二·（一）·2.分受众分析 · {rank_title} · 数据映射（勿写入正文）：{STAGE_2A_PAIN_PREFIX}{aud}；
|
||
{STAGE_2B_AO_PREFIX}positive_audience_c{aud}；
|
||
{STAGE_2B_AO_PREFIX}negative_audience_c{aud}；
|
||
{STAGE_2B_AO_PREFIX}neutral_audience_c{aud} -->
|
||
<p><strong>{rank_title}</strong>（正文请用「1.受众画像」CLUSTER_NAMES 中的纯中文受众名，勿写簇标签号）</p>
|
||
<p><strong>（1）用户需求</strong></p>
|
||
{_html_list_placeholder(2)}
|
||
<p><strong>（2）正面产品反馈</strong></p>
|
||
{_html_list_placeholder(2)}
|
||
<p><strong>（3）负面产品反馈</strong></p>
|
||
{_html_list_placeholder(2)}
|
||
<p><strong>（4）产品客观描述</strong></p>
|
||
{_html_list_placeholder(2)}"""
|
||
)
|
||
return "\n".join(blocks)
|
||
|
||
|
||
def _report_html_template(
|
||
product_name: str,
|
||
total_reviews: int,
|
||
bundles: Sequence[ClusterBundle],
|
||
*,
|
||
top2_audience_clusters: Sequence[int],
|
||
) -> str:
|
||
"""规范化 HTML 报告骨架：摘要 → 评论分析（（一）受众 +（二）全量）→ 改进建议。"""
|
||
audience_ids = _ordered_step2_audiences(bundles, top2_audience_clusters)
|
||
step2_html = _html_audience_step2_blocks(
|
||
audience_ids, top2_audience_clusters=top2_audience_clusters
|
||
)
|
||
return f"""<!-- 全局指令：严格遵守标准 HTML5。正文任何一处不得出现 1_audience|2、2a_pain_audience_c2|0 等原始聚类 ID 或 stage 名，全部替换为纯中文业务簇名。 -->
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8" />
|
||
<title>{product_name}产品站内评论分析报告</title>
|
||
</head>
|
||
<body>
|
||
<h1>{product_name}产品改进建议报告</h1>
|
||
|
||
<h2>一、摘要</h2>
|
||
<p>数据清洗后，共 {total_reviews} 条有效用户评论。</p>
|
||
<ul>
|
||
<li><strong>核心受众与场景：</strong>（说明文字，禁止出现原始 ID）</li>
|
||
<li><strong>最核心的用户需求：</strong>（说明文字）</li>
|
||
<li><strong>显著的产品反馈特征（正/负/客观）：</strong>（说明文字）</li>
|
||
<li><strong>提炼核心改进建议与机会：</strong>（一句话概括方向）</li>
|
||
</ul>
|
||
|
||
<h2>二、{product_name}评论分析</h2>
|
||
<p>统计口径：按结构化短语数占比（结构化短语数 ÷ 全部有效评论数）。</p>
|
||
|
||
<h3>（一）受众画像与分析</h3>
|
||
|
||
<!-- 数据映射：{STAGE_1_AUDIENCE}，勿写入正文 -->
|
||
<h4>1.受众画像</h4>
|
||
<p>（1）受众群体特征：</p>
|
||
{_html_list_placeholder(2)}
|
||
<p>（2）受众的主要需求：</p>
|
||
{_html_list_placeholder(2)}
|
||
|
||
<h4>2.分受众分析</h4>
|
||
{step2_html}
|
||
|
||
<h3>（二）全部用户需求与产品反馈</h3>
|
||
|
||
<!-- 数据映射：{STAGE_3A_PAIN} -->
|
||
<h4>1.全部受众需求分析</h4>
|
||
{_html_list_placeholder(3)}
|
||
|
||
<!-- 数据映射：3b_aspect_opinion_positive -->
|
||
<h4>2.正面产品反馈</h4>
|
||
{_html_list_placeholder(3)}
|
||
|
||
<!-- 数据映射：3b_aspect_opinion_negative -->
|
||
<h4>3.负面产品反馈</h4>
|
||
{_html_list_placeholder(3)}
|
||
|
||
<!-- 数据映射：3b_aspect_opinion_neutral -->
|
||
<h4>4.产品客观描述</h4>
|
||
{_html_list_placeholder(3)}
|
||
|
||
<h2>三、改进建议与机会</h2>
|
||
<ol>
|
||
<li><strong>配方与成分优化：</strong>（针对负面反馈与未满足需求的可执行建议）</li>
|
||
<li><strong>包装与品控升级：</strong>（具体执行建议）</li>
|
||
<li><strong>说明书/Listing优化：</strong>（具体执行建议）</li>
|
||
<li><strong>市场与产品机会：</strong>（基于正面反馈与客观描述中的可放大卖点、人群或场景机会）</li>
|
||
</ol>
|
||
</body>
|
||
</html>"""
|
||
|
||
|
||
def _normalize_report_html_fragment(html_text: str) -> str:
|
||
"""从完整 HTML 文档中提取可嵌入页面的 body 片段。"""
|
||
text = html_text.strip()
|
||
m = re.search(r"<body[^>]*>([\s\S]*?)</body>", text, flags=re.I)
|
||
if m:
|
||
return m.group(1).strip()
|
||
if re.search(r"<!DOCTYPE|<html[\s>]", text, flags=re.I):
|
||
text = re.sub(r"<!DOCTYPE[^>]*>", "", text, flags=re.I)
|
||
text = re.sub(r"<head[^>]*>[\s\S]*?</head>", "", text, flags=re.I)
|
||
text = re.sub(r"</?html[^>]*>", "", text, flags=re.I)
|
||
return text.strip()
|
||
return text
|
||
|
||
|
||
def _strip_report_top_heading(html_fragment: str) -> str:
|
||
"""去掉 LLM 报告正文开头的 h1，避免与页面级标题重复。"""
|
||
return re.sub(
|
||
r"^\s*<h1[^>]*>[\s\S]*?</h1>\s*",
|
||
"",
|
||
html_fragment,
|
||
count=1,
|
||
flags=re.I,
|
||
)
|
||
|
||
|
||
def _build_report_prompt(
|
||
*,
|
||
product_name: str,
|
||
industry: str,
|
||
total_reviews: int,
|
||
bundles: List[ClusterBundle],
|
||
word_freq: List[Tuple[str, int]],
|
||
top2_audience_clusters: Sequence[int],
|
||
) -> Tuple[str, str]:
|
||
n_table = WORD_FREQ_TABLE_N
|
||
n_classify = WORD_CATEGORY_CLASSIFY_N
|
||
n_max = max(n_table, n_classify)
|
||
wf_max = word_freq[:n_max]
|
||
wf_lines = "\n".join(f"{w}\t{c}" for w, c in wf_max)
|
||
cats_literal = "、".join(WORD_CATEGORIES)
|
||
stage_map = _cluster_stage_mapping_guide(
|
||
bundles, top2_audience_clusters=top2_audience_clusters
|
||
)
|
||
html_tpl = _report_html_template(
|
||
product_name,
|
||
total_reviews,
|
||
bundles,
|
||
top2_audience_clusters=top2_audience_clusters,
|
||
)
|
||
system = build_report_system()
|
||
if n_table == n_classify:
|
||
wf_block = (
|
||
f"【词频 Top{n_max}（word\\tcount，用于辅助撰写报告 + WORD_CATEGORY_JSON 分类，排名 1-{n_max}）】\n"
|
||
f"{wf_lines}"
|
||
)
|
||
else:
|
||
wf_block = (
|
||
f"【词频 Top{n_max}（word\\tcount，排名 1-{n_max}）】\n"
|
||
f"前 {n_table} 行用于辅助理解品类与撰写报告；前 {n_classify} 行**仅**用于 WORD_CATEGORY_JSON。\n"
|
||
f"{wf_lines}"
|
||
)
|
||
user = f"""请基于以下用户评论反馈的聚类结果，撰写一份详细的《{product_name}》产品改进建议报告。
|
||
|
||
行业背景：{industry}
|
||
数据清洗后有效评论总数 a = {total_reviews}
|
||
|
||
{wf_block}
|
||
|
||
【聚类流程与报告小节映射】
|
||
{stage_map}
|
||
|
||
【聚类 · 二·（一）受众画像与分析（part1）】
|
||
- {STAGE_1_AUDIENCE} → 1.受众画像 ·（1）受众群体特征、（2）受众的主要需求
|
||
- top2 受众（本批次簇标签：{", ".join(str(x) for x in top2_audience_clusters) or "见 2a/2b stage 后缀"}）→ 2.分受众分析：
|
||
2a 用户需求；2b 正面 / 负面 / 产品客观描述（Neutral 禁止写作「中性反馈」）
|
||
已按「结构化短语数」降序排列：
|
||
{_bundles_for_prompt(bundles, "part1")}
|
||
|
||
【聚类 · 二·（二）全部用户需求与产品反馈（part2）】
|
||
- {STAGE_3A_PAIN} → 1.全部受众需求分析
|
||
- 3b → 2.正面产品反馈 / 3.负面产品反馈 / 4.产品客观描述
|
||
已按「结构化短语数」降序排列：
|
||
{_bundles_for_prompt(bundles, "part2")}
|
||
|
||
{build_report_analysis_requirements(product_name=product_name, stage_1_audience=STAGE_1_AUDIENCE)}
|
||
|
||
{build_report_output_format()}
|
||
|
||
【HTML 结构模版（请替换括号占位为真实分析，可增删 <li>，保持层级）】
|
||
{html_tpl}
|
||
|
||
{build_report_json_markers(cats_literal=cats_literal)}
|
||
"""
|
||
return system, user
|
||
|
||
def _marker_body(raw: str, marker: str) -> str:
|
||
if marker not in raw:
|
||
return ""
|
||
part = raw.split(marker, 1)[1]
|
||
for end_marker in REPORT_MARKERS:
|
||
if end_marker != marker and end_marker in part:
|
||
part = part.split(end_marker, 1)[0]
|
||
part = part.strip()
|
||
part = re.sub(r"^```(?:json)?\s*", "", part, flags=re.I)
|
||
part = re.sub(r"\s*```\s*$", "", part)
|
||
return part
|
||
|
||
|
||
def _json_block_slice(raw: str, marker: str) -> Tuple[str, str]:
|
||
"""返回 (marker 后正文, 用于 json.loads 的子串)。"""
|
||
part = _marker_body(raw, marker)
|
||
if not part:
|
||
return part, ""
|
||
start, end = part.find("{"), part.rfind("}")
|
||
if start != -1 and end > start:
|
||
return part, part[start : end + 1]
|
||
if start != -1:
|
||
return part, part[start:].strip()
|
||
return part, ""
|
||
|
||
|
||
def _coerce_word_category_data(data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""将 LLM 各类别块统一为 {words: [...], analysis: str}。"""
|
||
out: Dict[str, Any] = {}
|
||
pending_analysis: str | None = None
|
||
for key, val in data.items():
|
||
if key == "analysis":
|
||
if isinstance(val, str):
|
||
pending_analysis = val
|
||
continue
|
||
if not isinstance(val, (dict, list)):
|
||
continue
|
||
if isinstance(val, list):
|
||
out[key] = {"words": val, "analysis": ""}
|
||
else:
|
||
words = val.get("words")
|
||
if words is None:
|
||
words = val.get("word")
|
||
out[key] = {
|
||
"words": list(words) if isinstance(words, list) else [],
|
||
"analysis": str(val.get("analysis") or ""),
|
||
}
|
||
if pending_analysis and len(out) == 1:
|
||
only_key = next(iter(out))
|
||
if not out[only_key].get("analysis"):
|
||
out[only_key]["analysis"] = pending_analysis
|
||
return out
|
||
|
||
|
||
def _trim_incomplete_json_tail(text: str) -> str:
|
||
"""去掉截断在引号/逗号上的尾部，便于补全括号。"""
|
||
s = text.rstrip()
|
||
while s.endswith(","):
|
||
s = s[:-1].rstrip()
|
||
m = re.search(r'(,\s*|\[\s*)\"[^\"\\]*$', s)
|
||
if m:
|
||
s = s[: m.start()].rstrip()
|
||
if s.endswith(","):
|
||
s = s[:-1].rstrip()
|
||
return s
|
||
|
||
|
||
def _close_truncated_json_object(text: str) -> str:
|
||
"""为截断在数组/对象中间的 JSON 片段补全括号（尽力而为）。"""
|
||
s = _close_truncated_json_braces(text)
|
||
if s and '"analysis"' not in s:
|
||
s = s.rstrip(", ") + ',"analysis":""'
|
||
if s.count("{") > s.count("}"):
|
||
s = s + "}"
|
||
return s
|
||
|
||
|
||
def _repair_word_category_json(js: str) -> Dict[str, Any] | None:
|
||
"""
|
||
修复 LLM 将各类别写成多段伪对象的情况，例如：
|
||
{"成分/原料":[...],"analysis":"..."}, "受众/使用对象":[...], "analysis":"..."}, ...
|
||
亦支持输出被截断、缺少最外层闭合括号的情形。
|
||
"""
|
||
text = js.strip()
|
||
if not text:
|
||
return None
|
||
if not text.startswith("{"):
|
||
text = "{" + text
|
||
parts = re.split(r"\}\s*,\s*(?=\")", text)
|
||
merged: Dict[str, Any] = {}
|
||
for i, part in enumerate(parts):
|
||
chunk = part.strip().rstrip(",").strip()
|
||
if not chunk:
|
||
continue
|
||
if i == 0:
|
||
if not chunk.endswith("}"):
|
||
chunk = _close_truncated_json_object(chunk)
|
||
else:
|
||
if not chunk.startswith("{"):
|
||
chunk = "{" + chunk
|
||
if not chunk.endswith("}"):
|
||
chunk = _close_truncated_json_object(chunk)
|
||
try:
|
||
obj = json.loads(chunk)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if not isinstance(obj, dict):
|
||
continue
|
||
merged.update(_coerce_word_category_data(obj))
|
||
return merged or None
|
||
|
||
|
||
def _extract_word_category_blocks_regex(text: str) -> Dict[str, Any] | None:
|
||
"""从截断/脏文本中抽取完整的「类别 + words + analysis」块。"""
|
||
if not text or "{" not in text:
|
||
return None
|
||
merged: Dict[str, Any] = {}
|
||
cat_alt = "|".join(re.escape(c) for c in WORD_CATEGORIES)
|
||
pattern = (
|
||
rf'\{{\s*"({cat_alt})"\s*:\s*(\[[^\]]*\])\s*,\s*'
|
||
r'"analysis"\s*:\s*"((?:[^"\\]|\\.)*)"\s*\}'
|
||
)
|
||
for m in re.finditer(pattern, text):
|
||
cat, words_json, analysis = m.group(1), m.group(2), m.group(3)
|
||
try:
|
||
words = json.loads(words_json)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if not isinstance(words, list):
|
||
continue
|
||
merged[cat] = {
|
||
"words": [str(w) for w in words if str(w).strip()],
|
||
"analysis": analysis.replace('\\"', '"'),
|
||
}
|
||
return merged or None
|
||
|
||
|
||
def _try_parse_word_category_json_block(raw: str) -> Tuple[Dict[str, Any], str | None]:
|
||
marker = "===WORD_CATEGORY_JSON==="
|
||
if marker not in raw:
|
||
return {}, f"缺少标记 {marker}"
|
||
part = _marker_body(raw, marker)
|
||
_part, js = _json_block_slice(raw, marker)
|
||
payload = js or (part[part.find("{") :] if "{" in part else part)
|
||
data: Dict[str, Any] | None = None
|
||
if payload:
|
||
try:
|
||
parsed = json.loads(payload)
|
||
if isinstance(parsed, dict):
|
||
data = _coerce_word_category_data(parsed)
|
||
except json.JSONDecodeError:
|
||
data = _repair_word_category_json(payload)
|
||
if data:
|
||
logger.info("已自动修复 %s 的非标准 JSON 结构", marker)
|
||
if not data and payload:
|
||
closed = _close_truncated_json_object(payload)
|
||
if closed != payload:
|
||
data = _repair_word_category_json(closed)
|
||
if data:
|
||
logger.info("已补全截断的 %s 并解析出 %s 类", marker, len(data))
|
||
if not data and part:
|
||
data = _extract_word_category_blocks_regex(part)
|
||
if data:
|
||
logger.info("已从 %s 截断文本中按类别块正则抽取 %s 类", marker, len(data))
|
||
if not data:
|
||
preview = (part or _part)[:200].replace("\n", " ")
|
||
return (
|
||
{},
|
||
f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}",
|
||
)
|
||
return data, None
|
||
|
||
|
||
def _close_truncated_json_braces(text: str) -> str:
|
||
"""仅补全 [] / {{}}，不注入 analysis 等字段。"""
|
||
s = _trim_incomplete_json_tail(text.strip())
|
||
if not s:
|
||
return s
|
||
if not s.startswith("{"):
|
||
s = "{" + s
|
||
if s.count("[") > s.count("]"):
|
||
s = s + "]"
|
||
if s.count("{") > s.count("}"):
|
||
s = s + "}"
|
||
return s
|
||
|
||
|
||
def _extract_json_kv_regex(text: str) -> Dict[str, str]:
|
||
"""从截断的 JSON 对象文本中提取已完整的 key:value 字符串对。"""
|
||
out: Dict[str, str] = {}
|
||
for m in re.finditer(
|
||
r'"((?:[^"\\]|\\.)+)"\s*:\s*"((?:[^"\\]|\\.)*)"',
|
||
text,
|
||
):
|
||
k = m.group(1).replace('\\"', '"')
|
||
v = m.group(2).replace('\\"', '"')
|
||
if k.strip() and v.strip():
|
||
out[k] = v
|
||
return out
|
||
|
||
|
||
def _loads_json_object_loose(payload: str) -> Tuple[Dict[str, Any] | None, str]:
|
||
"""
|
||
解析单个 JSON 对象。兼容尾部多余 `}}`、Extra data、以及真实截断时的补全/正则抽取。
|
||
返回 (dict, repair_note)；repair_note 非空时仅用于日志。
|
||
"""
|
||
text = payload.strip()
|
||
if not text or "{" not in text:
|
||
return None, ""
|
||
|
||
def _loads_once(s: str) -> Dict[str, Any] | None:
|
||
try:
|
||
obj = json.loads(s)
|
||
except json.JSONDecodeError as e:
|
||
if e.msg == "Extra data" or "Extra data" in e.msg:
|
||
try:
|
||
obj, _idx = json.JSONDecoder().raw_decode(s)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
else:
|
||
return None
|
||
return obj if isinstance(obj, dict) else None
|
||
|
||
obj = _loads_once(text)
|
||
if obj is not None:
|
||
tail = text[text.rfind("}") + 1 :].strip()
|
||
if tail:
|
||
return obj, "ignore_trailing_garbage"
|
||
return obj, ""
|
||
|
||
trimmed = text
|
||
while trimmed.endswith("}") and trimmed.count("{") < trimmed.count("}"):
|
||
trimmed = trimmed[:-1].rstrip()
|
||
obj = _loads_once(trimmed)
|
||
if obj is not None:
|
||
return obj, "trim_extra_brace"
|
||
|
||
closed = _close_truncated_json_braces(text)
|
||
if closed != text:
|
||
obj = _loads_once(closed)
|
||
if obj is not None:
|
||
return obj, "close_truncated"
|
||
|
||
kv = _extract_json_kv_regex(text)
|
||
if kv:
|
||
return kv, "regex_kv"
|
||
return None, ""
|
||
|
||
|
||
def _try_parse_json_block(raw: str, marker: str) -> Tuple[Dict[str, Any], str | None]:
|
||
if marker not in raw:
|
||
return {}, f"缺少标记 {marker}"
|
||
part = _marker_body(raw, marker)
|
||
_part, js = _json_block_slice(raw, marker)
|
||
payload = js or (part[part.find("{") :] if "{" in part else "")
|
||
if not payload:
|
||
preview = (part or _part)[:200].replace("\n", " ")
|
||
return {}, f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}"
|
||
data, repair = _loads_json_object_loose(payload)
|
||
if data is not None and repair:
|
||
if repair == "regex_kv":
|
||
logger.info("已从损坏的 %s 中 regex 抽取 %s 个键值对", marker, len(data))
|
||
elif repair == "close_truncated":
|
||
logger.info("已补全截断的 %s（%s 个键）", marker, len(data))
|
||
elif repair in ("ignore_trailing_garbage", "trim_extra_brace"):
|
||
logger.debug("已宽松解析 %s（%s 个键，%s）", marker, len(data), repair)
|
||
if data is None:
|
||
try:
|
||
json.loads(payload)
|
||
except json.JSONDecodeError as e:
|
||
pos = e.pos if e.pos is not None else 0
|
||
ctx = payload[max(0, pos - 50) : pos + 50]
|
||
return (
|
||
{},
|
||
f"{marker} JSON 解析失败: {e.msg}（行{e.lineno}列{e.colno}）；"
|
||
f"错误附近: ...{ctx!r}...",
|
||
)
|
||
return {}, f"{marker} 必须是 JSON 对象"
|
||
if not isinstance(data, dict):
|
||
return {}, f"{marker} 必须是 JSON 对象，实际为 {type(data).__name__}"
|
||
return data, None
|
||
|
||
|
||
def _extract_json_block(raw: str, marker: str) -> Dict[str, Any]:
|
||
data, err = _try_parse_json_block(raw, marker)
|
||
if err:
|
||
logger.warning("解析 %s 失败: %s", marker, err)
|
||
return data
|
||
|
||
|
||
def _extract_report_html_fragment(raw: str) -> str:
|
||
if "===REPORT_HTML===" not in raw:
|
||
return ""
|
||
part = raw.split("===REPORT_HTML===", 1)[1]
|
||
for marker in REPORT_MARKERS[1:]:
|
||
if marker in part:
|
||
part = part.split(marker, 1)[0]
|
||
report_html = part.strip()
|
||
report_html = re.sub(r"```html?", "", report_html, flags=re.I)
|
||
report_html = report_html.replace("```", "").strip()
|
||
return _normalize_report_html_fragment(report_html)
|
||
|
||
|
||
def _validate_word_zh_json(data: Dict[str, Any], marker: str) -> List[str]:
|
||
errs: List[str] = []
|
||
if not data:
|
||
errs.append(f"{marker} 为空对象")
|
||
return errs
|
||
for k, v in data.items():
|
||
if not str(k).strip():
|
||
errs.append(f"{marker} 含空键名")
|
||
if not isinstance(v, str) or not str(v).strip():
|
||
errs.append(f"{marker} 键 {k!r} 的值必须为非空字符串")
|
||
break
|
||
return errs
|
||
|
||
|
||
def _validate_word_category_json(data: Dict[str, Any], marker: str) -> List[str]:
|
||
errs: List[str] = []
|
||
if not data:
|
||
errs.append(f"{marker} 为空对象")
|
||
return errs
|
||
has_any_words = False
|
||
for cat, block in data.items():
|
||
if not isinstance(block, dict):
|
||
errs.append(f"{marker} 类别 {cat!r} 的值必须是对象")
|
||
continue
|
||
words = block.get("words")
|
||
if words is not None:
|
||
if not isinstance(words, list):
|
||
errs.append(f"{marker} 类别 {cat!r}.words 必须是数组")
|
||
elif words:
|
||
has_any_words = True
|
||
analysis = block.get("analysis")
|
||
if analysis is not None and not isinstance(analysis, str):
|
||
errs.append(f"{marker} 类别 {cat!r}.analysis 必须是字符串")
|
||
if not has_any_words:
|
||
errs.append(f"{marker} 所有类别的 words 均为空")
|
||
return errs
|
||
|
||
|
||
def _validate_cluster_names_json(
|
||
data: Dict[str, Any], marker: str, *, expect_keys: bool
|
||
) -> List[str]:
|
||
errs: List[str] = []
|
||
if expect_keys and not data:
|
||
errs.append(f"{marker} 为空对象（需要为聚类簇命名）")
|
||
return errs
|
||
for k, v in data.items():
|
||
if not str(k).strip():
|
||
errs.append(f"{marker} 含空键名")
|
||
if not isinstance(v, str) or not str(v).strip():
|
||
errs.append(f"{marker} 键 {k!r} 的簇名必须为非空字符串")
|
||
break
|
||
return errs
|
||
|
||
|
||
def _dedupe_errors(errors: List[str]) -> List[str]:
|
||
return list(dict.fromkeys(errors))
|
||
|
||
|
||
def _validate_report_response(
|
||
raw: str, *, expect_cluster_names: bool
|
||
) -> List[str]:
|
||
errors: List[str] = []
|
||
for marker in REPORT_MARKERS:
|
||
if marker not in raw:
|
||
if marker == "===CLUSTER_NAMES_JSON===" and expect_cluster_names:
|
||
logger.warning(
|
||
"缺少 %s（将使用程序默认簇名）；若频繁出现请检查模型 max_tokens 是否截断输出",
|
||
marker,
|
||
)
|
||
else:
|
||
errors.append(f"缺少标记 {marker}")
|
||
|
||
report_html = _extract_report_html_fragment(raw)
|
||
if not report_html:
|
||
errors.append("===REPORT_HTML=== 内容为空或无法提取 <body> 片段")
|
||
elif "<h1" not in report_html.lower():
|
||
errors.append("===REPORT_HTML=== 缺少 <h1> 标题")
|
||
elif "<h2" not in report_html.lower():
|
||
errors.append("===REPORT_HTML=== 缺少 <h2> 章节")
|
||
|
||
wzh, err = _try_parse_json_block(raw, "===WORD_ZH_JSON===")
|
||
if err:
|
||
errors.append(err)
|
||
else:
|
||
errors.extend(_validate_word_zh_json(wzh, "===WORD_ZH_JSON==="))
|
||
|
||
cats, err = _try_parse_word_category_json_block(raw)
|
||
if err:
|
||
errors.append(err)
|
||
else:
|
||
errors.extend(_validate_word_category_json(cats, "===WORD_CATEGORY_JSON==="))
|
||
|
||
if "===CLUSTER_NAMES_JSON===" in raw:
|
||
names, err = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===")
|
||
if err:
|
||
errors.append(err)
|
||
else:
|
||
errors.extend(
|
||
_validate_cluster_names_json(
|
||
names,
|
||
"===CLUSTER_NAMES_JSON===",
|
||
expect_keys=expect_cluster_names and not names,
|
||
)
|
||
)
|
||
return _dedupe_errors(errors)
|
||
|
||
|
||
def _build_report_correction_user_message(errors: List[str], raw: str) -> str:
|
||
err_block = "\n".join(f"- {e}" for e in errors)
|
||
max_chars = 100_000
|
||
raw_body = raw if len(raw) <= max_chars else raw[:max_chars] + "\n\n...(上文已截断)..."
|
||
cats_literal = "、".join(WORD_CATEGORIES)
|
||
return build_report_correction_message(
|
||
err_block=err_block,
|
||
raw_body=raw_body,
|
||
cats_literal=cats_literal,
|
||
)
|
||
|
||
|
||
def _fetch_report_llm_raw_with_retry(
|
||
*,
|
||
system: str,
|
||
user: str,
|
||
api_key: str,
|
||
expect_cluster_names: bool,
|
||
max_retries: int = REPORT_PARSE_MAX_RETRIES,
|
||
) -> str:
|
||
messages: List[Dict[str, str]] = [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
]
|
||
raw = _call_report_llm_messages(
|
||
messages,
|
||
api_key,
|
||
)
|
||
for attempt in range(max_retries + 1):
|
||
errors = _validate_report_response(raw, expect_cluster_names=expect_cluster_names)
|
||
if not errors:
|
||
if attempt > 0:
|
||
logger.info("报告 LLM 输出校验通过（第 %s 次修正后）", attempt + 1)
|
||
return raw
|
||
if attempt >= max_retries:
|
||
logger.warning(
|
||
"报告解析校验仍失败（已重试 %s 次），继续使用最后一次输出: %s",
|
||
max_retries,
|
||
errors,
|
||
)
|
||
return raw
|
||
logger.warning(
|
||
"报告解析校验失败（第 %s/%s 次），请求 LLM 修正: %s",
|
||
attempt + 1,
|
||
max_retries,
|
||
errors,
|
||
)
|
||
messages.append({"role": "assistant", "content": raw})
|
||
messages.append(
|
||
{
|
||
"role": "user",
|
||
"content": _build_report_correction_user_message(errors, raw),
|
||
}
|
||
)
|
||
raw = _call_report_llm_messages(
|
||
messages,
|
||
api_key,
|
||
)
|
||
return raw
|
||
|
||
|
||
def _parse_report_response(
|
||
raw: str,
|
||
) -> Tuple[str, Dict[str, str], Dict[str, Any], Dict[str, str]]:
|
||
report_html = _extract_report_html_fragment(raw)
|
||
word_zh_raw, _ = _try_parse_json_block(raw, "===WORD_ZH_JSON===")
|
||
word_zh = {str(k).lower(): str(v) for k, v in word_zh_raw.items()}
|
||
categories, _ = _try_parse_word_category_json_block(raw)
|
||
cluster_names_raw, _ = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===")
|
||
cluster_names = {str(k): str(v) for k, v in cluster_names_raw.items()}
|
||
return report_html, word_zh, categories, cluster_names
|
||
|
||
|
||
def _zh_for_word(word: str, word_zh: Dict[str, str]) -> str:
|
||
return word_zh.get(word.lower(), word_zh.get(word, word))
|
||
|
||
|
||
def _normalize_word_category_data(category_data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""兼容旧版词频分类名「痛点/场景」→「需求/场景」。"""
|
||
if not category_data:
|
||
return category_data
|
||
out = dict(category_data)
|
||
legacy = out.pop("痛点/场景", None)
|
||
if legacy is not None and "需求/场景" not in out:
|
||
out["需求/场景"] = legacy
|
||
out.pop("其他", None)
|
||
return out
|
||
|
||
|
||
def _ensure_word_category_skeleton(category_data: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""保证七个类别键存在且值为 {{words, analysis}} 结构。"""
|
||
base = _normalize_word_category_data(category_data or {})
|
||
out: Dict[str, Any] = {}
|
||
for cat in WORD_CATEGORIES:
|
||
block = base.get(cat)
|
||
if isinstance(block, dict):
|
||
words = block.get("words") if isinstance(block.get("words"), list) else []
|
||
analysis = str(block.get("analysis") or "")
|
||
else:
|
||
words, analysis = [], ""
|
||
out[cat] = {"words": list(words), "analysis": analysis}
|
||
return out
|
||
|
||
|
||
def _category_pool_count_map(
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
) -> Tuple[List[Tuple[str, int]], Dict[str, Tuple[str, int]], Set[str]]:
|
||
top = list(word_freq[:WORD_CATEGORY_CLASSIFY_N])
|
||
count_map = {w.lower(): (w, c) for w, c in top}
|
||
keys = set(count_map.keys())
|
||
return top, count_map, keys
|
||
|
||
|
||
def _valid_category_words_in_pool(
|
||
block: Any,
|
||
count_map: Dict[str, Tuple[str, int]],
|
||
top_keys: Set[str],
|
||
) -> List[str]:
|
||
if not isinstance(block, dict):
|
||
return []
|
||
out: List[str] = []
|
||
seen: Set[str] = set()
|
||
for w in block.get("words") or []:
|
||
key = str(w).lower().strip()
|
||
if not key or key in seen or key not in top_keys:
|
||
continue
|
||
en, cnt = count_map[key]
|
||
if cnt <= 0:
|
||
continue
|
||
seen.add(key)
|
||
out.append(en)
|
||
return out
|
||
|
||
|
||
def _unclassified_pool_words(
|
||
category_data: Dict[str, Any],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
) -> List[str]:
|
||
"""Top{WORD_CATEGORY_CLASSIFY_N} 中尚未归入任何类别的词（按词频降序）。"""
|
||
top, count_map, _top_keys = _category_pool_count_map(word_freq)
|
||
classified = set(_build_word_category_map(category_data).keys())
|
||
return [en for en, _ in top if en.lower() not in classified]
|
||
|
||
|
||
def _parse_word_assign_json(text: str) -> Dict[str, List[str]]:
|
||
"""解析逐词分类结果：{{"word": ["类别", ...], ...}}，空数组表示不分类。"""
|
||
text = _strip_think(text).replace("```json", "").replace("```", "").strip()
|
||
start, end = text.find("{"), text.rfind("}")
|
||
if start == -1 or end <= start:
|
||
return {}
|
||
try:
|
||
obj = json.loads(text[start : end + 1])
|
||
except json.JSONDecodeError:
|
||
obj, _ = _loads_json_object_loose(text[start : end + 1])
|
||
if not isinstance(obj, dict):
|
||
return {}
|
||
out: Dict[str, List[str]] = {}
|
||
for word, cats in obj.items():
|
||
w = str(word).strip()
|
||
if not w:
|
||
continue
|
||
if isinstance(cats, list):
|
||
out[w] = [str(c).strip() for c in cats if str(c).strip()]
|
||
elif isinstance(cats, str) and cats.strip():
|
||
out[w] = [cats.strip()]
|
||
else:
|
||
out[w] = []
|
||
return out
|
||
|
||
|
||
def _build_word_assign_prompt(
|
||
*,
|
||
industry: str,
|
||
product_name: str,
|
||
batch_words: Sequence[str],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
) -> Tuple[str, str]:
|
||
_top, count_map, _top_keys = _category_pool_count_map(word_freq)
|
||
lines: List[str] = []
|
||
for w in batch_words:
|
||
key = w.lower()
|
||
cnt = count_map.get(key, (w, 0))[1]
|
||
lines.append(f"{w}\t{cnt}")
|
||
cats_literal = "、".join(WORD_CATEGORIES)
|
||
return build_word_assign_prompts(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
word_lines="\n".join(lines),
|
||
cats_literal=cats_literal,
|
||
)
|
||
|
||
|
||
def _merge_word_assignments(
|
||
category_data: Dict[str, Any],
|
||
assignments: Dict[str, List[str]],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
) -> Dict[str, Any]:
|
||
_top, count_map, top_keys = _category_pool_count_map(word_freq)
|
||
out = _ensure_word_category_skeleton(category_data)
|
||
cat_set = set(WORD_CATEGORIES)
|
||
for word, cats in assignments.items():
|
||
key = str(word).lower().strip()
|
||
if not key or key not in top_keys:
|
||
continue
|
||
en, cnt = count_map[key]
|
||
if cnt <= 0:
|
||
continue
|
||
valid_cats = [c for c in cats if c in cat_set]
|
||
if not valid_cats:
|
||
continue
|
||
for cat in valid_cats:
|
||
block = out[cat]
|
||
existing = _valid_category_words_in_pool(block, count_map, top_keys)
|
||
if any(e.lower() == key for e in existing):
|
||
continue
|
||
block["words"] = list(existing) + [en]
|
||
return out
|
||
|
||
|
||
def _classify_remaining_words_in_pool(
|
||
category_data: Dict[str, Any],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
*,
|
||
industry: str,
|
||
product_name: str,
|
||
api_key: str,
|
||
) -> Dict[str, Any]:
|
||
"""对 Top{WORD_CATEGORY_CLASSIFY_N} 中尚未分类的词分批逐词尽量分类（可不归类）。"""
|
||
data = _ensure_word_category_skeleton(category_data)
|
||
pending = _unclassified_pool_words(data, word_freq)
|
||
top_n = WORD_CATEGORY_CLASSIFY_N
|
||
classified_n = top_n - len(pending)
|
||
if not pending:
|
||
logger.info(
|
||
"词频分类：Top%s 已全部有类别（%s 个词）",
|
||
top_n,
|
||
classified_n,
|
||
)
|
||
return data
|
||
|
||
logger.info(
|
||
"词频分类：Top%s 已分类 %s 个，待逐词补充分类 %s 个（每批 %s）",
|
||
top_n,
|
||
classified_n,
|
||
len(pending),
|
||
WORD_CATEGORY_ASSIGN_BATCH_SIZE,
|
||
)
|
||
batches = [
|
||
pending[i : i + WORD_CATEGORY_ASSIGN_BATCH_SIZE]
|
||
for i in range(0, len(pending), WORD_CATEGORY_ASSIGN_BATCH_SIZE)
|
||
]
|
||
for bi, batch in enumerate(batches, start=1):
|
||
system, user = _build_word_assign_prompt(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
batch_words=batch,
|
||
word_freq=word_freq,
|
||
)
|
||
raw = _call_llm_messages(
|
||
[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||
api_key,
|
||
temperature=0.2,
|
||
max_tokens=WORD_CATEGORY_ASSIGN_MAX_TOKENS,
|
||
timeout=REPORT_LLM_TIMEOUT_SEC,
|
||
)
|
||
before_keys = set(_build_word_category_map(data).keys())
|
||
assign = _parse_word_assign_json(raw)
|
||
if assign:
|
||
data = _merge_word_assignments(data, assign, word_freq)
|
||
after_keys = set(_build_word_category_map(data).keys())
|
||
newly = len(after_keys - before_keys)
|
||
logger.info(
|
||
"词频逐词分类 第 %s/%s 批：本批 %s 词，新归类 %s 个",
|
||
bi,
|
||
len(batches),
|
||
len(batch),
|
||
newly,
|
||
)
|
||
|
||
still = _unclassified_pool_words(data, word_freq)
|
||
logger.info(
|
||
"词频分类完成：Top%s 共归类 %s 个，未分类 %s 个（词云显示为未分类）",
|
||
top_n,
|
||
top_n - len(still),
|
||
len(still),
|
||
)
|
||
return data
|
||
|
||
|
||
def _categories_needing_analysis(
|
||
category_data: Dict[str, Any],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
) -> List[str]:
|
||
"""有词条但 analysis 为空的类别。"""
|
||
_top, count_map, top_keys = _category_pool_count_map(word_freq)
|
||
data = _ensure_word_category_skeleton(category_data)
|
||
need: List[str] = []
|
||
for cat in WORD_CATEGORIES:
|
||
words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys)
|
||
if words and not str(data[cat].get("analysis") or "").strip():
|
||
need.append(cat)
|
||
return need
|
||
|
||
|
||
def _parse_category_analysis_json(text: str) -> Dict[str, str]:
|
||
"""解析 {{\"类别\": \"一段中文解读\", ...}}。"""
|
||
text = _strip_think(text).replace("```json", "").replace("```", "").strip()
|
||
start, end = text.find("{"), text.rfind("}")
|
||
if start == -1 or end <= start:
|
||
return {}
|
||
try:
|
||
obj = json.loads(text[start : end + 1])
|
||
except json.JSONDecodeError:
|
||
obj, _ = _loads_json_object_loose(text[start : end + 1])
|
||
if not isinstance(obj, dict):
|
||
return {}
|
||
out: Dict[str, str] = {}
|
||
for k, v in obj.items():
|
||
cat = str(k).strip()
|
||
if cat in WORD_CATEGORIES and isinstance(v, str) and v.strip():
|
||
out[cat] = v.strip()
|
||
return out
|
||
|
||
|
||
def _build_category_analysis_prompt(
|
||
*,
|
||
industry: str,
|
||
product_name: str,
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
category_data: Dict[str, Any],
|
||
need_cats: Sequence[str],
|
||
) -> Tuple[str, str]:
|
||
_top, count_map, top_keys = _category_pool_count_map(word_freq)
|
||
data = _ensure_word_category_skeleton(category_data)
|
||
lines: List[str] = []
|
||
for cat in need_cats:
|
||
words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys)[
|
||
:WORD_CATEGORY_ANALYSIS_MAX_WORDS
|
||
]
|
||
parts = []
|
||
for w in words:
|
||
key = w.lower()
|
||
cnt = count_map.get(key, (w, 0))[1]
|
||
parts.append(f"{w}({cnt})")
|
||
lines.append(f"- {cat}：{', '.join(parts) if parts else '（无）'}")
|
||
cats_literal = "、".join(need_cats)
|
||
return build_category_analysis_prompts(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
category_lines="\n".join(lines),
|
||
cats_literal=cats_literal,
|
||
)
|
||
|
||
|
||
def _merge_category_analysis(
|
||
category_data: Dict[str, Any],
|
||
analyses: Dict[str, str],
|
||
) -> Dict[str, Any]:
|
||
out = _ensure_word_category_skeleton(category_data)
|
||
for cat, text in analyses.items():
|
||
if cat not in WORD_CATEGORIES:
|
||
continue
|
||
if text.strip() and not str(out[cat].get("analysis") or "").strip():
|
||
out[cat]["analysis"] = text.strip()
|
||
return out
|
||
|
||
|
||
def _fill_category_analysis(
|
||
category_data: Dict[str, Any],
|
||
word_freq: Sequence[Tuple[str, int]],
|
||
*,
|
||
industry: str,
|
||
product_name: str,
|
||
api_key: str,
|
||
) -> Dict[str, Any]:
|
||
"""为有词但缺少 analysis 的类别自动生成简短中文解读。"""
|
||
data = _ensure_word_category_skeleton(category_data)
|
||
need = _categories_needing_analysis(data, word_freq)
|
||
if not need:
|
||
return data
|
||
|
||
logger.info("词频分类：为 %s 个类别补写 analysis: %s", len(need), need)
|
||
system, user = _build_category_analysis_prompt(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
word_freq=word_freq,
|
||
category_data=data,
|
||
need_cats=need,
|
||
)
|
||
for attempt in range(2):
|
||
raw = _call_llm_messages(
|
||
[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||
api_key,
|
||
temperature=0.3,
|
||
max_tokens=WORD_CATEGORY_ANALYSIS_MAX_TOKENS,
|
||
timeout=REPORT_LLM_TIMEOUT_SEC,
|
||
)
|
||
parsed = _parse_category_analysis_json(raw)
|
||
if parsed:
|
||
data = _merge_category_analysis(data, parsed)
|
||
still = _categories_needing_analysis(data, word_freq)
|
||
if not still:
|
||
logger.info("词频分类各类 analysis 已补全")
|
||
return data
|
||
need = still
|
||
user = _build_category_analysis_prompt(
|
||
industry=industry,
|
||
product_name=product_name,
|
||
word_freq=word_freq,
|
||
category_data=data,
|
||
need_cats=need,
|
||
)[1]
|
||
logger.warning(
|
||
"词频 analysis 仍有 %s 类未生成（第 %s 次重试）: %s",
|
||
len(still),
|
||
attempt + 1,
|
||
still,
|
||
)
|
||
|
||
logger.warning("词频分类 analysis 仍未补全: %s", still)
|
||
return data
|
||
|
||
|
||
def _build_word_category_map(category_data: Dict[str, Any]) -> Dict[str, List[str]]:
|
||
"""英文词（小写）-> 所属分类列表（按 WORD_CATEGORIES 顺序，支持一词多类）。"""
|
||
category_data = _normalize_word_category_data(category_data)
|
||
by_word: Dict[str, List[str]] = defaultdict(list)
|
||
for cat in WORD_CATEGORIES:
|
||
block = category_data.get(cat)
|
||
if not isinstance(block, dict):
|
||
continue
|
||
for w in block.get("words") or []:
|
||
key = str(w).lower().strip()
|
||
if key and cat not in by_word[key]:
|
||
by_word[key].append(cat)
|
||
return dict(by_word)
|
||
|
||
|
||
def _audience_pie_data(bundles: List[ClusterBundle]) -> List[dict]:
|
||
"""受众饼图：1_audience 阶段各簇的去重评论数。"""
|
||
return [
|
||
{"name": b.cluster_title_zh, "value": b.review_count}
|
||
for b in bundles
|
||
if b.stage == "1_audience"
|
||
]
|
||
|
||
|
||
def _sentiment_bar_data(bundles: List[ClusterBundle]) -> Dict[str, int]:
|
||
"""情感分布柱状图：3b 阶段按正/负/中汇总结构化短语数。"""
|
||
agg: Dict[str, int] = {"positive": 0, "negative": 0, "neutral": 0}
|
||
for b in bundles:
|
||
for key in agg:
|
||
if b.stage == f"3b_aspect_opinion_{key}":
|
||
agg[key] += b.phrase_count
|
||
return agg
|
||
|
||
|
||
def _pain_top_chart_data(
|
||
bundles: List[ClusterBundle], *, top_n: int = 10
|
||
) -> List[dict]:
|
||
"""全量需求 Top-N 条形图：3a_pain_global 阶段各簇短语数。"""
|
||
subset = [
|
||
b for b in bundles
|
||
if b.stage == STAGE_3A_PAIN and b.cluster_label != -1
|
||
]
|
||
subset.sort(key=lambda b: -b.phrase_count)
|
||
return [
|
||
{"name": b.cluster_title_zh, "value": b.phrase_count}
|
||
for b in subset[:top_n]
|
||
]
|
||
|
||
|
||
def _negative_top_chart_data(
|
||
bundles: List[ClusterBundle], *, top_n: int = 10
|
||
) -> List[dict]:
|
||
"""负面反馈 Top-N 条形图。"""
|
||
subset = [
|
||
b for b in bundles
|
||
if b.stage == STAGE_3B_NEGATIVE and b.cluster_label != -1
|
||
]
|
||
subset.sort(key=lambda b: -b.phrase_count)
|
||
return [
|
||
{"name": b.cluster_title_zh, "value": b.phrase_count}
|
||
for b in subset[:top_n]
|
||
]
|
||
|
||
|
||
def _radar_chart_data(
|
||
category_data: Dict[str, Any],
|
||
word_freq: List[Tuple[str, int]],
|
||
) -> List[dict]:
|
||
"""词频分类雷达图：各类别词频总和。"""
|
||
count_map = {w.lower(): c for w, c in word_freq}
|
||
result: List[dict] = []
|
||
for cat in WORD_CATEGORIES:
|
||
block = category_data.get(cat, {})
|
||
words = block.get("words", []) if isinstance(block, dict) else []
|
||
total = sum(count_map.get(str(w).lower(), 0) for w in words)
|
||
result.append({"name": cat, "value": total})
|
||
return result
|
||
|
||
|
||
def _compute_dashboard_kpis(
|
||
total_reviews: int,
|
||
bundles: List[ClusterBundle],
|
||
) -> Dict[str, Any]:
|
||
"""Dashboard 四张 KPI 卡片的数据。"""
|
||
audience_count = sum(
|
||
1 for b in bundles
|
||
if b.stage == "1_audience" and b.cluster_label != -1
|
||
)
|
||
sent = _sentiment_bar_data(bundles)
|
||
total_sent = sum(sent.values()) or 1
|
||
cluster_count = sum(1 for b in bundles if b.cluster_label != -1)
|
||
return {
|
||
"total_reviews": total_reviews,
|
||
"audience_count": audience_count,
|
||
"positive_ratio": round(sent["positive"] / total_sent * 100, 1),
|
||
"negative_ratio": round(sent["negative"] / total_sent * 100, 1),
|
||
"neutral_ratio": round(sent["neutral"] / total_sent * 100, 1),
|
||
"cluster_count": cluster_count,
|
||
}
|
||
|
||
|
||
def _wordcloud_color_for_categories(categories: List[str]) -> str:
|
||
if not categories:
|
||
return WORDCLOUD_UNCATEGORIZED_COLOR
|
||
return CATEGORY_COLORS.get(categories[0], WORDCLOUD_UNCATEGORIZED_COLOR)
|
||
|
||
|
||
def _render_wordcloud_legend() -> str:
|
||
items = "".join(
|
||
f'<span class="wc-legend-item">'
|
||
f'<i class="wc-dot" style="background:{CATEGORY_COLORS[c]}"></i>'
|
||
f"{html.escape(c)}</span>"
|
||
for c in WORD_CATEGORIES
|
||
)
|
||
items += (
|
||
f'<span class="wc-legend-item">'
|
||
f'<i class="wc-dot" style="background:{WORDCLOUD_UNCATEGORIZED_COLOR}"></i>'
|
||
f"未分类</span>"
|
||
)
|
||
return f'<div class="wc-legend">{items}</div>'
|
||
|
||
|
||
def _wordcloud_data(
|
||
word_freq: List[Tuple[str, int]],
|
||
word_zh: Dict[str, str],
|
||
category_data: Dict[str, Any],
|
||
) -> List[dict]:
|
||
items = word_freq[:WORDCLOUD_TOP_N]
|
||
if not items:
|
||
return []
|
||
word_cats = _build_word_category_map(category_data)
|
||
counts = [c for _, c in items]
|
||
c_min, c_max = min(counts), max(counts)
|
||
span = c_max - c_min
|
||
out: List[dict] = []
|
||
for rank, (word, count) in enumerate(items, start=1):
|
||
norm = 1.0 if span <= 0 else (count - c_min) / span
|
||
value = max(1, int((norm**WORDCLOUD_SIZE_POWER) * 1000))
|
||
zh = _zh_for_word(word, word_zh)
|
||
cats = word_cats.get(word.lower(), [])
|
||
color = _wordcloud_color_for_categories(cats)
|
||
out.append(
|
||
{
|
||
"name": word,
|
||
"value": value,
|
||
"count": count,
|
||
"rank": rank,
|
||
"en": word,
|
||
"zh": zh,
|
||
"category": cats[0] if cats else "",
|
||
"categories": cats,
|
||
"textStyle": {"color": color},
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def _freq_rank_map(word_freq: List[Tuple[str, int]], n: int) -> Dict[str, int]:
|
||
return {w.lower(): i + 1 for i, (w, _) in enumerate(word_freq[:n])}
|
||
|
||
|
||
def _category_top_words(
|
||
category_data: Dict[str, Any],
|
||
word_freq: List[Tuple[str, int]],
|
||
*,
|
||
top_n: int = 10,
|
||
) -> Dict[str, List[Tuple[str, int, str]]]:
|
||
category_data = _normalize_word_category_data(category_data)
|
||
count_map = {w.lower(): (w, c) for w, c in word_freq}
|
||
rank_map = _freq_rank_map(word_freq, WORD_CATEGORY_CLASSIFY_N)
|
||
out: Dict[str, List[Tuple[str, int, str]]] = {}
|
||
for cat in WORD_CATEGORIES:
|
||
block = category_data.get(cat)
|
||
if not isinstance(block, dict):
|
||
out[cat] = []
|
||
continue
|
||
words = block.get("words") or []
|
||
rows: List[Tuple[str, int, str]] = []
|
||
seen: set[str] = set()
|
||
for w in words:
|
||
key = str(w).lower().strip()
|
||
if not key or key in seen:
|
||
continue
|
||
seen.add(key)
|
||
if key in count_map:
|
||
en, cnt = count_map[key]
|
||
else:
|
||
en, cnt = str(w), 0
|
||
if cnt <= 0:
|
||
continue
|
||
rows.append((en, cnt, str(rank_map.get(key, ""))))
|
||
rows.sort(key=lambda x: (-x[1], x[0]))
|
||
out[cat] = rows[:top_n]
|
||
return out
|
||
|
||
|
||
def _render_category_blocks(
|
||
category_data: Dict[str, Any],
|
||
category_words: Dict[str, List[Tuple[str, int, str]]],
|
||
word_zh: Dict[str, str],
|
||
) -> str:
|
||
category_data = _normalize_word_category_data(category_data)
|
||
parts = [
|
||
'<div class="wf-categories">',
|
||
"<h3>词频分类洞察</h3>",
|
||
]
|
||
for cat in WORD_CATEGORIES:
|
||
block = category_data.get(cat)
|
||
analysis = ""
|
||
if isinstance(block, dict):
|
||
analysis = str(block.get("analysis") or "").strip()
|
||
rows = category_words.get(cat, [])
|
||
parts.append(f'<div class="wf-cat"><h4>{html.escape(cat)}</h4>')
|
||
if analysis:
|
||
parts.append(f"<p>{html.escape(analysis)}</p>")
|
||
if rows:
|
||
parts.append(
|
||
"<table><thead><tr><th>排名</th><th>英文</th><th>中文</th><th>次数</th></tr></thead><tbody>"
|
||
)
|
||
for en, cnt, rank in rows:
|
||
zh = _zh_for_word(en, word_zh)
|
||
rank_cell = rank if rank else "—"
|
||
parts.append(
|
||
f"<tr><td>{html.escape(str(rank_cell))}</td>"
|
||
f"<td>{html.escape(en)}</td><td>{html.escape(zh)}</td>"
|
||
f"<td>{cnt}</td></tr>"
|
||
)
|
||
parts.append("</tbody></table>")
|
||
else:
|
||
parts.append('<p class="muted">（本类暂无词条）</p>')
|
||
parts.append("</div>")
|
||
parts.append("</div>")
|
||
return "\n".join(parts)
|
||
|
||
|
||
def _render_freq_table_pages(
|
||
word_freq: List[Tuple[str, int]], word_zh: Dict[str, str]
|
||
) -> str:
|
||
top = word_freq[:WORD_FREQ_TABLE_N]
|
||
total = sum(c for _, c in top) or 1
|
||
max_cnt = max((c for _, c in top), default=1)
|
||
row_lines: List[str] = []
|
||
for rank, (en, cnt) in enumerate(top, start=1):
|
||
page = (rank - 1) // WORD_FREQ_PAGE_SIZE + 1
|
||
zh = _zh_for_word(en, word_zh)
|
||
pct = cnt / total * 100.0
|
||
bar_w = max(4, int(cnt / max_cnt * 100))
|
||
row_lines.append(
|
||
f'<tr class="wf-row" data-page="{page}">'
|
||
f"<td>{rank}</td>"
|
||
f'<td class="wf-en">{html.escape(en)}</td>'
|
||
f"<td>{html.escape(zh)}</td>"
|
||
f'<td class="wf-count"><span class="wf-bar" style="width:{bar_w}%"></span>'
|
||
f"<span>{cnt}</span></td>"
|
||
f"<td>{pct:.2f}%</td></tr>"
|
||
)
|
||
n_pages = min(
|
||
WORD_FREQ_PAGES,
|
||
(len(top) + WORD_FREQ_PAGE_SIZE - 1) // WORD_FREQ_PAGE_SIZE or 1,
|
||
)
|
||
tabs = "".join(
|
||
f'<button type="button" class="wf-tab{" active" if i == 0 else ""}" '
|
||
f'data-page="{i + 1}">第{i + 1}页</button>'
|
||
for i in range(n_pages)
|
||
)
|
||
body = f"""<motion class="wf-pager">
|
||
<motion class="wf-tab-bar">{tabs}</div>
|
||
<div class="wf-nav">
|
||
<button type="button" id="wf-prev">上一页</button>
|
||
<span id="wf-page-label">第 1 / {n_pages} 页</span>
|
||
<button type="button" id="wf-next">下一页</button>
|
||
</div>
|
||
<div class="wf-table-wrap">
|
||
<table id="wf-freq-table" class="wf-freq-table">
|
||
<thead><tr>
|
||
<th>排名</th><th>英文</th><th>中文</th><th>次数</th><th>占比</th>
|
||
</tr></thead>
|
||
<tbody id="wf-tbody">
|
||
{chr(10).join(row_lines)}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<p class="muted">占比 = 该词次数 / Top{WORD_FREQ_TABLE_N} 词次数之和（相对占比）。</p>
|
||
</div>"""
|
||
return body.replace("<motion", "<div", 2)
|
||
|
||
|
||
def _render_wordfreq_section(
|
||
word_freq: List[Tuple[str, int]],
|
||
word_zh: Dict[str, str],
|
||
category_data: Dict[str, Any],
|
||
) -> str:
|
||
category_words = _category_top_words(category_data, word_freq, top_n=10)
|
||
cat_html = _render_category_blocks(category_data, category_words, word_zh)
|
||
table_html = _render_freq_table_pages(word_freq, word_zh)
|
||
return (
|
||
f'<section id="sec-wordfreq">\n'
|
||
f" <h2>词频分析</h2>\n"
|
||
f' <p class="muted">词云 Top {WORDCLOUD_TOP_N} · 词表 Top {WORD_FREQ_TABLE_N} · '
|
||
f"分类词表 Top {WORD_CATEGORY_CLASSIFY_N}</p>\n"
|
||
f" <h3>词云图</h3>\n"
|
||
f" {_render_wordcloud_legend()}\n"
|
||
f' <div id="wordcloud-chart"></div>\n'
|
||
f" {cat_html}\n"
|
||
f" <h3>全部词频表</h3>\n"
|
||
f" {table_html}\n"
|
||
f"</section>"
|
||
)
|
||
|
||
|
||
def _render_phrase_sections(appendix_bundles: List[ClusterBundle]) -> str:
|
||
by_stage: Dict[str, List[ClusterBundle]] = defaultdict(list)
|
||
for b in appendix_bundles:
|
||
by_stage[b.stage].append(b)
|
||
|
||
parts = ['<div id="sec-phrases" class="ai-verify-body">']
|
||
for stage in PHRASE_APPENDIX_STAGES:
|
||
title = _phrase_appendix_stage_title(stage)
|
||
parts.append(f'<h3 class="stage-h">{html.escape(title)}</h3>')
|
||
rows = sorted(by_stage.get(stage, []), key=lambda b: b.cluster_label)
|
||
if not rows:
|
||
parts.append('<p class="muted">（暂无）</p>')
|
||
continue
|
||
for b in rows:
|
||
pct = f"{b.ratio * 100:.1f}%"
|
||
parts.append(
|
||
f'<h4 id="{b.anchor_id}">{html.escape(b.cluster_title_zh)}'
|
||
f' <span class="muted">（{b.review_count} 条评论 · {pct}）</span></h4>'
|
||
)
|
||
display = b.embed_texts_zh or b.embed_texts
|
||
parts.append(
|
||
f'<details><summary>展开本簇 {len(display)} 条代表性表述（中文）</summary><ol>'
|
||
)
|
||
for txt in display:
|
||
parts.append(f"<li><p>{html.escape(txt)}</p></li>")
|
||
if not display:
|
||
parts.append("<li><p>（无可用表述）</p></li>")
|
||
parts.append("</ol></details>")
|
||
parts.append("</div>")
|
||
return "\n".join(parts)
|
||
|
||
|
||
def _render_structured_validation_section(
|
||
samples: List[Tuple[int, str, Dict[str, Any]]],
|
||
) -> str:
|
||
parts = [
|
||
'<div id="sec-structured" class="ai-verify-body">',
|
||
f'<p class="muted">展示前 {STRUCTURED_APPENDIX_SAMPLE_N} 条评论原文及对应结构化结果（按 source_row 升序）。</p>',
|
||
]
|
||
if not samples:
|
||
parts.append('<p class="muted">（暂无结构化样本，请先执行结构化步骤。）</p>')
|
||
parts.append("</div>")
|
||
return "\n".join(parts)
|
||
|
||
for i, (sr, content, display) in enumerate(samples, start=1):
|
||
js = json.dumps(display, ensure_ascii=False, separators=(",", ":"))
|
||
parts.append(f'<article class="struct-sample" id="struct-sample-{sr}">')
|
||
parts.append(f"<h4>样本 {i} <span class=\"muted\">（source_row={sr}）</span></h4>")
|
||
parts.append("<p><strong>评论原文：</strong></p>")
|
||
parts.append(f'<p class="struct-review">{html.escape(content)}</p>')
|
||
parts.append("<p><strong>结构化内容：</strong></p>")
|
||
parts.append(f'<pre class="struct-json">{html.escape(js)}</pre>')
|
||
parts.append("</article>")
|
||
parts.append("</div>")
|
||
return "\n".join(parts)
|
||
|
||
|
||
def _wrap_ai_verify_panel(title: str, inner_html: str) -> str:
|
||
"""将子版块包进可折叠面板（默认收起）。"""
|
||
return (
|
||
f'<details class="ai-verify-panel">\n'
|
||
f" <summary>{html.escape(title)}</summary>\n"
|
||
f" {inner_html}\n"
|
||
f"</details>"
|
||
)
|
||
|
||
|
||
def _render_ai_validation_section(
|
||
structured_html: str,
|
||
phrases_html: str,
|
||
) -> str:
|
||
"""AI 分析效果验证：结构化 + 聚类，机器固定顺序、可展开。"""
|
||
structured_panel = _wrap_ai_verify_panel("结构化提取效果验证", structured_html)
|
||
phrases_panel = _wrap_ai_verify_panel("聚类效果验证", phrases_html)
|
||
return (
|
||
'<section id="sec-ai-verify">\n'
|
||
" <h2>模型中间过程可视化</h2>\n"
|
||
' <p class="muted">以下为结构化与聚类结果的抽样展示，点击标题展开查看。</p>\n'
|
||
f" {structured_panel}\n"
|
||
f" {phrases_panel}\n"
|
||
"</section>"
|
||
)
|
||
|
||
|
||
def _assemble_html(
|
||
*,
|
||
product_name: str,
|
||
wordfreq_html: str,
|
||
report_html: str,
|
||
phrases_html: str,
|
||
structured_html: str,
|
||
wordcloud_data: List[dict],
|
||
audience_pie_data: List[dict],
|
||
sentiment_data: Dict[str, int],
|
||
pain_top_data: List[dict],
|
||
negative_top_data: List[dict],
|
||
radar_data: List[dict],
|
||
kpis: Dict[str, Any],
|
||
) -> str:
|
||
"""按固定版块顺序拼装最终 HTML：Dashboard → 洞察报告 → 词频 → 附录。"""
|
||
page_title = f"{product_name} · 评论分析报告"
|
||
report_body = _strip_report_top_heading(report_html)
|
||
report_section = (
|
||
'<section id="sec-report">\n'
|
||
f" {report_body}\n"
|
||
"</section>"
|
||
)
|
||
ai_verify_section = _render_ai_validation_section(structured_html, phrases_html)
|
||
|
||
wc_json = json.dumps(wordcloud_data, ensure_ascii=False)
|
||
aud_json = json.dumps(audience_pie_data, ensure_ascii=False)
|
||
sent_json = json.dumps(sentiment_data, ensure_ascii=False)
|
||
pain_json = json.dumps(pain_top_data, ensure_ascii=False)
|
||
neg_json = json.dumps(negative_top_data, ensure_ascii=False)
|
||
radar_json = json.dumps(radar_data, ensure_ascii=False)
|
||
|
||
kpi_reviews = f"{kpis['total_reviews']:,}"
|
||
kpi_audiences = kpis["audience_count"]
|
||
kpi_pos = kpis["positive_ratio"]
|
||
kpi_neg = kpis["negative_ratio"]
|
||
|
||
body_sections = "\n".join((report_section, wordfreq_html, ai_verify_section))
|
||
|
||
return f"""<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>{page_title}</title>
|
||
<script src="https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js"></script>
|
||
<script src="https://cdn.jsdelivr.net/npm/echarts-wordcloud@2/dist/echarts-wordcloud.min.js"></script>
|
||
<style>
|
||
:root {{ --c-primary: #2563eb; --c-pos: #16a34a; --c-neg: #dc2626; --c-neu: #6366f1;
|
||
--c-bg: #f8fafc; --c-border: #e2e8f0; --radius: 12px; }}
|
||
*, *::before, *::after {{ box-sizing: border-box; }}
|
||
body {{ font-family: -apple-system, "PingFang SC", "Microsoft YaHei", sans-serif;
|
||
margin: 0; padding: 0; line-height: 1.6; color: #1e293b; background: #fff; }}
|
||
.muted {{ color: #64748b; font-size: 0.9em; }}
|
||
|
||
/* --- 导航栏 --- */
|
||
.report-nav {{ position: sticky; top: 0; z-index: 100; background: rgba(255,255,255,0.92);
|
||
backdrop-filter: blur(8px); border-bottom: 1px solid var(--c-border);
|
||
display: flex; align-items: center; gap: 0; padding: 0 24px; overflow-x: auto; }}
|
||
.report-nav a {{ padding: 14px 18px; font-size: 0.92em; font-weight: 500;
|
||
color: #475569; text-decoration: none; white-space: nowrap;
|
||
border-bottom: 2px solid transparent; transition: all 0.2s; }}
|
||
.report-nav a:hover {{ color: var(--c-primary); }}
|
||
.report-nav a.active {{ color: var(--c-primary); border-bottom-color: var(--c-primary); }}
|
||
|
||
.page-wrap {{ max-width: 1200px; margin: 0 auto; padding: 24px 28px 60px; }}
|
||
.page-title {{ font-size: 1.6em; font-weight: 700; margin: 0 0 8px; color: #0f172a; }}
|
||
.page-subtitle {{ color: #64748b; margin: 0 0 32px; font-size: 0.95em; }}
|
||
|
||
/* --- Dashboard --- */
|
||
.kpi-grid {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; margin-bottom: 28px; }}
|
||
.kpi-card {{ background: var(--c-bg); border: 1px solid var(--c-border); border-radius: var(--radius);
|
||
padding: 20px 18px; text-align: center; }}
|
||
.kpi-value {{ font-size: 2em; font-weight: 700; color: #0f172a; line-height: 1.2; }}
|
||
.kpi-label {{ font-size: 0.85em; color: #64748b; margin-top: 4px; }}
|
||
.kpi-card.pos .kpi-value {{ color: var(--c-pos); }}
|
||
.kpi-card.neg .kpi-value {{ color: var(--c-neg); }}
|
||
.chart-grid {{ display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 32px; }}
|
||
.chart-box {{ background: var(--c-bg); border: 1px solid var(--c-border);
|
||
border-radius: var(--radius); padding: 16px; }}
|
||
.chart-box h3 {{ margin: 0 0 8px; font-size: 1em; font-weight: 600; color: #334155; }}
|
||
.chart-container {{ width: 100%; height: 360px; }}
|
||
.chart-container.tall {{ height: 420px; }}
|
||
|
||
/* --- 各区块通用 --- */
|
||
section {{ margin-top: 40px; }}
|
||
section > h2 {{ font-size: 1.3em; border-bottom: 2px solid var(--c-primary);
|
||
padding-bottom: 10px; color: #0f172a; }}
|
||
|
||
/* --- 词频 --- */
|
||
#wordcloud-chart {{ width: 100%; height: 520px; margin: 16px 0 24px; }}
|
||
.wc-legend {{ display: flex; flex-wrap: wrap; gap: 8px 14px; margin: 8px 0 12px;
|
||
font-size: 0.88em; color: #444; }}
|
||
.wc-legend-item {{ display: inline-flex; align-items: center; gap: 6px; }}
|
||
.wc-dot {{ display: inline-block; width: 10px; height: 10px; border-radius: 50%; }}
|
||
table {{ border-collapse: collapse; width: 100%; margin: 12px 0; }}
|
||
th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
|
||
th {{ background: #f0f0f0; }}
|
||
.wf-categories {{ margin: 24px 0; }}
|
||
.wf-cat {{ margin-bottom: 20px; }}
|
||
.wf-pager {{ margin: 16px 0; }}
|
||
.wf-tab-bar {{ margin-bottom: 8px; }}
|
||
.wf-tab {{ margin: 4px 8px 4px 0; padding: 6px 12px; cursor: pointer;
|
||
border: 1px solid #ccc; background: #fff; border-radius: 4px; }}
|
||
.wf-tab.active {{ background: var(--c-primary); color: #fff; border-color: var(--c-primary); }}
|
||
.wf-nav {{ margin: 12px 0; display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }}
|
||
.wf-table-wrap {{ max-height: 520px; overflow: auto; border: 1px solid var(--c-border); border-radius: 8px; }}
|
||
.wf-freq-table thead th {{ position: sticky; top: 0; z-index: 1; box-shadow: 0 1px 0 #ddd; }}
|
||
.wf-freq-table tbody tr:nth-child(even) {{ background: #fafafa; }}
|
||
.wf-freq-table tbody tr:hover {{ background: #f0f7ff; }}
|
||
.wf-row {{ display: none; }}
|
||
.wf-row.wf-visible {{ display: table-row; }}
|
||
.wf-count {{ min-width: 120px; }}
|
||
.wf-count .wf-bar {{ display: inline-block; height: 10px; margin-right: 8px;
|
||
background: linear-gradient(90deg, #93c5fd, var(--c-primary)); border-radius: 2px; vertical-align: middle; }}
|
||
.wf-en {{ font-family: ui-monospace, monospace; font-size: 0.92em; }}
|
||
|
||
/* --- 附录 --- */
|
||
details {{ margin: 12px 0 24px; padding: 8px; background: #fafafa; border-radius: 6px; }}
|
||
details.ai-verify-panel {{ margin: 16px 0; padding: 0; border: 1px solid var(--c-border);
|
||
background: #fff; }}
|
||
details.ai-verify-panel > summary {{ padding: 12px 16px; font-size: 1.05em; font-weight: 600;
|
||
cursor: pointer; list-style: none; background: var(--c-bg); border-radius: 6px 6px 0 0; }}
|
||
details.ai-verify-panel > summary::-webkit-details-marker {{ display: none; }}
|
||
details.ai-verify-panel > summary::before {{ content: "\\25B8 "; color: var(--c-primary); }}
|
||
details.ai-verify-panel[open] > summary::before {{ content: "\\25BE "; }}
|
||
details.ai-verify-panel[open] > summary {{ border-bottom: 1px solid var(--c-border); }}
|
||
.ai-verify-body {{ padding: 12px 16px 16px; }}
|
||
.stage-h {{ margin-top: 32px; border-bottom: 1px solid #eee; padding-bottom: 8px; }}
|
||
.struct-sample {{ margin: 24px 0; padding: 16px; border: 1px solid var(--c-border);
|
||
border-radius: 8px; background: #fafafa; }}
|
||
.struct-sample h4 {{ margin: 0 0 12px; }}
|
||
.struct-review {{ white-space: pre-wrap; word-break: break-word; margin: 8px 0 16px; }}
|
||
.struct-json {{ margin: 0; padding: 12px; background: #fff; border: 1px solid var(--c-border);
|
||
border-radius: 6px; overflow-x: auto; font-size: 0.9em; line-height: 1.5; }}
|
||
|
||
@media (max-width: 768px) {{
|
||
.kpi-grid {{ grid-template-columns: repeat(2, 1fr); }}
|
||
.chart-grid {{ grid-template-columns: 1fr; }}
|
||
.report-nav {{ padding: 0 12px; }}
|
||
.report-nav a {{ padding: 12px 12px; font-size: 0.85em; }}
|
||
.page-wrap {{ padding: 16px 14px 40px; }}
|
||
}}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
|
||
<nav class="report-nav" id="main-nav">
|
||
<a href="#sec-dashboard">数据总览</a>
|
||
<a href="#sec-report">用户洞察</a>
|
||
<a href="#sec-wordfreq">词频分析</a>
|
||
<a href="#sec-ai-verify">附录</a>
|
||
</nav>
|
||
|
||
<div class="page-wrap">
|
||
<h1 class="page-title">{page_title}</h1>
|
||
<p class="page-subtitle">数据 \\u2192 洞察 \\u2192 机会 \\u2192 方案</p>
|
||
|
||
<!-- ========== 数据总览 ========== -->
|
||
<section id="sec-dashboard">
|
||
<h2>数据总览</h2>
|
||
<div class="kpi-grid">
|
||
<div class="kpi-card">
|
||
<div class="kpi-value">{kpi_reviews}</div>
|
||
<div class="kpi-label">有效评论数</div>
|
||
</div>
|
||
<div class="kpi-card">
|
||
<div class="kpi-value">{kpi_audiences}</div>
|
||
<div class="kpi-label">受众群体</div>
|
||
</div>
|
||
<div class="kpi-card pos">
|
||
<div class="kpi-value">{kpi_pos}%</div>
|
||
<div class="kpi-label">正面反馈占比</div>
|
||
</div>
|
||
<div class="kpi-card neg">
|
||
<div class="kpi-value">{kpi_neg}%</div>
|
||
<div class="kpi-label">负面反馈占比</div>
|
||
</div>
|
||
</div>
|
||
<div class="chart-grid">
|
||
<div class="chart-box">
|
||
<h3>受众画像分布</h3>
|
||
<div id="audience-chart" class="chart-container"></div>
|
||
</div>
|
||
<div class="chart-box">
|
||
<h3>产品反馈情感分布</h3>
|
||
<div id="sentiment-chart" class="chart-container"></div>
|
||
</div>
|
||
<div class="chart-box">
|
||
<h3>用户需求 Top 10</h3>
|
||
<div id="pain-top-chart" class="chart-container tall"></div>
|
||
</div>
|
||
<div class="chart-box">
|
||
<h3>负面反馈 Top 10</h3>
|
||
<div id="negative-top-chart" class="chart-container tall"></div>
|
||
</div>
|
||
</div>
|
||
<div class="chart-grid">
|
||
<div class="chart-box">
|
||
<h3>词频分类关注度</h3>
|
||
<div id="radar-chart" class="chart-container"></div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- ========== 用户洞察与行动方案 ========== -->
|
||
{body_sections}
|
||
|
||
</div><!-- .page-wrap -->
|
||
|
||
<script>
|
||
(function() {{
|
||
/* --- 工具函数 --- */
|
||
var E = typeof echarts !== 'undefined' ? echarts : null;
|
||
function initChart(id, opt) {{
|
||
var el = document.getElementById(id);
|
||
if (!el || !E) return null;
|
||
var c = E.init(el);
|
||
c.setOption(opt);
|
||
window.addEventListener('resize', function() {{ c.resize(); }});
|
||
return c;
|
||
}}
|
||
|
||
/* --- 受众饼图 --- */
|
||
var audData = {aud_json};
|
||
if (audData.length) {{
|
||
initChart('audience-chart', {{
|
||
tooltip: {{ trigger: 'item', formatter: '{{b}}: {{c}} 条 ({{d}}%)' }},
|
||
color: ['#3b82f6','#f59e0b','#10b981','#8b5cf6','#ef4444','#06b6d4','#ec4899','#84cc16'],
|
||
series: [{{
|
||
type: 'pie', radius: ['30%','70%'], roseType: 'radius',
|
||
itemStyle: {{ borderRadius: 6, borderColor: '#fff', borderWidth: 2 }},
|
||
label: {{ formatter: '{{b}}\\n{{d}}%' }},
|
||
data: audData
|
||
}}]
|
||
}});
|
||
}}
|
||
|
||
/* --- 情感柱状图 --- */
|
||
var sentRaw = {sent_json};
|
||
initChart('sentiment-chart', {{
|
||
tooltip: {{ trigger: 'axis' }},
|
||
xAxis: {{ type: 'category', data: ['\\u6b63\\u9762\\u53cd\\u9988','\\u8d1f\\u9762\\u53cd\\u9988','\\u5ba2\\u89c2\\u63cf\\u8ff0'],
|
||
axisLabel: {{ fontSize: 13 }} }},
|
||
yAxis: {{ type: 'value', name: '\\u7ed3\\u6784\\u5316\\u77ed\\u8bed\\u6570' }},
|
||
series: [{{
|
||
type: 'bar', barWidth: '45%',
|
||
data: [
|
||
{{ value: sentRaw.positive || 0, itemStyle: {{ color: '#16a34a' }} }},
|
||
{{ value: sentRaw.negative || 0, itemStyle: {{ color: '#dc2626' }} }},
|
||
{{ value: sentRaw.neutral || 0, itemStyle: {{ color: '#6366f1' }} }}
|
||
],
|
||
label: {{ show: true, position: 'top', fontWeight: 'bold' }}
|
||
}}]
|
||
}});
|
||
|
||
/* --- 需求 Top-N 横向 Bar --- */
|
||
var painData = {pain_json};
|
||
if (painData.length) {{
|
||
initChart('pain-top-chart', {{
|
||
tooltip: {{ trigger: 'axis', axisPointer: {{ type: 'shadow' }} }},
|
||
grid: {{ left: '35%', right: '8%', top: 10, bottom: 20 }},
|
||
yAxis: {{ type: 'category', data: painData.map(function(d){{ return d.name; }}).reverse(),
|
||
axisLabel: {{ width: 200, overflow: 'truncate', fontSize: 12 }} }},
|
||
xAxis: {{ type: 'value', name: '\\u77ed\\u8bed\\u6570' }},
|
||
series: [{{ type: 'bar', data: painData.map(function(d){{ return d.value; }}).reverse(),
|
||
itemStyle: {{ color: '#3b82f6', borderRadius: [0,4,4,0] }},
|
||
label: {{ show: true, position: 'right' }} }}]
|
||
}});
|
||
}}
|
||
|
||
/* --- 负面反馈 Top-N 横向 Bar --- */
|
||
var negData = {neg_json};
|
||
if (negData.length) {{
|
||
initChart('negative-top-chart', {{
|
||
tooltip: {{ trigger: 'axis', axisPointer: {{ type: 'shadow' }} }},
|
||
grid: {{ left: '35%', right: '8%', top: 10, bottom: 20 }},
|
||
yAxis: {{ type: 'category', data: negData.map(function(d){{ return d.name; }}).reverse(),
|
||
axisLabel: {{ width: 200, overflow: 'truncate', fontSize: 12 }} }},
|
||
xAxis: {{ type: 'value', name: '\\u77ed\\u8bed\\u6570' }},
|
||
series: [{{ type: 'bar', data: negData.map(function(d){{ return d.value; }}).reverse(),
|
||
itemStyle: {{ color: '#dc2626', borderRadius: [0,4,4,0] }},
|
||
label: {{ show: true, position: 'right' }} }}]
|
||
}});
|
||
}}
|
||
|
||
/* --- 雷达图 --- */
|
||
var radarData = {radar_json};
|
||
if (radarData.length) {{
|
||
var maxVal = Math.max.apply(null, radarData.map(function(d){{ return d.value; }})) || 1;
|
||
initChart('radar-chart', {{
|
||
tooltip: {{}},
|
||
radar: {{
|
||
indicator: radarData.map(function(d) {{
|
||
return {{ name: d.name, max: Math.ceil(maxVal * 1.2) }};
|
||
}}),
|
||
shape: 'circle',
|
||
splitArea: {{ areaStyle: {{ color: ['rgba(37,99,235,0.02)','rgba(37,99,235,0.05)'] }} }}
|
||
}},
|
||
series: [{{
|
||
type: 'radar',
|
||
data: [{{ value: radarData.map(function(d){{ return d.value; }}),
|
||
areaStyle: {{ color: 'rgba(37,99,235,0.15)' }},
|
||
lineStyle: {{ color: '#3b82f6', width: 2 }},
|
||
itemStyle: {{ color: '#3b82f6' }} }}]
|
||
}}]
|
||
}});
|
||
}}
|
||
|
||
/* --- 词云 --- */
|
||
var wcData = {wc_json};
|
||
if (wcData.length) {{
|
||
initChart('wordcloud-chart', {{
|
||
tooltip: {{
|
||
show: true,
|
||
formatter: function(p) {{
|
||
var d = p.data || {{}};
|
||
var rank = d.rank ? ('#' + d.rank + ' ') : '';
|
||
var cnt = d.count != null ? d.count : p.value;
|
||
var zh = d.zh && d.zh !== d.name ? ('<br/>\\u4e2d\\u6587: ' + d.zh) : '';
|
||
var cat = (d.categories && d.categories.length)
|
||
? ('<br/>\\u5206\\u7c7b: ' + d.categories.join('\\u3001'))
|
||
: (d.category ? ('<br/>\\u5206\\u7c7b: ' + d.category) : '<br/>\\u5206\\u7c7b: \\u672a\\u5206\\u7c7b');
|
||
return rank + (d.name || p.name) + zh + cat + '<br/>\\u6b21\\u6570: ' + cnt;
|
||
}}
|
||
}},
|
||
series: [{{
|
||
type: 'wordCloud', shape: 'circle', width: '95%', height: '95%',
|
||
sizeRange: [{WORDCLOUD_SIZE_MIN}, {WORDCLOUD_SIZE_MAX}],
|
||
rotationRange: [-15, 15], rotationStep: 15, gridSize: 8,
|
||
drawOutOfBound: false, layoutAnimation: true,
|
||
textStyle: {{ fontFamily: 'system-ui, -apple-system, sans-serif' }},
|
||
emphasis: {{ focus: 'self', textStyle: {{ shadowBlur: 6, shadowColor: '#333' }} }},
|
||
data: wcData
|
||
}}]
|
||
}});
|
||
}}
|
||
|
||
/* --- 词频分页 --- */
|
||
var rows = Array.from(document.querySelectorAll('.wf-row'));
|
||
var tabs = Array.from(document.querySelectorAll('.wf-tab'));
|
||
var label = document.getElementById('wf-page-label');
|
||
var pageCount = tabs.length || 1;
|
||
var cur = 1;
|
||
function showPage(n) {{
|
||
cur = Math.max(1, Math.min(n, pageCount));
|
||
rows.forEach(function(r) {{
|
||
r.classList.toggle('wf-visible', parseInt(r.dataset.page, 10) === cur);
|
||
}});
|
||
tabs.forEach(function(t) {{
|
||
t.classList.toggle('active', parseInt(t.dataset.page, 10) === cur);
|
||
}});
|
||
if (label) label.textContent = '\\u7b2c ' + cur + ' / ' + pageCount + ' \\u9875';
|
||
}}
|
||
tabs.forEach(function(t) {{
|
||
t.addEventListener('click', function() {{ showPage(parseInt(t.dataset.page, 10)); }});
|
||
}});
|
||
var prev = document.getElementById('wf-prev');
|
||
var next = document.getElementById('wf-next');
|
||
if (prev) prev.addEventListener('click', function() {{ showPage(cur - 1); }});
|
||
if (next) next.addEventListener('click', function() {{ showPage(cur + 1); }});
|
||
showPage(1);
|
||
|
||
/* --- 导航高亮 --- */
|
||
var navLinks = document.querySelectorAll('.report-nav a');
|
||
var sectionIds = Array.from(navLinks).map(function(a) {{ return a.getAttribute('href').slice(1); }});
|
||
function updateNav() {{
|
||
var scrollY = window.scrollY + 80;
|
||
var active = sectionIds[0];
|
||
sectionIds.forEach(function(id) {{
|
||
var el = document.getElementById(id);
|
||
if (el && el.offsetTop <= scrollY) active = id;
|
||
}});
|
||
navLinks.forEach(function(a) {{
|
||
a.classList.toggle('active', a.getAttribute('href') === '#' + active);
|
||
}});
|
||
}}
|
||
window.addEventListener('scroll', updateNav);
|
||
updateNav();
|
||
}})();
|
||
</script>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
def generate_report(
|
||
*,
|
||
product_name: str,
|
||
industry: str,
|
||
cleaned_csv: Path,
|
||
cluster_db: Path,
|
||
embed_db: Path,
|
||
word_freq_csv: Path,
|
||
output_html: Path,
|
||
structured_db: Path = STRUCTURED_DB,
|
||
min_cluster_review_ratio: float | None = None,
|
||
save_llm_raw: bool = False,
|
||
llm_raw_path: Path | None = None,
|
||
) -> dict:
|
||
sync_voc_report_constants(sys.modules[__name__])
|
||
api_key = require_chat_api_key()
|
||
|
||
cleaned_csv = cleaned_csv.resolve()
|
||
word_freq = _load_word_freq(
|
||
word_freq_csv,
|
||
max(WORDCLOUD_TOP_N, WORD_FREQ_TABLE_N, WORD_CATEGORY_CLASSIFY_N),
|
||
)
|
||
total_reviews, bundles = build_cluster_bundles(
|
||
cluster_db=cluster_db,
|
||
embed_db=embed_db,
|
||
cleaned_csv=cleaned_csv,
|
||
min_cluster_review_ratio=min_cluster_review_ratio,
|
||
)
|
||
top2_audience = _load_step2_top2_audience(cluster_db)
|
||
if not top2_audience:
|
||
top2_audience = _ordered_step2_audiences(bundles, [])[:2]
|
||
logger.info("阶段二 top2 受众簇: %s", top2_audience)
|
||
|
||
logger.info(
|
||
"调用 LLM 生成分析报告（%s，reasoning_effort=%s）…",
|
||
REPORT_MODEL,
|
||
REPORT_REASONING_EFFORT,
|
||
)
|
||
system, user = _build_report_prompt(
|
||
product_name=product_name,
|
||
industry=industry,
|
||
total_reviews=total_reviews,
|
||
bundles=bundles,
|
||
word_freq=word_freq,
|
||
top2_audience_clusters=top2_audience,
|
||
)
|
||
raw = _fetch_report_llm_raw_with_retry(
|
||
system=system,
|
||
user=user,
|
||
api_key=api_key,
|
||
expect_cluster_names=len(bundles) > 0,
|
||
)
|
||
saved_raw_path: str | None = None
|
||
if save_llm_raw:
|
||
raw_path = (llm_raw_path or output_html.parent / "report_llm_raw.txt").resolve()
|
||
raw_path.parent.mkdir(parents=True, exist_ok=True)
|
||
raw_path.write_text(raw, encoding="utf-8")
|
||
saved_raw_path = str(raw_path)
|
||
logger.info("已保存 LLM 原文 %s", raw_path)
|
||
|
||
report_html, word_zh, category_data, cluster_names = _parse_report_response(raw)
|
||
category_data = _classify_remaining_words_in_pool(
|
||
category_data,
|
||
word_freq,
|
||
industry=industry,
|
||
product_name=product_name,
|
||
api_key=api_key,
|
||
)
|
||
category_data = _fill_category_analysis(
|
||
category_data,
|
||
word_freq,
|
||
industry=industry,
|
||
product_name=product_name,
|
||
api_key=api_key,
|
||
)
|
||
_apply_cluster_names(bundles, cluster_names)
|
||
|
||
appendix_bundles = _bundles_for_phrase_appendix(bundles)
|
||
all_phrases: List[str] = []
|
||
for b in appendix_bundles:
|
||
all_phrases.extend(b.embed_texts)
|
||
zh_map = _translate_unique_phrases(all_phrases, api_key)
|
||
_fill_phrase_translations(appendix_bundles, zh_map)
|
||
|
||
wc_data = _wordcloud_data(word_freq, word_zh, category_data)
|
||
wordfreq_sec = _render_wordfreq_section(word_freq, word_zh, category_data)
|
||
phrases_sec = _render_phrase_sections(appendix_bundles)
|
||
struct_samples = _build_structured_appendix_samples(
|
||
structured_db=structured_db.resolve(),
|
||
cleaned_csv=cleaned_csv,
|
||
)
|
||
structured_sec = _render_structured_validation_section(struct_samples)
|
||
|
||
aud_pie = _audience_pie_data(bundles)
|
||
sent_bar = _sentiment_bar_data(bundles)
|
||
pain_top = _pain_top_chart_data(bundles, top_n=10)
|
||
neg_top = _negative_top_chart_data(bundles, top_n=10)
|
||
radar = _radar_chart_data(category_data, word_freq)
|
||
kpis = _compute_dashboard_kpis(total_reviews, bundles)
|
||
|
||
html = _assemble_html(
|
||
product_name=product_name,
|
||
wordfreq_html=wordfreq_sec,
|
||
report_html=report_html,
|
||
phrases_html=phrases_sec,
|
||
structured_html=structured_sec,
|
||
wordcloud_data=wc_data,
|
||
audience_pie_data=aud_pie,
|
||
sentiment_data=sent_bar,
|
||
pain_top_data=pain_top,
|
||
negative_top_data=neg_top,
|
||
radar_data=radar,
|
||
kpis=kpis,
|
||
)
|
||
output_html.parent.mkdir(parents=True, exist_ok=True)
|
||
output_html.write_text(html, encoding="utf-8")
|
||
logger.info("已写入 %s", output_html)
|
||
return {
|
||
"report_html": str(output_html),
|
||
"total_reviews": total_reviews,
|
||
"cluster_groups": len(bundles),
|
||
"min_cluster_review_ratio": min_cluster_review_ratio,
|
||
"structured_samples": len(struct_samples),
|
||
"llm_raw_path": saved_raw_path,
|
||
}
|