c91e7f8c20
移除结构化 audience 字段，强化 voc_业务_2 源评论归因匹配与 Persona 引用展示，更新 README 与流水线默认清理 SQLite。 Co-authored-by: Cursor <cursoragent@cursor.com>
461 lines
17 KiB
Python
461 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
LLM 分析模块：调用 DeepSeek LLM 完成 Persona/KANO/JTBD/根因分析。
|
||
复用父目录 voc_llm.py 的 API Key 和 Client 配置。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import sys
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Tuple, TypeVar
|
||
|
||
T = TypeVar("T")
|
||
|
||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
if str(_PROJECT_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||
|
||
from voc_llm import CHAT_MODEL, chat_extra_body, create_chat_client, require_chat_api_key
|
||
from prompt_loader import llm_params, optional_block, system_prompt, user_prompt
|
||
|
||
logger = logging.getLogger("voc.llm_analyzer")
|
||
|
||
DEFAULT_MODEL = CHAT_MODEL
|
||
REPORT_MODEL = "deepseek-v4-pro"
|
||
REPORT_REASONING_EFFORT = "max"
|
||
DEFAULT_LLM_WORKERS = 4
|
||
|
||
# 报告构建模式（build_report.py 调用 configure_report_llm 后启用 max thinking）
|
||
_report_mode = False
|
||
_report_model = REPORT_MODEL
|
||
_report_reasoning_effort = REPORT_REASONING_EFFORT
|
||
REPORT_MAX_TOKENS = 128000
|
||
REPORT_TIMEOUT = 900.0
|
||
|
||
|
||
def configure_report_llm(cfg: Optional[dict] = None) -> None:
|
||
"""启用报告 LLM：deepseek-v4-pro + reasoning_effort=max。"""
|
||
global _report_mode, _report_model, _report_reasoning_effort, REPORT_MAX_TOKENS
|
||
cfg = cfg or {}
|
||
_report_mode = True
|
||
_report_model = (cfg.get("report_model") or REPORT_MODEL).strip() or REPORT_MODEL
|
||
_report_reasoning_effort = (cfg.get("report_reasoning_effort") or REPORT_REASONING_EFFORT).strip() or "max"
|
||
if cfg.get("report_max_tokens"):
|
||
REPORT_MAX_TOKENS = max(8000, int(cfg["report_max_tokens"]))
|
||
logger.info(
|
||
"报告 LLM 已配置: model=%s, reasoning_effort=%s, max_tokens=%s",
|
||
_report_model, _report_reasoning_effort, REPORT_MAX_TOKENS,
|
||
)
|
||
|
||
|
||
def get_llm_workers(cfg: Optional[dict] = None) -> int:
|
||
"""LLM 并发线程数（config llm_max_workers 或环境变量 VOC_LLM_WORKERS）。"""
|
||
if cfg and cfg.get("llm_max_workers"):
|
||
return max(1, int(cfg["llm_max_workers"]))
|
||
env = os.environ.get("VOC_LLM_WORKERS")
|
||
if env:
|
||
return max(1, int(env))
|
||
return DEFAULT_LLM_WORKERS
|
||
|
||
|
||
def _strip_think(text: str) -> str:
|
||
if not text:
|
||
return text
|
||
text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE)
|
||
text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
|
||
return text.strip()
|
||
|
||
|
||
def _call_llm(system, user, *, model=None, temperature=0.3, max_tokens=16000, timeout=300.0, reasoning=None):
|
||
api_key = require_chat_api_key()
|
||
use_report = _report_mode if reasoning is None else reasoning
|
||
use_model = model or (_report_model if use_report else DEFAULT_MODEL)
|
||
effective_max = max(max_tokens, REPORT_MAX_TOKENS) if use_report else max_tokens
|
||
effective_timeout = max(timeout, REPORT_TIMEOUT) if use_report else timeout
|
||
client = create_chat_client(api_key=api_key, timeout=effective_timeout)
|
||
kwargs = {"model": use_model, "messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
], "max_tokens": effective_max}
|
||
if use_report:
|
||
kwargs["reasoning_effort"] = _report_reasoning_effort
|
||
kwargs["extra_body"] = {"thinking": {"type": "enabled"}}
|
||
else:
|
||
kwargs["temperature"] = temperature
|
||
eb = chat_extra_body(use_model)
|
||
if eb:
|
||
kwargs["extra_body"] = eb
|
||
resp = client.chat.completions.create(**kwargs)
|
||
msg = resp.choices[0].message
|
||
content = msg.content or ""
|
||
if not content.strip():
|
||
content = getattr(msg, "reasoning_content", None) or ""
|
||
if use_report and not content.strip():
|
||
fr = getattr(resp.choices[0], "finish_reason", None)
|
||
raise RuntimeError(
|
||
f"报告 LLM content 为空（model={use_model}，finish_reason={fr!r}，max_tokens={effective_max}）；"
|
||
"请增大 report_max_tokens 或降低 reasoning_effort"
|
||
)
|
||
return _strip_think(content)
|
||
|
||
|
||
def _parse_json(text):
|
||
try:
|
||
return json.loads(text)
|
||
except json.JSONDecodeError:
|
||
pass
|
||
m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(1).strip())
|
||
except json.JSONDecodeError:
|
||
pass
|
||
m = re.search(r"\{[\s\S]*\}", text)
|
||
if m:
|
||
try:
|
||
return json.loads(m.group(0))
|
||
except json.JSONDecodeError:
|
||
pass
|
||
logger.warning("无法解析 JSON: %s...", text[:200])
|
||
return {}
|
||
|
||
|
||
def _j(obj: Any) -> str:
|
||
return json.dumps(obj, ensure_ascii=False, indent=2)
|
||
|
||
|
||
# ── Persona ──
|
||
|
||
def build_persona_prompt(cluster_data, product_name: str = "", industry: str = ""):
|
||
catalog = cluster_data.get("persona_cluster_catalog") or []
|
||
return user_prompt(
|
||
"persona",
|
||
product_name=product_name or "主产品",
|
||
industry=industry or "当前品类",
|
||
catalog_json=_j(catalog),
|
||
audience_clusters_json=_j(cluster_data.get("audience_clusters", [])),
|
||
global_pains_json=_j(cluster_data.get("global_pains", [])),
|
||
global_negative_json=_j(cluster_data.get("global_negative", [])),
|
||
global_positive_json=_j(cluster_data.get("global_positive", [])),
|
||
)
|
||
|
||
|
||
def discover_personas(
|
||
cluster_data,
|
||
product_name: str = "",
|
||
industry: str = "",
|
||
):
|
||
logger.info("LLM: Persona发现（产品=%s）...", product_name or "主产品")
|
||
params = llm_params("persona")
|
||
raw = _call_llm(
|
||
system_prompt("persona"),
|
||
build_persona_prompt(cluster_data, product_name, industry),
|
||
**params,
|
||
)
|
||
result = _parse_json(raw)
|
||
personas = result.get("personas", [])
|
||
logger.info("发现 %s 个Persona", len(personas))
|
||
return personas
|
||
|
||
|
||
# ── 主题 ──
|
||
|
||
def discover_themes(cluster_data, neg_review_count=0, theme_type="negative"):
|
||
logger.info("LLM: 发现%s主题...", theme_type)
|
||
data_key = "global_negative" if theme_type == "negative" else "global_positive"
|
||
extra = ""
|
||
if theme_type == "negative":
|
||
extra = optional_block(
|
||
"theme", "negative_extra",
|
||
neg_review_count=neg_review_count,
|
||
p0_threshold=int(neg_review_count * 0.2),
|
||
)
|
||
user = user_prompt(
|
||
"theme",
|
||
theme_type=theme_type,
|
||
extra_block=extra,
|
||
cluster_data_json=_j(cluster_data.get(data_key, [])),
|
||
)
|
||
params = llm_params("theme")
|
||
raw = _call_llm(system_prompt("theme"), user, **params)
|
||
result = _parse_json(raw)
|
||
themes = result.get("themes", [])
|
||
logger.info("发现 %s 个%s主题", len(themes), theme_type)
|
||
return themes
|
||
|
||
|
||
def discover_themes_both(
|
||
cluster_data: Dict[str, Any],
|
||
neg_review_count: int,
|
||
pos_review_count: int,
|
||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||
"""并发发现差评/好评主题。"""
|
||
logger.info("LLM: 并发发现差评+好评主题...")
|
||
with ThreadPoolExecutor(max_workers=2) as ex:
|
||
f_neg = ex.submit(discover_themes, cluster_data, neg_review_count, "negative")
|
||
f_pos = ex.submit(discover_themes, cluster_data, pos_review_count, "positive")
|
||
return f_neg.result(), f_pos.result()
|
||
|
||
|
||
def analyze_kano_jtbd_keywords_parallel(
|
||
neg_themes: List[Dict[str, Any]],
|
||
pos_themes: List[Dict[str, Any]],
|
||
personas: List[Dict[str, Any]],
|
||
neg_keyword_groups: List[Dict[str, Any]],
|
||
pos_keyword_groups: List[Dict[str, Any]],
|
||
product_name: str = "",
|
||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], Dict[str, Any]]:
|
||
"""并发执行 KANO、JTBD、情感关键词（三者互不依赖）。"""
|
||
logger.info("LLM: 并发 KANO + JTBD + 情感关键词...")
|
||
with ThreadPoolExecutor(max_workers=3) as ex:
|
||
f_kano = ex.submit(analyze_kano, neg_themes, pos_themes, personas)
|
||
f_jtbd = ex.submit(analyze_jtbd, personas)
|
||
f_kw = ex.submit(
|
||
analyze_keywords, neg_keyword_groups, pos_keyword_groups, personas, product_name,
|
||
)
|
||
return f_kano.result(), f_jtbd.result(), f_kw.result()
|
||
|
||
|
||
def analyze_kano(neg_themes, pos_themes, personas):
|
||
logger.info("LLM: KANO分析...")
|
||
reverse_block = optional_block("kano", "reverse_search_block")
|
||
user = user_prompt(
|
||
"kano",
|
||
reverse_search_block=reverse_block,
|
||
neg_themes_json=_j(neg_themes),
|
||
pos_themes_json=_j(pos_themes),
|
||
personas_json=_j(personas),
|
||
)
|
||
params = llm_params("kano")
|
||
raw = _call_llm(system_prompt("kano"), user, **params)
|
||
result = _parse_json(raw)
|
||
kano = result.get("kano", [])
|
||
logger.info("KANO: %s条", len(kano))
|
||
return kano
|
||
|
||
|
||
# ── JTBD ──
|
||
|
||
def analyze_jtbd(personas):
|
||
logger.info("LLM: JTBD分析...")
|
||
user = user_prompt(
|
||
"jtbd",
|
||
persona_count=len(personas),
|
||
personas_json=_j(personas),
|
||
)
|
||
params = llm_params("jtbd")
|
||
raw = _call_llm(system_prompt("jtbd"), user, **params)
|
||
result = _parse_json(raw)
|
||
jtbd = result.get("jtbd", [])
|
||
logger.info("JTBD: %s条", len(jtbd))
|
||
return jtbd
|
||
|
||
|
||
# ── 矩阵 ──
|
||
|
||
def analyze_matrix(personas, kano, cluster_data, total_reviews, market_avg: float = 0.0):
|
||
_ = cluster_data
|
||
logger.info("LLM: 矩阵分析...")
|
||
market_hint = ""
|
||
if market_avg and market_avg < 3.5:
|
||
market_hint = optional_block("matrix", "market_low_hint", market_avg=market_avg)
|
||
top_personas = personas[:4]
|
||
user = user_prompt(
|
||
"matrix",
|
||
market_hint=market_hint,
|
||
top_personas_json=_j(top_personas),
|
||
kano_json=_j(kano),
|
||
total_reviews=total_reviews,
|
||
market_avg=market_avg,
|
||
)
|
||
params = llm_params("matrix")
|
||
raw = _call_llm(system_prompt("matrix"), user, **params)
|
||
result = _parse_json(raw)
|
||
matrix = result.get("matrix", [])
|
||
logger.info("矩阵: %s行", len(matrix))
|
||
return matrix
|
||
|
||
|
||
# ── 根因 ──
|
||
|
||
def analyze_rootcause_per_persona(
|
||
persona,
|
||
neg_themes,
|
||
bound_clusters: Optional[Dict[str, Any]] = None,
|
||
product_name: str = "",
|
||
industry: str = "",
|
||
):
|
||
name = persona.get("name", "?")
|
||
theme_names = [t.get("name") for t in neg_themes if t.get("name")]
|
||
product_ctx = product_name or "主产品"
|
||
industry_ctx = industry or "当前品类"
|
||
logger.info("LLM: 根因-%s...", name)
|
||
# 构建 Persona 自身绑定簇的摘要数据（替代旧 per_aud_data）
|
||
bc_data = bound_clusters or {}
|
||
user = user_prompt(
|
||
"rootcause",
|
||
persona_name=name,
|
||
product_name=product_ctx,
|
||
industry=industry_ctx,
|
||
persona_json=_j(persona),
|
||
bound_clusters_json=_j(bc_data),
|
||
theme_names_json=_j(theme_names),
|
||
)
|
||
params = llm_params("rootcause")
|
||
raw = _call_llm(system_prompt("rootcause"), user, **params)
|
||
result = _parse_json(raw)
|
||
return result
|
||
|
||
|
||
def analyze_all_rootcauses(
|
||
personas: List[Dict[str, Any]],
|
||
neg_themes: List[Dict[str, Any]],
|
||
all_clusters: Optional[Dict[str, List[Any]]] = None,
|
||
max_workers: int = DEFAULT_LLM_WORKERS,
|
||
product_name: str = "",
|
||
industry: str = "",
|
||
min_hit_count: int = 5,
|
||
) -> List[Dict[str, Any]]:
|
||
"""并发按 Persona 根因分析；跳过命中不足的 Persona。不再依赖 per_audience。"""
|
||
|
||
def _bound_clusters_for(persona: Dict[str, Any]) -> Dict[str, Any]:
|
||
"""从 Persona 自身 cluster_refs 构建绑定簇的摘要数据。"""
|
||
refs = persona.get("cluster_refs") or {}
|
||
bc_data: Dict[str, Any] = {}
|
||
if isinstance(refs, dict):
|
||
for dim, ref in refs.items():
|
||
if isinstance(ref, dict) and ref.get("stage") and ref.get("label") is not None:
|
||
stage = ref["stage"]
|
||
label = ref["label"]
|
||
# 尝试从 all_clusters 中查找实际簇数据
|
||
if all_clusters and stage in all_clusters:
|
||
for c in all_clusters[stage]:
|
||
if int(getattr(c, "cluster_label", -1)) == int(label):
|
||
bc_data[f"{dim}_dim"] = {
|
||
"stage": stage,
|
||
"label": label,
|
||
"top_phrases": getattr(c, "top_phrases", [])[:10],
|
||
}
|
||
break
|
||
return bc_data
|
||
|
||
eligible = [
|
||
(i, p) for i, p in enumerate(personas)
|
||
if p.get("hit_count", 0) >= min_hit_count
|
||
]
|
||
n = len(eligible)
|
||
if n == 0:
|
||
logger.warning("无 Persona 达到根因分析命中阈值（min_hit_count=%s）", min_hit_count)
|
||
return [
|
||
{"persona_name": p.get("name", f"P{i}"), "persona_index": i,
|
||
"root_causes": [], "affected_themes": [], "skipped": True,
|
||
"skip_reason": f"聚类命中 {p.get('hit_count', 0)} 条，低于阈值 {min_hit_count}"}
|
||
for i, p in enumerate(personas)
|
||
]
|
||
|
||
workers = max(1, min(max_workers, n))
|
||
logger.info("LLM: 并发根因分析 %s/%s 个 Persona（workers=%s）...", n, len(personas), workers)
|
||
|
||
def _one(idx: int, persona: Dict[str, Any]) -> Dict[str, Any]:
|
||
bc_data = _bound_clusters_for(persona)
|
||
rc = analyze_rootcause_per_persona(
|
||
persona, neg_themes,
|
||
bound_clusters=bc_data,
|
||
product_name=product_name, industry=industry,
|
||
)
|
||
rc["persona_name"] = persona.get("name", f"P{idx}")
|
||
rc["persona_index"] = idx
|
||
rc["skipped"] = False
|
||
return rc
|
||
|
||
results_map: Dict[int, Dict[str, Any]] = {}
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
futures = {ex.submit(_one, i, p): (i, p) for i, p in eligible}
|
||
for fut in as_completed(futures):
|
||
i, _ = futures[fut]
|
||
results_map[i] = fut.result()
|
||
|
||
ordered: List[Dict[str, Any]] = []
|
||
for i, p in enumerate(personas):
|
||
if i in results_map:
|
||
ordered.append(results_map[i])
|
||
else:
|
||
ordered.append({
|
||
"persona_name": p.get("name", f"P{i}"),
|
||
"persona_index": i,
|
||
"root_causes": [],
|
||
"affected_themes": [],
|
||
"skipped": True,
|
||
"skip_reason": f"聚类命中 {p.get('hit_count', 0)} 条，低于阈值 {min_hit_count}",
|
||
})
|
||
return ordered
|
||
|
||
|
||
# ── 情感关键词 ──
|
||
|
||
def analyze_keywords(
|
||
neg_groups: List[Dict[str, Any]],
|
||
pos_groups: List[Dict[str, Any]],
|
||
personas,
|
||
product_name: str = "",
|
||
):
|
||
logger.info("LLM: 情感关键词（差评+好评词组）...")
|
||
skip_hint = "product/item/the/and 及品类核心词"
|
||
if product_name:
|
||
skip_hint += f"；当前产品「{product_name}」相关词"
|
||
user = user_prompt(
|
||
"keyword",
|
||
neg_groups_json=_j(neg_groups),
|
||
pos_groups_json=_j(pos_groups),
|
||
personas_json=_j(personas),
|
||
skip_hint=skip_hint,
|
||
)
|
||
params = llm_params("keyword")
|
||
raw = _call_llm(system_prompt("keyword"), user, **params)
|
||
result = _parse_json(raw)
|
||
if "negative" in result or "positive" in result:
|
||
logger.info("情感词: 差评 %s 组 / 好评 %s 组", len(result.get("negative", [])), len(result.get("positive", [])))
|
||
return result
|
||
legacy = result.get("keywords", [])
|
||
logger.info("情感词(legacy): %s条", len(legacy))
|
||
return {"negative": legacy, "positive": []}
|
||
|
||
|
||
|
||
# ── 产品/行业自动识别 ──
|
||
|
||
def detect_product_and_industry(sample_reviews: list, dir_name: str = "") -> tuple[str, str]:
|
||
"""从样本评论和目录名中自动识别产品名和行业。返回 (product_name, industry)。"""
|
||
logger.info("LLM: 识别产品/行业...")
|
||
sample_text = "\n---\n".join(
|
||
f"[{i+1}] {r[:300]}" for i, r in enumerate(sample_reviews[:50])
|
||
)
|
||
dir_info = ""
|
||
if dir_name:
|
||
dir_info = f'\n## 数据来源目录名\n> {dir_name}\n（目录名可能包含产品/品类关键词）\n'
|
||
user = user_prompt("product_detect", sample_text=sample_text, dir_info=dir_info)
|
||
params = llm_params("product_detect")
|
||
raw = _call_llm(system_prompt("product_detect"), user, **params, reasoning=False)
|
||
result = _parse_json(raw)
|
||
product = result.get("product_name", "").strip()
|
||
industry = result.get("industry", "亚马逊电商").strip()
|
||
logger.info("识别结果: 产品=%s | 行业=%s", product, industry)
|
||
return product or "亚马逊商品", industry or "亚马逊电商"
|
||
|
||
# ── 市场竞争 ──
|
||
|
||
def market_competition_judgment(weighted_avg):
|
||
if weighted_avg < 3.5:
|
||
return ("系统性缺陷 · 新品进入窗口期",
|
||
f"加权均分 {weighted_avg}（低于 3.5），按方法论判定为「市场存在严重系统性缺陷，是新品进入的明确窗口期」。")
|
||
elif weighted_avg <= 4.0:
|
||
return ("有改进空间",
|
||
f"加权均分 {weighted_avg}（3.5–4.0 区间），属于「市场有改进空间，部分功能存在普遍短板」。")
|
||
else:
|
||
return ("市场成熟",
|
||
f"加权均分 {weighted_avg}（高于 4.0），属于「市场整体较成熟，需通过差异化或细分切入」。")
|