"""
基于聚类结果与词频 CSV 生成 output/voc_report.html（词云 + 词频 + AI 报告 + 各簇表述）。

用法::
  python3 main_voc分析.py 
  python3 main_voc分析.py --only-step 7 --industry "..." --product "..."
  # 仅纳入本 stage 内评论占比 ≥10% 的簇：
  python3 main_voc分析.py --only-step 7 --filter-small-clusters ...
  # 调试：保存报告 LLM 完整原文到 output/report_llm_raw.txt
  python3 main_voc分析.py --only-step 7 --save-llm-raw   --industry "..." --product "..."
"""
from __future__ import annotations

import csv
import html
import json
import logging
import os
import re
import sqlite3
import sys
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Sequence, Set, Tuple

from openai import OpenAI

from prompts.loader import (
    build_category_analysis_prompts,
    build_report_analysis_requirements,
    build_report_correction_message,
    build_report_json_markers,
    build_report_output_format,
    build_report_system,
    build_word_assign_prompts,
    sync_voc_report_constants,
)

logger = logging.getLogger("voc_report")

PROJECT_ROOT = Path(__file__).resolve().parent
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
STRUCTURED_APPENDIX_SAMPLE_N = 15
DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
MODEL_NAME = os.environ.get("DASHSCOPE_MODEL", "qwen3.6-flash").strip()

WORDCLOUD_TOP_N = 180
WORD_FREQ_TABLE_N = 180  # 页面词频表、排名展示上限
WORD_CATEGORY_CLASSIFY_N = 180  # LLM 词频分类词表范围（≥ 词表展示数）
WORD_FREQ_PAGE_SIZE = 30
WORD_FREQ_PAGES = 6
# 词云字号映射：0.5 次幂缓解「超高频词过大、长尾过小」
WORDCLOUD_SIZE_POWER = 0.5
WORDCLOUD_SIZE_MIN = 14
WORDCLOUD_SIZE_MAX = 96
MAX_PHRASES_PER_CLUSTER = 15
# 启用簇筛选时的默认阈值（本 stage 内去重评论占比）
DEFAULT_CLUSTER_MIN_REVIEW_RATIO = 0.10
OUTLIER_LABEL_ZH = "未归类"

_STAGE_3B_TITLES: Dict[str, str] = {
    "3b_aspect_opinion_positive": "全量正面产品反馈",
    "3b_aspect_opinion_negative": "全量负面产品反馈",
    "3b_aspect_opinion_neutral": "全量产品客观描述",
}
_STAGE_2B_BLOCK_TITLE: Dict[str, str] = {
    "positive": "正面产品反馈",
    "negative": "负面产品反馈",
    "neutral": "产品客观描述",
}
# 词频分类展示名迁移（兼容旧 LLM 输出）
_WORD_CATEGORY_DISPLAY_ALIASES: Dict[str, str] = {
    "痛点/场景": "需求/场景",
}
_STAGE_SORT_KEY: Dict[str, int] = {
    "3a_pain_global": 0,
    "3b_aspect_opinion_positive": 1,
    "3b_aspect_opinion_negative": 2,
    "3b_aspect_opinion_neutral": 3,
}
# 与 聚类.py 流程一致的 stage 前缀（用于模版与映射说明）
STAGE_1_AUDIENCE = "1_audience"
STAGE_2A_PAIN_PREFIX = "2a_pain_audience_c"
STAGE_2B_AO_PREFIX = "2b_aspect_opinion_"
STAGE_3A_PAIN = "3a_pain_global"
STAGE_3B_NEGATIVE = "3b_aspect_opinion_negative"
STAGE_3B_AO_PREFIX = "3b_aspect_opinion_"
# 底部「聚类效果验证」固定展示的 stage（写死，不含 -1 离群簇）
PHRASE_APPENDIX_STAGES: Tuple[str, ...] = (STAGE_3A_PAIN, STAGE_3B_NEGATIVE)
STEP2_TOP2_META_STAGE = "step2_filter"
_AUDIENCE_RANK_ZH: Tuple[str, ...] = ("第一", "第二")

WORD_CATEGORIES: Tuple[str, ...] = (
    "成分/原料",
    "剂型",
    "受众/使用对象",
    "功效/功能",
    "需求/场景",
    "品质/体验",
    "价格/价值",
    "物流/包装",
)

# 词云按分类配色（与 WORD_CATEGORIES 顺序一一对应）
CATEGORY_COLORS: Dict[str, str] = {
    "成分/原料": "#dc2626",
    "剂型": "#db2777",
    "受众/使用对象": "#2563eb",
    "功效/功能": "#16a34a",
    "需求/场景": "#ea580c",
    "品质/体验": "#7c3aed",
    "价格/价值": "#ca8a04",
    "物流/包装": "#0891b2",
}
WORDCLOUD_UNCATEGORIZED_COLOR = "#9ca3af"

TRANSLATE_CHARS_PER_REQUEST = 200_000
TRANSLATE_MAX_OUTPUT_TOKENS = 65536
REPORT_MAX_OUTPUT_TOKENS = 65536
TRANSLATE_LLM_TIMEOUT_SEC = 600.0
REPORT_LLM_TIMEOUT_SEC = 600.0
REPORT_PARSE_MAX_RETRIES = 2
# JSON 段放前、HTML 放后：输出被 max_tokens 截断时优先保留可解析的 JSON
REPORT_MARKERS: Tuple[str, ...] = (
    "===WORD_ZH_JSON===",
    "===WORD_CATEGORY_JSON===",
    "===CLUSTER_NAMES_JSON===",
    "===REPORT_HTML===",
)
WORD_CATEGORY_ASSIGN_BATCH_SIZE = 40
WORD_CATEGORY_ASSIGN_MAX_TOKENS = 8192
WORD_CATEGORY_ANALYSIS_MAX_WORDS = 25  # 生成 analysis 时每类最多展示的词条数
WORD_CATEGORY_ANALYSIS_MAX_TOKENS = 4096


@dataclass
class ClusterBundle:
    section: str
    stage: str
    stage_title_zh: str
    cluster_label: int
    cluster_title_zh: str
    anchor_id: str
    review_count: int
    phrase_count: int
    ratio: float  # 占本 stage 去重评论比（小簇过滤用）
    ratio_global: float  # 短语数 / 全部评论 a（报告洞察用）
    embed_texts: List[str] = field(default_factory=list)
    embed_texts_zh: List[str] = field(default_factory=list)


def _resolve_api_key() -> str:
    v = os.environ.get("DASHSCOPE_API_KEY", "").strip()
    if v:
        return v
    fp = os.environ.get("DASHSCOPE_API_KEY_FILE", "").strip()
    if fp:
        p = Path(fp).expanduser()
        if p.is_file():
            return p.read_text(encoding="utf-8").strip().strip('"').strip("'")
    local = PROJECT_ROOT / ".dashscope_key"
    if local.is_file():
        return local.read_text(encoding="utf-8").strip().strip('"').strip("'")
    return ""


def _strip_think(text: str) -> str:
    if not text:
        return text
    text = re.sub(
        r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
    )
    text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
    return text.strip()


def _call_llm_messages(
    messages: Sequence[Dict[str, str]],
    api_key: str,
    *,
    temperature: float = 0.3,
    max_tokens: int = 16384,
    timeout: float = 300.0,
) -> str:
    client = OpenAI(
        api_key=api_key, base_url=DASHSCOPE_BASE_URL, timeout=timeout
    )
    extra_body: Dict[str, Any] = {}
    if MODEL_NAME.lower().startswith(("qwen3.6", "qwen3.5", "qwen3")):
        extra_body["enable_thinking"] = False
    resp = client.chat.completions.create(
        model=MODEL_NAME,
        messages=list(messages),
        temperature=temperature,
        max_tokens=max_tokens,
        **({"extra_body": extra_body} if extra_body else {}),
    )
    msg = resp.choices[0].message
    text = msg.content or getattr(msg, "reasoning_content", None) or ""
    if not text.strip():
        raise RuntimeError("LLM 返回为空")
    return _strip_think(text)


def _call_llm(
    system: str,
    user: str,
    api_key: str,
    *,
    temperature: float = 0.3,
    max_tokens: int = 16384,
    timeout: float = 300.0,
) -> str:
    return _call_llm_messages(
        [
            {"role": "system", "content": system},
            {"role": "user", "content": user},
        ],
        api_key,
        temperature=temperature,
        max_tokens=max_tokens,
        timeout=timeout,
    )


def _load_review_count(csv_path: Path) -> int:
    n = 0
    with csv_path.open(encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f)
        for row in reader:
            if (row.get("content") or "").strip():
                n += 1
    return n


def _load_review_texts_by_source_row(csv_path: Path) -> Dict[int, str]:
    """与 结构化_server.load_reviews_from_file 一致：第 1 条数据行 source_row=1。"""
    out: Dict[int, str] = {}
    with csv_path.open(encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f)
        for i, row in enumerate(reader, start=1):
            text = (row.get("content") or "").strip()
            if text:
                out[i] = text
    return out


def _latest_structured_job_id(conn: sqlite3.Connection) -> int:
    row = conn.execute(
        "SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1"
    ).fetchone()
    if not row:
        raise RuntimeError("voc_structured.sqlite 中无结构化任务")
    return int(row[0])


def _load_structured_samples(
    structured_db: Path,
    *,
    job_id: int | None = None,
    limit: int = STRUCTURED_APPENDIX_SAMPLE_N,
) -> List[Tuple[int, Dict[str, Any]]]:
    conn = sqlite3.connect(structured_db)
    try:
        jid = job_id if job_id is not None else _latest_structured_job_id(conn)
        cur = conn.execute(
            """
            SELECT source_row, extraction_json
            FROM comment_extractions
            WHERE job_id = ?
            ORDER BY source_row
            LIMIT ?
            """,
            (jid, limit),
        )
        return [(int(sr), json.loads(js)) for sr, js in cur.fetchall()]
    finally:
        conn.close()


def _extraction_to_display_zh(ext: Dict[str, Any]) -> Dict[str, Any]:
    """将库内英文字段转为报告展示用中文键（与业务阅读一致）。"""
    feedback: List[Dict[str, str]] = []
    for item in ext.get("product_feedback") or []:
        if not isinstance(item, dict):
            continue
        feedback.append(
            {
                "方面": str(item.get("aspect", "")).strip(),
                "观点": str(item.get("opinion", "")).strip(),
                "态度": str(item.get("sentiment", "")).strip(),
                "类别标签": str(item.get("category", "")).strip(),
            }
        )
    pains = ext.get("pain_points") or []
    if not isinstance(pains, list):
        pains = []
    return {
        "受众": str(ext.get("audience", "")).strip(),
        "需求/痛点": [str(p).strip() for p in pains if str(p).strip()],
        "产品反馈": feedback,
    }


def _build_structured_appendix_samples(
    *,
    structured_db: Path,
    cleaned_csv: Path,
    job_id: int | None = None,
    limit: int = STRUCTURED_APPENDIX_SAMPLE_N,
) -> List[Tuple[int, str, Dict[str, Any]]]:
    if not structured_db.is_file():
        logger.warning("未找到结构化库 %s，跳过结构化提取效果验证", structured_db)
        return []
    texts = _load_review_texts_by_source_row(cleaned_csv)
    try:
        rows = _load_structured_samples(structured_db, job_id=job_id, limit=limit)
    except (RuntimeError, sqlite3.Error) as e:
        logger.warning("读取结构化样本失败: %s", e)
        return []
    out: List[Tuple[int, str, Dict[str, Any]]] = []
    for sr, ext in rows:
        content = texts.get(sr, "").strip() or "（原文缺失）"
        out.append((sr, content, _extraction_to_display_zh(ext)))
    return out


def _load_word_freq(path: Path, top_n: int) -> List[Tuple[str, int]]:
    rows: List[Tuple[str, int]] = []
    with path.open(encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f)
        for row in reader:
            w = (row.get("word") or "").strip()
            if not w:
                continue
            rows.append((w, int(row.get("count") or 0)))
    rows.sort(key=lambda x: x[1], reverse=True)
    return rows[:top_n]


def _stage_section(stage: str) -> str:
    if stage == "1_audience" or stage.startswith("2a_") or stage.startswith("2b_"):
        return "part1"
    return "part2"


def _audience_cluster_from_stage(stage: str) -> int | None:
    if stage.startswith("2a_pain_audience_c"):
        try:
            return int(stage.split("_c")[-1])
        except ValueError:
            return None
    m = re.match(
        r"2b_aspect_opinion_(?:positive|negative|neutral)_audience_c(\d+)$",
        stage,
    )
    if m:
        return int(m.group(1))
    return None


def _sentiment_suffix_from_2b_stage(stage: str) -> str | None:
    m = re.match(
        r"2b_aspect_opinion_(positive|negative|neutral)_audience_c\d+$",
        stage,
    )
    return m.group(1) if m else None


def _stage_title_zh(stage: str, audience_names: Dict[int, str]) -> str:
    if stage == "1_audience":
        return "受众画像聚类（阶段一）"
    if stage in _STAGE_3B_TITLES:
        return _STAGE_3B_TITLES[stage]
    aud_c = _audience_cluster_from_stage(stage)
    if aud_c is not None:
        aud_name = audience_names.get(aud_c, f"受众{aud_c}")
        if stage.startswith("2a_"):
            return f"{aud_name} · 用户需求（阶段二）"
        sent = _sentiment_suffix_from_2b_stage(stage)
        if sent:
            label = _STAGE_2B_BLOCK_TITLE.get(sent, sent)
            return f"{aud_name} · {label}（阶段二）"
        return f"{aud_name} · 产品反馈（阶段二）"
    if stage == "3a_pain_global":
        return "全量用户需求（阶段三）"
    return stage


def _phrase_appendix_stage_title(stage: str) -> str:
    if stage == STAGE_3A_PAIN:
        return "全量用户需求（阶段三）"
    if stage in _STAGE_3B_TITLES:
        return _STAGE_3B_TITLES[stage]
    return stage


def _bundles_for_phrase_appendix(bundles: List[ClusterBundle]) -> List[ClusterBundle]:
    return [
        b
        for b in bundles
        if b.stage in PHRASE_APPENDIX_STAGES and b.cluster_label != -1
    ]


def _bundle_sort_key(b: ClusterBundle) -> Tuple[int, int, str, int]:
    sec = 0 if b.section == "part1" else 1
    stage_ord = _STAGE_SORT_KEY.get(b.stage, 0)
    lab = -1 if b.cluster_label == -1 else b.cluster_label
    return (sec, stage_ord, b.stage, lab)


def _cluster_key(stage: str, label: int) -> str:
    return f"{stage}|{label}"


def _anchor_id(stage: str, label: int) -> str:
    safe = re.sub(r"[^a-zA-Z0-9_-]", "_", stage)
    return f"cluster-{safe}-{label}"


def _latest_cluster_run(conn: sqlite3.Connection) -> Tuple[int, int]:
    row = conn.execute(
        "SELECT id, job_id FROM cluster_runs ORDER BY id DESC LIMIT 1"
    ).fetchone()
    if not row:
        raise RuntimeError("voc_clustering.sqlite 中无聚类记录")
    return int(row[0]), int(row[1])


def _load_assignments(
    cconn: sqlite3.Connection, run_id: int
) -> List[Tuple[str, int, int, str, int]]:
    cur = cconn.execute(
        """
        SELECT stage, cluster_label, source_row, embed_text, embedding_item_id
        FROM cluster_assignments
        WHERE run_id = ?
        ORDER BY stage, cluster_label, source_row
        """,
        (run_id,),
    )
    return [
        (str(s), int(lab), int(sr), str(et), int(eid))
        for s, lab, sr, et, eid in cur.fetchall()
    ]


def build_cluster_bundles(
    *,
    cluster_db: Path,
    cleaned_csv: Path,
    embed_db: Path | None = None,  # 保留参数兼容 VOC分析，不再使用
    min_cluster_review_ratio: float | None = None,
) -> Tuple[int, List[ClusterBundle]]:
    del embed_db
    total_reviews = _load_review_count(cleaned_csv)

    cconn = sqlite3.connect(cluster_db)
    try:
        run_id, _ = _latest_cluster_run(cconn)
        raw = _load_assignments(cconn, run_id)
    finally:
        cconn.close()

    grouped: Dict[Tuple[str, int], List[Tuple[int, str, int]]] = defaultdict(list)
    texts_by_key: Dict[Tuple[str, int], List[str]] = defaultdict(list)
    stage_source_rows: Dict[str, set[int]] = defaultdict(set)

    for stage, lab, sr, et, _eid in raw:
        key = (stage, lab)
        grouped[key].append((sr, et, _eid))
        texts_by_key[key].append(et)
        stage_source_rows[stage].add(sr)

    bundles: List[ClusterBundle] = []
    for (stage, lab), items in sorted(grouped.items(), key=lambda x: (x[0][0], x[0][1])):
        sec = _stage_section(stage)
        if sec not in ("part1", "part2"):
            continue
        unique_srs = sorted({sr for sr, _, _ in items})
        cnt = len(unique_srs)
        stage_total = len(stage_source_rows.get(stage, set()))
        phrase_cnt = len(items)
        stage_ratio = cnt / stage_total if stage_total else 0.0
        ratio = stage_ratio
        ratio_global = phrase_cnt / total_reviews if total_reviews else 0.0
        if (
            min_cluster_review_ratio is not None
            and stage_total > 0
            and stage_ratio < min_cluster_review_ratio
        ):
            logger.info(
                "报告跳过小簇 %s / %s：%s 条评论 (本步骤 %.1f%% < %.0f%%)",
                stage,
                lab,
                cnt,
                stage_ratio * 100,
                min_cluster_review_ratio * 100,
            )
            continue
        text_ctr = Counter(texts_by_key[(stage, lab)])
        reps = [t for t, _ in text_ctr.most_common(MAX_PHRASES_PER_CLUSTER)]

        bundles.append(
            ClusterBundle(
                section=sec,
                stage=stage,
                stage_title_zh=stage,
                cluster_label=lab,
                cluster_title_zh=str(lab),
                anchor_id=_anchor_id(stage, lab),
                review_count=cnt,
                phrase_count=phrase_cnt,
                ratio=ratio,
                ratio_global=ratio_global,
                embed_texts=reps,
            )
        )

    bundles.sort(key=_bundle_sort_key)
    return total_reviews, bundles


def _apply_cluster_names(
    bundles: List[ClusterBundle], names: Dict[str, str]
) -> None:
    audience_names: Dict[int, str] = {}
    for b in bundles:
        if b.stage != "1_audience":
            continue
        if b.cluster_label == -1:
            audience_names[-1] = OUTLIER_LABEL_ZH
            b.cluster_title_zh = OUTLIER_LABEL_ZH
            continue
        key = _cluster_key(b.stage, b.cluster_label)
        title = names.get(key, "").strip() or f"受众群体 {b.cluster_label}"
        b.cluster_title_zh = title
        audience_names[b.cluster_label] = title

    for b in bundles:
        b.stage_title_zh = _stage_title_zh(b.stage, audience_names)
        if b.stage == "1_audience":
            continue
        if b.cluster_label == -1:
            aud_c = _audience_cluster_from_stage(b.stage)
            if aud_c is not None:
                aud = audience_names.get(aud_c, f"受众{aud_c}")
                b.cluster_title_zh = f"{aud} · {OUTLIER_LABEL_ZH}"
            else:
                b.cluster_title_zh = OUTLIER_LABEL_ZH
            continue
        key = _cluster_key(b.stage, b.cluster_label)
        sub = names.get(key, "").strip()
        aud_c = _audience_cluster_from_stage(b.stage)
        if aud_c is not None:
            aud = audience_names.get(aud_c, f"受众{aud_c}")
            b.cluster_title_zh = f"{aud} · {sub}" if sub else f"{aud} · 子簇 {b.cluster_label}"
        else:
            b.cluster_title_zh = sub or f"主题簇 {b.cluster_label}"


def _parse_numbered_translations(
    raw: str, count: int, *, originals: Sequence[str]
) -> List[str]:
    out: List[str] = []
    for i in range(count):
        m = re.search(rf"\[{i + 1}\]\s*([\s\S]*?)(?=\n\[{i + 2}\]|\Z)", raw)
        out.append(m.group(1).strip() if m else originals[i])
    while len(out) < count:
        j = len(out)
        out.append(originals[j] if j < len(originals) else "")
    return out[:count]


def _chunk_texts_for_translation(texts: List[str]) -> List[List[str]]:
    batches: List[List[str]] = []
    current: List[str] = []
    size = 0
    for text in texts:
        block_size = len(text) + 32
        if current and size + block_size > TRANSLATE_CHARS_PER_REQUEST:
            batches.append(current)
            current = []
            size = 0
        current.append(text)
        size += block_size
    if current:
        batches.append(current)
    return batches


def _translate_unique_phrases(
    phrases: List[str], api_key: str
) -> Dict[str, str]:
    unique: List[str] = []
    seen: set[str] = set()
    for p in phrases:
        t = p.strip()
        if not t or t in seen:
            continue
        seen.add(t)
        unique.append(t)
    if not unique:
        return {}

    merged: Dict[str, str] = {}
    batches = _chunk_texts_for_translation(unique)
    for bi, batch in enumerate(batches, start=1):
        logger.info(
            "批量翻译短语（第 %s/%s 批）：%s 条",
            bi,
            len(batches),
            len(batch),
        )
        body = "\n\n".join(f"[{i + 1}]\n{t}" for i, t in enumerate(batch))
        user = (
            f"将以下 {len(batch)} 条英文 VOC 结构化短语逐条译为流畅简体中文。\n"
            "输出格式：仍用 [1]、[2]… 编号；不要解释、不要 Markdown。\n\n"
            + body
        )
        raw = _call_llm(
            "你是专业英中翻译。按编号输出全部译文，不要遗漏。",
            user,
            api_key,
            temperature=0.2,
            max_tokens=TRANSLATE_MAX_OUTPUT_TOKENS,
            timeout=TRANSLATE_LLM_TIMEOUT_SEC,
        )
        zh_list = _parse_numbered_translations(raw, len(batch), originals=batch)
        for en, zh in zip(batch, zh_list):
            merged[en] = zh
    return merged


def _fill_phrase_translations(
    bundles: List[ClusterBundle], zh_map: Dict[str, str]
) -> None:
    for b in bundles:
        b.embed_texts_zh = [zh_map.get(t, t) for t in b.embed_texts]


def _bundles_for_prompt(bundles: List[ClusterBundle], section: str) -> str:
    """按短语数降序，供 LLM 按规模分析各簇。"""
    subset = [b for b in bundles if b.section == section]
    subset.sort(key=lambda b: (-b.phrase_count, b.stage, b.cluster_label))
    lines: List[str] = []
    for b in subset:
        pct_a = f"{b.ratio_global * 100:.1f}%"
        pct_stage = f"{b.ratio * 100:.1f}%"
        key = _cluster_key(b.stage, b.cluster_label)
        lines.append(
            f"- id={key}；阶段={b.stage}；簇标签={b.cluster_label}；"
            f"结构化短语数={b.phrase_count}（占全部评论 a 的 {pct_a}）；"
            f"去重评论数={b.review_count}（占本阶段 {pct_stage}）；"
            f"代表短语：{' | '.join(b.embed_texts)}"
        )
    return "\n".join(lines) if lines else "（无）"


def _audience_cluster_ids_from_bundles(bundles: Sequence[ClusterBundle]) -> List[int]:
    ids: set[int] = set()
    for b in bundles:
        aud = _audience_cluster_from_stage(b.stage)
        if aud is not None:
            ids.add(aud)
    return sorted(ids)


def _load_step2_top2_audience(cluster_db: Path) -> List[int]:
    """读取聚类阶段二入选的前两个受众簇标签（与 聚类.py _top2_audience_clusters 一致）。"""
    try:
        cconn = sqlite3.connect(cluster_db)
        try:
            run_id, _ = _latest_cluster_run(cconn)
            row = cconn.execute(
                """
                SELECT meta_json FROM cluster_stage_meta
                WHERE run_id = ? AND stage = ?
                """,
                (run_id, STEP2_TOP2_META_STAGE),
            ).fetchone()
        finally:
            cconn.close()
        if not row:
            return []
        meta = json.loads(str(row[0]))
        raw = meta.get("top2_audience_clusters")
        if not isinstance(raw, list):
            return []
        return [int(x) for x in raw]
    except (json.JSONDecodeError, TypeError, ValueError, sqlite3.Error) as e:
        logger.warning("读取 step2 top2 受众簇失败: %s", e)
        return []


def _ordered_step2_audiences(
    bundles: Sequence[ClusterBundle], top2_from_db: Sequence[int]
) -> List[int]:
    """阶段二模版顺序：优先 DB 中 top2 排名，再补齐 bundles 里出现的其它受众簇。"""
    in_bundles = set(_audience_cluster_ids_from_bundles(bundles))
    ordered: List[int] = []
    for aud in top2_from_db:
        if aud in in_bundles and aud not in ordered:
            ordered.append(aud)
    for aud in sorted(in_bundles):
        if aud not in ordered:
            ordered.append(aud)
    return ordered


def _audience_rank_title(rank_index: int, audience_cluster: int) -> str:
    del audience_cluster  # 仅用于调用方传入映射注释，勿出现在正文标题
    rank_zh = (
        _AUDIENCE_RANK_ZH[rank_index]
        if rank_index < len(_AUDIENCE_RANK_ZH)
        else f"第{rank_index + 1}"
    )
    return f"{rank_zh}大受众簇"


def _stages_in_bundles(bundles: Sequence[ClusterBundle]) -> List[str]:
    return sorted({b.stage for b in bundles})


def _cluster_stage_mapping_guide(
    bundles: Sequence[ClusterBundle],
    *,
    top2_audience_clusters: Sequence[int],
) -> str:
    """将本批次实际 stage 映射到 HTML 模版小节（与 聚类.py 五段流程一致）。"""
    stages = _stages_in_bundles(bundles)
    if not stages:
        return "（本批次无聚类簇数据）"
    aud_ids = _ordered_step2_audiences(bundles, top2_audience_clusters)
    top2_txt = (
        "、".join(str(a) for a in top2_audience_clusters)
        if top2_audience_clusters
        else "（见各 2a/2b stage 后缀）"
    )
    lines = [
        "阶段二入选规则（聚类.py）：1_audience 完成后，按各受众簇「去重评论数」降序，"
        f"仅对规模最大的前 2 个簇（本批次 top2 簇标签={top2_txt}）分别做：",
        "  · 2a：簇内 pain_point → 报告中称「用户需求」（每受众 1 个 stage）",
        "  · 2b：簇内 aspect_opinion → 正面/负面产品反馈、产品客观描述（每受众 3 个 stage）",
        "聚类 stage → HTML 模版小节（仅分析下列已出现的 stage；无数据的小节可省略 <li>）：",
        f"- {STAGE_1_AUDIENCE} → 二·（一）·1.受众画像 ·（1）受众群体特征 &（2）受众的主要需求",
    ]
    for i, aud in enumerate(aud_ids):
        rank = _audience_rank_title(i, aud)
        lines.append(
            f"- {STAGE_2A_PAIN_PREFIX}{aud} → 二·（一）·2.分受众分析 · {rank} ·（1）用户需求（2a）"
        )
        for suf, block_title in _STAGE_2B_BLOCK_TITLE.items():
            st = f"{STAGE_2B_AO_PREFIX}{suf}_audience_c{aud}"
            slot = {"positive": "（2）", "negative": "（3）", "neutral": "（4）"}[suf]
            sent_en = {"positive": "Positive", "negative": "Negative", "neutral": "Neutral"}[
                suf
            ]
            lines.append(
                f"- {st} → 二·（一）·2.分受众分析 · {rank} ·{slot}{block_title}（2b，sentiment={sent_en}）"
            )
    lines.append(f"- {STAGE_3A_PAIN} → 二·（二）·1.全部受众需求分析（3a）")
    for slot, suf, title in (
        ("2.", "positive", "正面产品反馈"),
        ("3.", "negative", "负面产品反馈"),
        ("4.", "neutral", "产品客观描述"),
    ):
        st = f"{STAGE_3B_AO_PREFIX}{suf}"
        marker = "（本批次未出现可省略）" if st not in stages else ""
        lines.append(f"- {st} → 二·（二）·{slot}{title}（3b）{marker}")
    lines.append(
        "- 占比口径：括号内 (XX.X%) = 该簇结构化短语数 ÷ a；"
        "同一 <ul> 内各 <li> 按短语数从高到低排列。"
    )
    lines.append(
        "⚠️ 【排版与格式极度重要警告】："
        "这是面向高管的业务报告，直接暴露原始机器 ID（如 1_audience|2、2a_pain_audience_c2|0）"
        "极其不专业且严重影响阅读！正文及任何列表标题中【绝对禁止】出现原始 ID、stage 名、"
        "簇标签数字；必须且只能使用提炼后的「纯中文业务簇名」。"
    )
    lines.append(
        "- 簇 -1 请统一命名并使用「未归类」或「离群反馈」；"
        "2a/2b 的 CLUSTER_NAMES JSON 中只写子主题，但在 HTML 正文排版时请自行补全为通顺纯中文"
        "（例如：犬类受众-消化不适需求、适口性），不得保留 audience_c2 等代码片段；"
        "2b 子主题禁止带「好评/差评/正面/负面」等情感词（情感由 stage 区分）。"
    )
    lines.append(
        "- 展示用语：全文只用「用户需求 / 产品反馈 / 产品客观描述」，禁止「观点 / 评价 / 痛点」等旧称。"
    )
    return "\n".join(lines)


def _html_list_placeholder(n: int = 2) -> str:
    items = "\n".join(
        "    <li><strong>[纯中文业务簇名，绝不许带原始ID]</strong> "
        "(XX.X%)：结合代表短语写深入的业务洞察。</li>"
        for _ in range(n)
    )
    return f"  <ul>\n{items}\n  </ul>"


def _html_step2_intro(top2_audience_clusters: Sequence[int]) -> str:
    top2_txt = (
        "、".join(str(a) for a in top2_audience_clusters[:2])
        if top2_audience_clusters
        else "见聚类库 step2_filter"
    )
    return (
        f"  <!-- 二·（一）·2.分受众分析 · top2 簇标签（禁止写入正文）：{top2_txt} -->\n"
        "  <p class=\"muted\">仅对评论量排名前 2 的受众进行分析。</p>"
    )


def _html_audience_step2_blocks(
    audience_ids: Sequence[int],
    *,
    top2_audience_clusters: Sequence[int],
) -> str:
    if not audience_ids:
        return (
            _html_step2_intro(top2_audience_clusters)
            + "\n  <p>（本批次无「2.分受众分析」聚类结果，请先运行 聚类.py）</p>\n"
            + _html_list_placeholder(1)
        )
    blocks: List[str] = [_html_step2_intro(top2_audience_clusters)]
    for i, aud in enumerate(audience_ids):
        rank_title = _audience_rank_title(i, aud)
        blocks.append(
            f"""  <!-- 二·（一）·2.分受众分析 · {rank_title} · 数据映射（勿写入正文）：{STAGE_2A_PAIN_PREFIX}{aud}；
       {STAGE_2B_AO_PREFIX}positive_audience_c{aud}；
       {STAGE_2B_AO_PREFIX}negative_audience_c{aud}；
       {STAGE_2B_AO_PREFIX}neutral_audience_c{aud} -->
  <p><strong>{rank_title}</strong>（正文请用「1.受众画像」CLUSTER_NAMES 中的纯中文受众名，勿写簇标签号）</p>
  <p><strong>（1）用户需求</strong></p>
{_html_list_placeholder(2)}
  <p><strong>（2）正面产品反馈</strong></p>
{_html_list_placeholder(2)}
  <p><strong>（3）负面产品反馈</strong></p>
{_html_list_placeholder(2)}
  <p><strong>（4）产品客观描述</strong></p>
{_html_list_placeholder(2)}"""
        )
    return "\n".join(blocks)


def _report_html_template(
    product_name: str,
    total_reviews: int,
    bundles: Sequence[ClusterBundle],
    *,
    top2_audience_clusters: Sequence[int],
) -> str:
    """规范化 HTML 报告骨架：摘要 → 评论分析（（一）受众 +（二）全量）→ 改进建议。"""
    audience_ids = _ordered_step2_audiences(bundles, top2_audience_clusters)
    step2_html = _html_audience_step2_blocks(
        audience_ids, top2_audience_clusters=top2_audience_clusters
    )
    return f"""<!-- 全局指令：严格遵守标准 HTML5。正文任何一处不得出现 1_audience|2、2a_pain_audience_c2|0 等原始聚类 ID 或 stage 名，全部替换为纯中文业务簇名。 -->
<html lang="zh-CN">
<head>
  <meta charset="UTF-8" />
  <title>{product_name}产品站内评论分析报告</title>
</head>
<body>
  <h1>{product_name}产品改进建议报告</h1>

  <h2>一、摘要</h2>
  <p>数据清洗后，共 {total_reviews} 条有效用户评论。</p>
  <ul>
    <li><strong>核心受众与场景：</strong>（说明文字，禁止出现原始 ID）</li>
    <li><strong>最核心的用户需求：</strong>（说明文字）</li>
    <li><strong>显著的产品反馈特征（正/负/客观）：</strong>（说明文字）</li>
    <li><strong>提炼核心改进建议与机会：</strong>（一句话概括方向）</li>
  </ul>

  <h2>二、{product_name}评论分析</h2>
  <p>统计口径：按结构化短语数占比（结构化短语数 ÷ 全部有效评论数）。</p>

  <h3>（一）受众画像与分析</h3>

  <!-- 数据映射：{STAGE_1_AUDIENCE}，勿写入正文 -->
  <h4>1.受众画像</h4>
  <p>（1）受众群体特征：</p>
{_html_list_placeholder(2)}
  <p>（2）受众的主要需求：</p>
{_html_list_placeholder(2)}

  <h4>2.分受众分析</h4>
{step2_html}

  <h3>（二）全部用户需求与产品反馈</h3>

  <!-- 数据映射：{STAGE_3A_PAIN} -->
  <h4>1.全部受众需求分析</h4>
{_html_list_placeholder(3)}

  <!-- 数据映射：3b_aspect_opinion_positive -->
  <h4>2.正面产品反馈</h4>
{_html_list_placeholder(3)}

  <!-- 数据映射：3b_aspect_opinion_negative -->
  <h4>3.负面产品反馈</h4>
{_html_list_placeholder(3)}

  <!-- 数据映射：3b_aspect_opinion_neutral -->
  <h4>4.产品客观描述</h4>
{_html_list_placeholder(3)}

  <h2>三、改进建议与机会</h2>
  <ol>
    <li><strong>配方与成分优化：</strong>（针对负面反馈与未满足需求的可执行建议）</li>
    <li><strong>包装与品控升级：</strong>（具体执行建议）</li>
    <li><strong>说明书/Listing优化：</strong>（具体执行建议）</li>
    <li><strong>市场与产品机会：</strong>（基于正面反馈与客观描述中的可放大卖点、人群或场景机会）</li>
  </ol>
</body>
</html>"""


def _normalize_report_html_fragment(html_text: str) -> str:
    """从完整 HTML 文档中提取可嵌入页面的 body 片段。"""
    text = html_text.strip()
    m = re.search(r"<body[^>]*>([\s\S]*?)</body>", text, flags=re.I)
    if m:
        return m.group(1).strip()
    if re.search(r"<!DOCTYPE|<html[\s>]", text, flags=re.I):
        text = re.sub(r"<!DOCTYPE[^>]*>", "", text, flags=re.I)
        text = re.sub(r"<head[^>]*>[\s\S]*?</head>", "", text, flags=re.I)
        text = re.sub(r"</?html[^>]*>", "", text, flags=re.I)
        return text.strip()
    return text


def _strip_report_top_heading(html_fragment: str) -> str:
    """去掉 LLM 报告正文开头的 h1，避免与页面级标题重复。"""
    return re.sub(
        r"^\s*<h1[^>]*>[\s\S]*?</h1>\s*",
        "",
        html_fragment,
        count=1,
        flags=re.I,
    )


def _build_report_prompt(
    *,
    product_name: str,
    industry: str,
    total_reviews: int,
    bundles: List[ClusterBundle],
    word_freq: List[Tuple[str, int]],
    top2_audience_clusters: Sequence[int],
) -> Tuple[str, str]:
    wf_table = word_freq[:WORD_FREQ_TABLE_N]
    wf_classify = word_freq[:WORD_CATEGORY_CLASSIFY_N]
    wf_table_lines = "\n".join(f"{w}\t{c}" for w, c in wf_table)
    wf_classify_lines = "\n".join(f"{w}\t{c}" for w, c in wf_classify)
    cats_literal = "、".join(WORD_CATEGORIES)
    stage_map = _cluster_stage_mapping_guide(
        bundles, top2_audience_clusters=top2_audience_clusters
    )
    html_tpl = _report_html_template(
        product_name,
        total_reviews,
        bundles,
        top2_audience_clusters=top2_audience_clusters,
    )
    system = build_report_system()
    user = f"""请基于以下用户评论反馈的聚类结果，撰写一份详细的《{product_name}》产品改进建议报告。

行业背景：{industry}
数据清洗后有效评论总数 a = {total_reviews}

【词频 Top{WORD_FREQ_TABLE_N}（word\\tcount，辅助理解品类与撰写报告，排名 1-{WORD_FREQ_TABLE_N}）】
{wf_table_lines}

【词频分类词表 Top{WORD_CATEGORY_CLASSIFY_N}（word\\tcount，**仅**用于 WORD_CATEGORY_JSON，排名 1-{WORD_CATEGORY_CLASSIFY_N}）】
{wf_classify_lines}

【聚类流程与报告小节映射】
{stage_map}

【聚类 · 二·（一）受众画像与分析（part1）】
- {STAGE_1_AUDIENCE} → 1.受众画像 ·（1）受众群体特征、（2）受众的主要需求
- top2 受众（本批次簇标签：{", ".join(str(x) for x in top2_audience_clusters) or "见 2a/2b stage 后缀"}）→ 2.分受众分析：
  2a 用户需求；2b 正面 / 负面 / 产品客观描述（Neutral 禁止写作「中性反馈」）
已按「结构化短语数」降序排列：
{_bundles_for_prompt(bundles, "part1")}

【聚类 · 二·（二）全部用户需求与产品反馈（part2）】
- {STAGE_3A_PAIN} → 1.全部受众需求分析
- 3b → 2.正面产品反馈 / 3.负面产品反馈 / 4.产品客观描述
已按「结构化短语数」降序排列：
{_bundles_for_prompt(bundles, "part2")}

{build_report_analysis_requirements(product_name=product_name, stage_1_audience=STAGE_1_AUDIENCE)}

{build_report_output_format()}

【HTML 结构模版（请替换括号占位为真实分析，可增删 <li>，保持层级）】
{html_tpl}

{build_report_json_markers(cats_literal=cats_literal)}
"""
    return system, user

def _marker_body(raw: str, marker: str) -> str:
    if marker not in raw:
        return ""
    part = raw.split(marker, 1)[1]
    for end_marker in REPORT_MARKERS:
        if end_marker != marker and end_marker in part:
            part = part.split(end_marker, 1)[0]
    part = part.strip()
    part = re.sub(r"^```(?:json)?\s*", "", part, flags=re.I)
    part = re.sub(r"\s*```\s*$", "", part)
    return part


def _json_block_slice(raw: str, marker: str) -> Tuple[str, str]:
    """返回 (marker 后正文, 用于 json.loads 的子串)。"""
    part = _marker_body(raw, marker)
    if not part:
        return part, ""
    start, end = part.find("{"), part.rfind("}")
    if start != -1 and end > start:
        return part, part[start : end + 1]
    if start != -1:
        return part, part[start:].strip()
    return part, ""


def _coerce_word_category_data(data: Dict[str, Any]) -> Dict[str, Any]:
    """将 LLM 各类别块统一为 {words: [...], analysis: str}。"""
    out: Dict[str, Any] = {}
    pending_analysis: str | None = None
    for key, val in data.items():
        if key == "analysis":
            if isinstance(val, str):
                pending_analysis = val
            continue
        if not isinstance(val, (dict, list)):
            continue
        if isinstance(val, list):
            out[key] = {"words": val, "analysis": ""}
        else:
            words = val.get("words")
            if words is None:
                words = val.get("word")
            out[key] = {
                "words": list(words) if isinstance(words, list) else [],
                "analysis": str(val.get("analysis") or ""),
            }
    if pending_analysis and len(out) == 1:
        only_key = next(iter(out))
        if not out[only_key].get("analysis"):
            out[only_key]["analysis"] = pending_analysis
    return out


def _trim_incomplete_json_tail(text: str) -> str:
    """去掉截断在引号/逗号上的尾部，便于补全括号。"""
    s = text.rstrip()
    while s.endswith(","):
        s = s[:-1].rstrip()
    m = re.search(r'(,\s*|\[\s*)\"[^\"\\]*$', s)
    if m:
        s = s[: m.start()].rstrip()
        if s.endswith(","):
            s = s[:-1].rstrip()
    return s


def _close_truncated_json_object(text: str) -> str:
    """为截断在数组/对象中间的 JSON 片段补全括号（尽力而为）。"""
    s = _close_truncated_json_braces(text)
    if s and '"analysis"' not in s:
        s = s.rstrip(", ") + ',"analysis":""'
        if s.count("{") > s.count("}"):
            s = s + "}"
    return s


def _repair_word_category_json(js: str) -> Dict[str, Any] | None:
    """
    修复 LLM 将各类别写成多段伪对象的情况，例如：
    {"成分/原料":[...],"analysis":"..."}, "受众/使用对象":[...], "analysis":"..."}, ...
    亦支持输出被截断、缺少最外层闭合括号的情形。
    """
    text = js.strip()
    if not text:
        return None
    if not text.startswith("{"):
        text = "{" + text
    parts = re.split(r"\}\s*,\s*(?=\")", text)
    merged: Dict[str, Any] = {}
    for i, part in enumerate(parts):
        chunk = part.strip().rstrip(",").strip()
        if not chunk:
            continue
        if i == 0:
            if not chunk.endswith("}"):
                chunk = _close_truncated_json_object(chunk)
        else:
            if not chunk.startswith("{"):
                chunk = "{" + chunk
            if not chunk.endswith("}"):
                chunk = _close_truncated_json_object(chunk)
        try:
            obj = json.loads(chunk)
        except json.JSONDecodeError:
            continue
        if not isinstance(obj, dict):
            continue
        merged.update(_coerce_word_category_data(obj))
    return merged or None


def _extract_word_category_blocks_regex(text: str) -> Dict[str, Any] | None:
    """从截断/脏文本中抽取完整的「类别 + words + analysis」块。"""
    if not text or "{" not in text:
        return None
    merged: Dict[str, Any] = {}
    cat_alt = "|".join(re.escape(c) for c in WORD_CATEGORIES)
    pattern = (
        rf'\{{\s*"({cat_alt})"\s*:\s*(\[[^\]]*\])\s*,\s*'
        r'"analysis"\s*:\s*"((?:[^"\\]|\\.)*)"\s*\}'
    )
    for m in re.finditer(pattern, text):
        cat, words_json, analysis = m.group(1), m.group(2), m.group(3)
        try:
            words = json.loads(words_json)
        except json.JSONDecodeError:
            continue
        if not isinstance(words, list):
            continue
        merged[cat] = {
            "words": [str(w) for w in words if str(w).strip()],
            "analysis": analysis.replace('\\"', '"'),
        }
    return merged or None


def _try_parse_word_category_json_block(raw: str) -> Tuple[Dict[str, Any], str | None]:
    marker = "===WORD_CATEGORY_JSON==="
    if marker not in raw:
        return {}, f"缺少标记 {marker}"
    part = _marker_body(raw, marker)
    _part, js = _json_block_slice(raw, marker)
    payload = js or (part[part.find("{") :] if "{" in part else part)
    data: Dict[str, Any] | None = None
    if payload:
        try:
            parsed = json.loads(payload)
            if isinstance(parsed, dict):
                data = _coerce_word_category_data(parsed)
        except json.JSONDecodeError:
            data = _repair_word_category_json(payload)
            if data:
                logger.info("已自动修复 %s 的非标准 JSON 结构", marker)
    if not data and payload:
        closed = _close_truncated_json_object(payload)
        if closed != payload:
            data = _repair_word_category_json(closed)
            if data:
                logger.info("已补全截断的 %s 并解析出 %s 类", marker, len(data))
    if not data and part:
        data = _extract_word_category_blocks_regex(part)
        if data:
            logger.info("已从 %s 截断文本中按类别块正则抽取 %s 类", marker, len(data))
    if not data:
        preview = (part or _part)[:200].replace("\n", " ")
        return (
            {},
            f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}",
        )
    return data, None


def _close_truncated_json_braces(text: str) -> str:
    """仅补全 [] / {{}}，不注入 analysis 等字段。"""
    s = _trim_incomplete_json_tail(text.strip())
    if not s:
        return s
    if not s.startswith("{"):
        s = "{" + s
    if s.count("[") > s.count("]"):
        s = s + "]"
    if s.count("{") > s.count("}"):
        s = s + "}"
    return s


def _extract_json_kv_regex(text: str) -> Dict[str, str]:
    """从截断的 JSON 对象文本中提取已完整的 key:value 字符串对。"""
    out: Dict[str, str] = {}
    for m in re.finditer(
        r'"((?:[^"\\]|\\.)+)"\s*:\s*"((?:[^"\\]|\\.)*)"',
        text,
    ):
        k = m.group(1).replace('\\"', '"')
        v = m.group(2).replace('\\"', '"')
        if k.strip() and v.strip():
            out[k] = v
    return out


def _loads_json_object_loose(payload: str) -> Tuple[Dict[str, Any] | None, str]:
    """
    解析单个 JSON 对象。兼容尾部多余 `}}`、Extra data、以及真实截断时的补全/正则抽取。
    返回 (dict, repair_note)；repair_note 非空时仅用于日志。
    """
    text = payload.strip()
    if not text or "{" not in text:
        return None, ""

    def _loads_once(s: str) -> Dict[str, Any] | None:
        try:
            obj = json.loads(s)
        except json.JSONDecodeError as e:
            if e.msg == "Extra data" or "Extra data" in e.msg:
                try:
                    obj, _idx = json.JSONDecoder().raw_decode(s)
                except json.JSONDecodeError:
                    return None
            else:
                return None
        return obj if isinstance(obj, dict) else None

    obj = _loads_once(text)
    if obj is not None:
        tail = text[text.rfind("}") + 1 :].strip()
        if tail:
            return obj, "ignore_trailing_garbage"
        return obj, ""

    trimmed = text
    while trimmed.endswith("}") and trimmed.count("{") < trimmed.count("}"):
        trimmed = trimmed[:-1].rstrip()
        obj = _loads_once(trimmed)
        if obj is not None:
            return obj, "trim_extra_brace"

    closed = _close_truncated_json_braces(text)
    if closed != text:
        obj = _loads_once(closed)
        if obj is not None:
            return obj, "close_truncated"

    kv = _extract_json_kv_regex(text)
    if kv:
        return kv, "regex_kv"
    return None, ""


def _try_parse_json_block(raw: str, marker: str) -> Tuple[Dict[str, Any], str | None]:
    if marker not in raw:
        return {}, f"缺少标记 {marker}"
    part = _marker_body(raw, marker)
    _part, js = _json_block_slice(raw, marker)
    payload = js or (part[part.find("{") :] if "{" in part else "")
    if not payload:
        preview = (part or _part)[:200].replace("\n", " ")
        return {}, f"{marker} 段内未找到合法 JSON 对象（无完整 {{...}}）；开头片段: {preview!r}"
    data, repair = _loads_json_object_loose(payload)
    if data is not None and repair:
        if repair == "regex_kv":
            logger.info("已从损坏的 %s 中 regex 抽取 %s 个键值对", marker, len(data))
        elif repair == "close_truncated":
            logger.info("已补全截断的 %s（%s 个键）", marker, len(data))
        elif repair in ("ignore_trailing_garbage", "trim_extra_brace"):
            logger.debug("已宽松解析 %s（%s 个键，%s）", marker, len(data), repair)
    if data is None:
        try:
            json.loads(payload)
        except json.JSONDecodeError as e:
            pos = e.pos if e.pos is not None else 0
            ctx = payload[max(0, pos - 50) : pos + 50]
            return (
                {},
                f"{marker} JSON 解析失败: {e.msg}（行{e.lineno}列{e.colno}）；"
                f"错误附近: ...{ctx!r}...",
            )
        return {}, f"{marker} 必须是 JSON 对象"
    if not isinstance(data, dict):
        return {}, f"{marker} 必须是 JSON 对象，实际为 {type(data).__name__}"
    return data, None


def _extract_json_block(raw: str, marker: str) -> Dict[str, Any]:
    data, err = _try_parse_json_block(raw, marker)
    if err:
        logger.warning("解析 %s 失败: %s", marker, err)
    return data


def _extract_report_html_fragment(raw: str) -> str:
    if "===REPORT_HTML===" not in raw:
        return ""
    part = raw.split("===REPORT_HTML===", 1)[1]
    for marker in REPORT_MARKERS[1:]:
        if marker in part:
            part = part.split(marker, 1)[0]
    report_html = part.strip()
    report_html = re.sub(r"```html?", "", report_html, flags=re.I)
    report_html = report_html.replace("```", "").strip()
    return _normalize_report_html_fragment(report_html)


def _validate_word_zh_json(data: Dict[str, Any], marker: str) -> List[str]:
    errs: List[str] = []
    if not data:
        errs.append(f"{marker} 为空对象")
        return errs
    for k, v in data.items():
        if not str(k).strip():
            errs.append(f"{marker} 含空键名")
        if not isinstance(v, str) or not str(v).strip():
            errs.append(f"{marker} 键 {k!r} 的值必须为非空字符串")
            break
    return errs


def _validate_word_category_json(data: Dict[str, Any], marker: str) -> List[str]:
    errs: List[str] = []
    if not data:
        errs.append(f"{marker} 为空对象")
        return errs
    has_any_words = False
    for cat, block in data.items():
        if not isinstance(block, dict):
            errs.append(f"{marker} 类别 {cat!r} 的值必须是对象")
            continue
        words = block.get("words")
        if words is not None:
            if not isinstance(words, list):
                errs.append(f"{marker} 类别 {cat!r}.words 必须是数组")
            elif words:
                has_any_words = True
        analysis = block.get("analysis")
        if analysis is not None and not isinstance(analysis, str):
            errs.append(f"{marker} 类别 {cat!r}.analysis 必须是字符串")
    if not has_any_words:
        errs.append(f"{marker} 所有类别的 words 均为空")
    return errs


def _validate_cluster_names_json(
    data: Dict[str, Any], marker: str, *, expect_keys: bool
) -> List[str]:
    errs: List[str] = []
    if expect_keys and not data:
        errs.append(f"{marker} 为空对象（需要为聚类簇命名）")
        return errs
    for k, v in data.items():
        if not str(k).strip():
            errs.append(f"{marker} 含空键名")
        if not isinstance(v, str) or not str(v).strip():
            errs.append(f"{marker} 键 {k!r} 的簇名必须为非空字符串")
            break
    return errs


def _dedupe_errors(errors: List[str]) -> List[str]:
    return list(dict.fromkeys(errors))


def _validate_report_response(
    raw: str, *, expect_cluster_names: bool
) -> List[str]:
    errors: List[str] = []
    for marker in REPORT_MARKERS:
        if marker not in raw:
            if marker == "===CLUSTER_NAMES_JSON===" and expect_cluster_names:
                logger.warning(
                    "缺少 %s（将使用程序默认簇名）；若频繁出现请检查模型 max_tokens 是否截断输出",
                    marker,
                )
            else:
                errors.append(f"缺少标记 {marker}")

    report_html = _extract_report_html_fragment(raw)
    if not report_html:
        errors.append("===REPORT_HTML=== 内容为空或无法提取 <body> 片段")
    elif "<h1" not in report_html.lower():
        errors.append("===REPORT_HTML=== 缺少 <h1> 标题")
    elif "<h2" not in report_html.lower():
        errors.append("===REPORT_HTML=== 缺少 <h2> 章节")

    wzh, err = _try_parse_json_block(raw, "===WORD_ZH_JSON===")
    if err:
        errors.append(err)
    else:
        errors.extend(_validate_word_zh_json(wzh, "===WORD_ZH_JSON==="))

    cats, err = _try_parse_word_category_json_block(raw)
    if err:
        errors.append(err)
    else:
        errors.extend(_validate_word_category_json(cats, "===WORD_CATEGORY_JSON==="))

    if "===CLUSTER_NAMES_JSON===" in raw:
        names, err = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===")
        if err:
            errors.append(err)
        else:
            errors.extend(
                _validate_cluster_names_json(
                    names,
                    "===CLUSTER_NAMES_JSON===",
                    expect_keys=expect_cluster_names and not names,
                )
            )
    return _dedupe_errors(errors)


def _build_report_correction_user_message(errors: List[str], raw: str) -> str:
    err_block = "\n".join(f"- {e}" for e in errors)
    max_chars = 100_000
    raw_body = raw if len(raw) <= max_chars else raw[:max_chars] + "\n\n...(上文已截断)..."
    cats_literal = "、".join(WORD_CATEGORIES)
    return build_report_correction_message(
        err_block=err_block,
        raw_body=raw_body,
        cats_literal=cats_literal,
    )


def _fetch_report_llm_raw_with_retry(
    *,
    system: str,
    user: str,
    api_key: str,
    expect_cluster_names: bool,
    max_retries: int = REPORT_PARSE_MAX_RETRIES,
) -> str:
    messages: List[Dict[str, str]] = [
        {"role": "system", "content": system},
        {"role": "user", "content": user},
    ]
    raw = _call_llm_messages(
        messages,
        api_key,
        max_tokens=REPORT_MAX_OUTPUT_TOKENS,
        timeout=REPORT_LLM_TIMEOUT_SEC,
    )
    for attempt in range(max_retries + 1):
        errors = _validate_report_response(raw, expect_cluster_names=expect_cluster_names)
        if not errors:
            if attempt > 0:
                logger.info("报告 LLM 输出校验通过（第 %s 次修正后）", attempt + 1)
            return raw
        if attempt >= max_retries:
            logger.warning(
                "报告解析校验仍失败（已重试 %s 次），继续使用最后一次输出: %s",
                max_retries,
                errors,
            )
            return raw
        logger.warning(
            "报告解析校验失败（第 %s/%s 次），请求 LLM 修正: %s",
            attempt + 1,
            max_retries,
            errors,
        )
        messages.append({"role": "assistant", "content": raw})
        messages.append(
            {
                "role": "user",
                "content": _build_report_correction_user_message(errors, raw),
            }
        )
        raw = _call_llm_messages(
            messages,
            api_key,
            max_tokens=REPORT_MAX_OUTPUT_TOKENS,
            timeout=REPORT_LLM_TIMEOUT_SEC,
        )
    return raw


def _parse_report_response(
    raw: str,
) -> Tuple[str, Dict[str, str], Dict[str, Any], Dict[str, str]]:
    report_html = _extract_report_html_fragment(raw)
    word_zh_raw, _ = _try_parse_json_block(raw, "===WORD_ZH_JSON===")
    word_zh = {str(k).lower(): str(v) for k, v in word_zh_raw.items()}
    categories, _ = _try_parse_word_category_json_block(raw)
    cluster_names_raw, _ = _try_parse_json_block(raw, "===CLUSTER_NAMES_JSON===")
    cluster_names = {str(k): str(v) for k, v in cluster_names_raw.items()}
    return report_html, word_zh, categories, cluster_names


def _zh_for_word(word: str, word_zh: Dict[str, str]) -> str:
    return word_zh.get(word.lower(), word_zh.get(word, word))


def _normalize_word_category_data(category_data: Dict[str, Any]) -> Dict[str, Any]:
    """兼容旧版词频分类名「痛点/场景」→「需求/场景」。"""
    if not category_data:
        return category_data
    out = dict(category_data)
    legacy = out.pop("痛点/场景", None)
    if legacy is not None and "需求/场景" not in out:
        out["需求/场景"] = legacy
    out.pop("其他", None)
    return out


def _ensure_word_category_skeleton(category_data: Dict[str, Any]) -> Dict[str, Any]:
    """保证七个类别键存在且值为 {{words, analysis}} 结构。"""
    base = _normalize_word_category_data(category_data or {})
    out: Dict[str, Any] = {}
    for cat in WORD_CATEGORIES:
        block = base.get(cat)
        if isinstance(block, dict):
            words = block.get("words") if isinstance(block.get("words"), list) else []
            analysis = str(block.get("analysis") or "")
        else:
            words, analysis = [], ""
        out[cat] = {"words": list(words), "analysis": analysis}
    return out


def _category_pool_count_map(
    word_freq: Sequence[Tuple[str, int]],
) -> Tuple[List[Tuple[str, int]], Dict[str, Tuple[str, int]], Set[str]]:
    top = list(word_freq[:WORD_CATEGORY_CLASSIFY_N])
    count_map = {w.lower(): (w, c) for w, c in top}
    keys = set(count_map.keys())
    return top, count_map, keys


def _valid_category_words_in_pool(
    block: Any,
    count_map: Dict[str, Tuple[str, int]],
    top_keys: Set[str],
) -> List[str]:
    if not isinstance(block, dict):
        return []
    out: List[str] = []
    seen: Set[str] = set()
    for w in block.get("words") or []:
        key = str(w).lower().strip()
        if not key or key in seen or key not in top_keys:
            continue
        en, cnt = count_map[key]
        if cnt <= 0:
            continue
        seen.add(key)
        out.append(en)
    return out


def _unclassified_pool_words(
    category_data: Dict[str, Any],
    word_freq: Sequence[Tuple[str, int]],
) -> List[str]:
    """Top{WORD_CATEGORY_CLASSIFY_N} 中尚未归入任何类别的词（按词频降序）。"""
    top, count_map, _top_keys = _category_pool_count_map(word_freq)
    classified = set(_build_word_category_map(category_data).keys())
    return [en for en, _ in top if en.lower() not in classified]


def _parse_word_assign_json(text: str) -> Dict[str, List[str]]:
    """解析逐词分类结果：{{"word": ["类别", ...], ...}}，空数组表示不分类。"""
    text = _strip_think(text).replace("```json", "").replace("```", "").strip()
    start, end = text.find("{"), text.rfind("}")
    if start == -1 or end <= start:
        return {}
    try:
        obj = json.loads(text[start : end + 1])
    except json.JSONDecodeError:
        obj, _ = _loads_json_object_loose(text[start : end + 1])
    if not isinstance(obj, dict):
        return {}
    out: Dict[str, List[str]] = {}
    for word, cats in obj.items():
        w = str(word).strip()
        if not w:
            continue
        if isinstance(cats, list):
            out[w] = [str(c).strip() for c in cats if str(c).strip()]
        elif isinstance(cats, str) and cats.strip():
            out[w] = [cats.strip()]
        else:
            out[w] = []
    return out


def _build_word_assign_prompt(
    *,
    industry: str,
    product_name: str,
    batch_words: Sequence[str],
    word_freq: Sequence[Tuple[str, int]],
) -> Tuple[str, str]:
    _top, count_map, _top_keys = _category_pool_count_map(word_freq)
    lines: List[str] = []
    for w in batch_words:
        key = w.lower()
        cnt = count_map.get(key, (w, 0))[1]
        lines.append(f"{w}\t{cnt}")
    cats_literal = "、".join(WORD_CATEGORIES)
    return build_word_assign_prompts(
        industry=industry,
        product_name=product_name,
        word_lines="\n".join(lines),
        cats_literal=cats_literal,
    )


def _merge_word_assignments(
    category_data: Dict[str, Any],
    assignments: Dict[str, List[str]],
    word_freq: Sequence[Tuple[str, int]],
) -> Dict[str, Any]:
    _top, count_map, top_keys = _category_pool_count_map(word_freq)
    out = _ensure_word_category_skeleton(category_data)
    cat_set = set(WORD_CATEGORIES)
    for word, cats in assignments.items():
        key = str(word).lower().strip()
        if not key or key not in top_keys:
            continue
        en, cnt = count_map[key]
        if cnt <= 0:
            continue
        valid_cats = [c for c in cats if c in cat_set]
        if not valid_cats:
            continue
        for cat in valid_cats:
            block = out[cat]
            existing = _valid_category_words_in_pool(block, count_map, top_keys)
            if any(e.lower() == key for e in existing):
                continue
            block["words"] = list(existing) + [en]
    return out


def _classify_remaining_words_in_pool(
    category_data: Dict[str, Any],
    word_freq: Sequence[Tuple[str, int]],
    *,
    industry: str,
    product_name: str,
    api_key: str,
) -> Dict[str, Any]:
    """对 Top{WORD_CATEGORY_CLASSIFY_N} 中尚未分类的词分批逐词尽量分类（可不归类）。"""
    data = _ensure_word_category_skeleton(category_data)
    pending = _unclassified_pool_words(data, word_freq)
    top_n = WORD_CATEGORY_CLASSIFY_N
    classified_n = top_n - len(pending)
    if not pending:
        logger.info(
            "词频分类：Top%s 已全部有类别（%s 个词）",
            top_n,
            classified_n,
        )
        return data

    logger.info(
        "词频分类：Top%s 已分类 %s 个，待逐词补充分类 %s 个（每批 %s）",
        top_n,
        classified_n,
        len(pending),
        WORD_CATEGORY_ASSIGN_BATCH_SIZE,
    )
    batches = [
        pending[i : i + WORD_CATEGORY_ASSIGN_BATCH_SIZE]
        for i in range(0, len(pending), WORD_CATEGORY_ASSIGN_BATCH_SIZE)
    ]
    for bi, batch in enumerate(batches, start=1):
        system, user = _build_word_assign_prompt(
            industry=industry,
            product_name=product_name,
            batch_words=batch,
            word_freq=word_freq,
        )
        raw = _call_llm_messages(
            [{"role": "system", "content": system}, {"role": "user", "content": user}],
            api_key,
            temperature=0.2,
            max_tokens=WORD_CATEGORY_ASSIGN_MAX_TOKENS,
            timeout=REPORT_LLM_TIMEOUT_SEC,
        )
        before_keys = set(_build_word_category_map(data).keys())
        assign = _parse_word_assign_json(raw)
        if assign:
            data = _merge_word_assignments(data, assign, word_freq)
        after_keys = set(_build_word_category_map(data).keys())
        newly = len(after_keys - before_keys)
        logger.info(
            "词频逐词分类 第 %s/%s 批：本批 %s 词，新归类 %s 个",
            bi,
            len(batches),
            len(batch),
            newly,
        )

    still = _unclassified_pool_words(data, word_freq)
    logger.info(
        "词频分类完成：Top%s 共归类 %s 个，未分类 %s 个（词云显示为未分类）",
        top_n,
        top_n - len(still),
        len(still),
    )
    return data


def _categories_needing_analysis(
    category_data: Dict[str, Any],
    word_freq: Sequence[Tuple[str, int]],
) -> List[str]:
    """有词条但 analysis 为空的类别。"""
    _top, count_map, top_keys = _category_pool_count_map(word_freq)
    data = _ensure_word_category_skeleton(category_data)
    need: List[str] = []
    for cat in WORD_CATEGORIES:
        words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys)
        if words and not str(data[cat].get("analysis") or "").strip():
            need.append(cat)
    return need


def _parse_category_analysis_json(text: str) -> Dict[str, str]:
    """解析 {{\"类别\": \"一段中文解读\", ...}}。"""
    text = _strip_think(text).replace("```json", "").replace("```", "").strip()
    start, end = text.find("{"), text.rfind("}")
    if start == -1 or end <= start:
        return {}
    try:
        obj = json.loads(text[start : end + 1])
    except json.JSONDecodeError:
        obj, _ = _loads_json_object_loose(text[start : end + 1])
    if not isinstance(obj, dict):
        return {}
    out: Dict[str, str] = {}
    for k, v in obj.items():
        cat = str(k).strip()
        if cat in WORD_CATEGORIES and isinstance(v, str) and v.strip():
            out[cat] = v.strip()
    return out


def _build_category_analysis_prompt(
    *,
    industry: str,
    product_name: str,
    word_freq: Sequence[Tuple[str, int]],
    category_data: Dict[str, Any],
    need_cats: Sequence[str],
) -> Tuple[str, str]:
    _top, count_map, top_keys = _category_pool_count_map(word_freq)
    data = _ensure_word_category_skeleton(category_data)
    lines: List[str] = []
    for cat in need_cats:
        words = _valid_category_words_in_pool(data.get(cat), count_map, top_keys)[
            :WORD_CATEGORY_ANALYSIS_MAX_WORDS
        ]
        parts = []
        for w in words:
            key = w.lower()
            cnt = count_map.get(key, (w, 0))[1]
            parts.append(f"{w}({cnt})")
        lines.append(f"- {cat}：{', '.join(parts) if parts else '（无）'}")
    cats_literal = "、".join(need_cats)
    return build_category_analysis_prompts(
        industry=industry,
        product_name=product_name,
        category_lines="\n".join(lines),
        cats_literal=cats_literal,
    )


def _merge_category_analysis(
    category_data: Dict[str, Any],
    analyses: Dict[str, str],
) -> Dict[str, Any]:
    out = _ensure_word_category_skeleton(category_data)
    for cat, text in analyses.items():
        if cat not in WORD_CATEGORIES:
            continue
        if text.strip() and not str(out[cat].get("analysis") or "").strip():
            out[cat]["analysis"] = text.strip()
    return out


def _fill_category_analysis(
    category_data: Dict[str, Any],
    word_freq: Sequence[Tuple[str, int]],
    *,
    industry: str,
    product_name: str,
    api_key: str,
) -> Dict[str, Any]:
    """为有词但缺少 analysis 的类别自动生成简短中文解读。"""
    data = _ensure_word_category_skeleton(category_data)
    need = _categories_needing_analysis(data, word_freq)
    if not need:
        return data

    logger.info("词频分类：为 %s 个类别补写 analysis: %s", len(need), need)
    system, user = _build_category_analysis_prompt(
        industry=industry,
        product_name=product_name,
        word_freq=word_freq,
        category_data=data,
        need_cats=need,
    )
    for attempt in range(2):
        raw = _call_llm_messages(
            [{"role": "system", "content": system}, {"role": "user", "content": user}],
            api_key,
            temperature=0.3,
            max_tokens=WORD_CATEGORY_ANALYSIS_MAX_TOKENS,
            timeout=REPORT_LLM_TIMEOUT_SEC,
        )
        parsed = _parse_category_analysis_json(raw)
        if parsed:
            data = _merge_category_analysis(data, parsed)
        still = _categories_needing_analysis(data, word_freq)
        if not still:
            logger.info("词频分类各类 analysis 已补全")
            return data
        need = still
        user = _build_category_analysis_prompt(
            industry=industry,
            product_name=product_name,
            word_freq=word_freq,
            category_data=data,
            need_cats=need,
        )[1]
        logger.warning(
            "词频 analysis 仍有 %s 类未生成（第 %s 次重试）: %s",
            len(still),
            attempt + 1,
            still,
        )

    logger.warning("词频分类 analysis 仍未补全: %s", still)
    return data


def _build_word_category_map(category_data: Dict[str, Any]) -> Dict[str, List[str]]:
    """英文词（小写）-> 所属分类列表（按 WORD_CATEGORIES 顺序，支持一词多类）。"""
    category_data = _normalize_word_category_data(category_data)
    by_word: Dict[str, List[str]] = defaultdict(list)
    for cat in WORD_CATEGORIES:
        block = category_data.get(cat)
        if not isinstance(block, dict):
            continue
        for w in block.get("words") or []:
            key = str(w).lower().strip()
            if key and cat not in by_word[key]:
                by_word[key].append(cat)
    return dict(by_word)


def _wordcloud_color_for_categories(categories: List[str]) -> str:
    if not categories:
        return WORDCLOUD_UNCATEGORIZED_COLOR
    return CATEGORY_COLORS.get(categories[0], WORDCLOUD_UNCATEGORIZED_COLOR)


def _render_wordcloud_legend() -> str:
    items = "".join(
        f'<span class="wc-legend-item">'
        f'<i class="wc-dot" style="background:{CATEGORY_COLORS[c]}"></i>'
        f"{html.escape(c)}</span>"
        for c in WORD_CATEGORIES
    )
    items += (
        f'<span class="wc-legend-item">'
        f'<i class="wc-dot" style="background:{WORDCLOUD_UNCATEGORIZED_COLOR}"></i>'
        f"未分类</span>"
    )
    return f'<div class="wc-legend">{items}</div>'


def _wordcloud_data(
    word_freq: List[Tuple[str, int]],
    word_zh: Dict[str, str],
    category_data: Dict[str, Any],
) -> List[dict]:
    items = word_freq[:WORDCLOUD_TOP_N]
    if not items:
        return []
    word_cats = _build_word_category_map(category_data)
    counts = [c for _, c in items]
    c_min, c_max = min(counts), max(counts)
    span = c_max - c_min
    out: List[dict] = []
    for rank, (word, count) in enumerate(items, start=1):
        norm = 1.0 if span <= 0 else (count - c_min) / span
        value = max(1, int((norm**WORDCLOUD_SIZE_POWER) * 1000))
        zh = _zh_for_word(word, word_zh)
        cats = word_cats.get(word.lower(), [])
        color = _wordcloud_color_for_categories(cats)
        out.append(
            {
                "name": word,
                "value": value,
                "count": count,
                "rank": rank,
                "en": word,
                "zh": zh,
                "category": cats[0] if cats else "",
                "categories": cats,
                "textStyle": {"color": color},
            }
        )
    return out


def _freq_rank_map(word_freq: List[Tuple[str, int]], n: int) -> Dict[str, int]:
    return {w.lower(): i + 1 for i, (w, _) in enumerate(word_freq[:n])}


def _category_top_words(
    category_data: Dict[str, Any],
    word_freq: List[Tuple[str, int]],
    *,
    top_n: int = 10,
) -> Dict[str, List[Tuple[str, int, str]]]:
    category_data = _normalize_word_category_data(category_data)
    count_map = {w.lower(): (w, c) for w, c in word_freq}
    rank_map = _freq_rank_map(word_freq, WORD_CATEGORY_CLASSIFY_N)
    out: Dict[str, List[Tuple[str, int, str]]] = {}
    for cat in WORD_CATEGORIES:
        block = category_data.get(cat)
        if not isinstance(block, dict):
            out[cat] = []
            continue
        words = block.get("words") or []
        rows: List[Tuple[str, int, str]] = []
        seen: set[str] = set()
        for w in words:
            key = str(w).lower().strip()
            if not key or key in seen:
                continue
            seen.add(key)
            if key in count_map:
                en, cnt = count_map[key]
            else:
                en, cnt = str(w), 0
            if cnt <= 0:
                continue
            rows.append((en, cnt, str(rank_map.get(key, ""))))
        rows.sort(key=lambda x: (-x[1], x[0]))
        out[cat] = rows[:top_n]
    return out


def _render_category_blocks(
    category_data: Dict[str, Any],
    category_words: Dict[str, List[Tuple[str, int, str]]],
    word_zh: Dict[str, str],
) -> str:
    category_data = _normalize_word_category_data(category_data)
    parts = [
        '<div class="wf-categories">',
        "<h3>词频分类洞察</h3>",
    ]
    for cat in WORD_CATEGORIES:
        block = category_data.get(cat)
        analysis = ""
        if isinstance(block, dict):
            analysis = str(block.get("analysis") or "").strip()
        rows = category_words.get(cat, [])
        parts.append(f'<div class="wf-cat"><h4>{html.escape(cat)}</h4>')
        if analysis:
            parts.append(f"<p>{html.escape(analysis)}</p>")
        if rows:
            parts.append(
                "<table><thead><tr><th>排名</th><th>英文</th><th>中文</th><th>次数</th></tr></thead><tbody>"
            )
            for en, cnt, rank in rows:
                zh = _zh_for_word(en, word_zh)
                rank_cell = rank if rank else "—"
                parts.append(
                    f"<tr><td>{html.escape(str(rank_cell))}</td>"
                    f"<td>{html.escape(en)}</td><td>{html.escape(zh)}</td>"
                    f"<td>{cnt}</td></tr>"
                )
            parts.append("</tbody></table>")
        else:
            parts.append('<p class="muted">（本类暂无词条）</p>')
        parts.append("</div>")
    parts.append("</div>")
    return "\n".join(parts)


def _render_freq_table_pages(
    word_freq: List[Tuple[str, int]], word_zh: Dict[str, str]
) -> str:
    top = word_freq[:WORD_FREQ_TABLE_N]
    total = sum(c for _, c in top) or 1
    max_cnt = max((c for _, c in top), default=1)
    row_lines: List[str] = []
    for rank, (en, cnt) in enumerate(top, start=1):
        page = (rank - 1) // WORD_FREQ_PAGE_SIZE + 1
        zh = _zh_for_word(en, word_zh)
        pct = cnt / total * 100.0
        bar_w = max(4, int(cnt / max_cnt * 100))
        row_lines.append(
            f'<tr class="wf-row" data-page="{page}">'
            f"<td>{rank}</td>"
            f'<td class="wf-en">{html.escape(en)}</td>'
            f"<td>{html.escape(zh)}</td>"
            f'<td class="wf-count"><span class="wf-bar" style="width:{bar_w}%"></span>'
            f"<span>{cnt}</span></td>"
            f"<td>{pct:.2f}%</td></tr>"
        )
    n_pages = min(
        WORD_FREQ_PAGES,
        (len(top) + WORD_FREQ_PAGE_SIZE - 1) // WORD_FREQ_PAGE_SIZE or 1,
    )
    tabs = "".join(
        f'<button type="button" class="wf-tab{" active" if i == 0 else ""}" '
        f'data-page="{i + 1}">第{i + 1}页</button>'
        for i in range(n_pages)
    )
    body = f"""<motion class="wf-pager">
  <motion class="wf-tab-bar">{tabs}</div>
  <div class="wf-nav">
    <button type="button" id="wf-prev">上一页</button>
    <span id="wf-page-label">第 1 / {n_pages} 页</span>
    <button type="button" id="wf-next">下一页</button>
  </div>
  <div class="wf-table-wrap">
    <table id="wf-freq-table" class="wf-freq-table">
      <thead><tr>
        <th>排名</th><th>英文</th><th>中文</th><th>次数</th><th>占比</th>
      </tr></thead>
      <tbody id="wf-tbody">
        {chr(10).join(row_lines)}
      </tbody>
    </table>
  </div>
  <p class="muted">占比 = 该词次数 / Top{WORD_FREQ_TABLE_N} 词次数之和（相对占比）。</p>
</div>"""
    return body.replace("<motion", "<div", 2)


def _render_wordfreq_section(
    word_freq: List[Tuple[str, int]],
    word_zh: Dict[str, str],
    category_data: Dict[str, Any],
) -> str:
    category_words = _category_top_words(category_data, word_freq, top_n=10)
    cat_html = _render_category_blocks(category_data, category_words, word_zh)
    table_html = _render_freq_table_pages(word_freq, word_zh)
    return (
        f'<section id="sec-wordfreq">\n'
        f"  <h2>词频分析</h2>\n"
        f'  <p class="muted">词云 Top {WORDCLOUD_TOP_N} · 词表 Top {WORD_FREQ_TABLE_N} · '
        f"分类词表 Top {WORD_CATEGORY_CLASSIFY_N}</p>\n"
        f"  <h3>词云图</h3>\n"
        f"  {_render_wordcloud_legend()}\n"
        f'  <div id="wordcloud-chart"></div>\n'
        f"  {cat_html}\n"
        f"  <h3>全部词频表</h3>\n"
        f"  {table_html}\n"
        f"</section>"
    )


def _render_phrase_sections(appendix_bundles: List[ClusterBundle]) -> str:
    by_stage: Dict[str, List[ClusterBundle]] = defaultdict(list)
    for b in appendix_bundles:
        by_stage[b.stage].append(b)

    parts = ['<div id="sec-phrases" class="ai-verify-body">']
    for stage in PHRASE_APPENDIX_STAGES:
        title = _phrase_appendix_stage_title(stage)
        parts.append(f'<h3 class="stage-h">{html.escape(title)}</h3>')
        rows = sorted(by_stage.get(stage, []), key=lambda b: b.cluster_label)
        if not rows:
            parts.append('<p class="muted">（暂无）</p>')
            continue
        for b in rows:
            pct = f"{b.ratio * 100:.1f}%"
            parts.append(
                f'<h4 id="{b.anchor_id}">{html.escape(b.cluster_title_zh)}'
                f' <span class="muted">（{b.review_count} 条评论 · {pct}）</span></h4>'
            )
            display = b.embed_texts_zh or b.embed_texts
            parts.append(
                f'<details><summary>展开本簇 {len(display)} 条代表性表述（中文）</summary><ol>'
            )
            for txt in display:
                parts.append(f"<li><p>{html.escape(txt)}</p></li>")
            if not display:
                parts.append("<li><p>（无可用表述）</p></li>")
            parts.append("</ol></details>")
    parts.append("</div>")
    return "\n".join(parts)


def _render_structured_validation_section(
    samples: List[Tuple[int, str, Dict[str, Any]]],
) -> str:
    parts = [
        '<div id="sec-structured" class="ai-verify-body">',
        f'<p class="muted">展示前 {STRUCTURED_APPENDIX_SAMPLE_N} 条评论原文及对应结构化结果（按 source_row 升序）。</p>',
    ]
    if not samples:
        parts.append('<p class="muted">（暂无结构化样本，请先执行结构化步骤。）</p>')
        parts.append("</div>")
        return "\n".join(parts)

    for i, (sr, content, display) in enumerate(samples, start=1):
        js = json.dumps(display, ensure_ascii=False, separators=(",", ":"))
        parts.append(f'<article class="struct-sample" id="struct-sample-{sr}">')
        parts.append(f"<h4>样本 {i} <span class=\"muted\">（source_row={sr}）</span></h4>")
        parts.append("<p><strong>评论原文：</strong></p>")
        parts.append(f'<p class="struct-review">{html.escape(content)}</p>')
        parts.append("<p><strong>结构化内容：</strong></p>")
        parts.append(f'<pre class="struct-json">{html.escape(js)}</pre>')
        parts.append("</article>")
    parts.append("</div>")
    return "\n".join(parts)


def _wrap_ai_verify_panel(title: str, inner_html: str) -> str:
    """将子版块包进可折叠面板（默认收起）。"""
    return (
        f'<details class="ai-verify-panel">\n'
        f"  <summary>{html.escape(title)}</summary>\n"
        f"  {inner_html}\n"
        f"</details>"
    )


def _render_ai_validation_section(
    structured_html: str,
    phrases_html: str,
) -> str:
    """AI 分析效果验证：结构化 + 聚类，机器固定顺序、可展开。"""
    structured_panel = _wrap_ai_verify_panel("结构化提取效果验证", structured_html)
    phrases_panel = _wrap_ai_verify_panel("聚类效果验证", phrases_html)
    return (
        '<section id="sec-ai-verify">\n'
        "  <h2>AI分析效果验证</h2>\n"
        '  <p class="muted">以下为结构化与聚类结果的抽样展示，点击标题展开查看。</p>\n'
        f"  {structured_panel}\n"
        f"  {phrases_panel}\n"
        "</section>"
    )


def _assemble_html(
    *,
    product_name: str,
    wordfreq_html: str,
    report_html: str,
    phrases_html: str,
    structured_html: str,
    wordcloud_data: List[dict],
) -> str:
    """按固定顺序机器拼装最终 HTML（不由 LLM 决定版块顺序）。"""
    data_json = json.dumps(wordcloud_data, ensure_ascii=False)
    page_title = f"{product_name} · 评论分析报告"
    report_body = _strip_report_top_heading(report_html)
    report_section = (
        '<section id="sec-report">\n'
        "  <h2>产品改进建议报告</h2>\n"
        f"  {report_body}\n"
        "</section>"
    )
    ai_verify_section = _render_ai_validation_section(structured_html, phrases_html)
    # 顺序固定：改进建议 → 词频 → AI 分析效果验证（内含结构化 / 聚类，可展开）
    body_sections = "\n".join((report_section, wordfreq_html, ai_verify_section))
    return f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>{page_title}</title>
  <script src="https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/echarts-wordcloud@2/dist/echarts-wordcloud.min.js"></script>
  <style>
    body {{ font-family: -apple-system, "PingFang SC", "Microsoft YaHei", sans-serif;
      margin: 0 auto; padding: 24px; line-height: 1.6; color: #222; max-width: 1100px; }}
    .muted {{ color: #666; font-size: 0.9em; }}
    #wordcloud-chart {{ width: 100%; height: 520px; margin: 16px 0 24px; }}
    .wc-legend {{ display: flex; flex-wrap: wrap; gap: 8px 14px; margin: 8px 0 12px;
      font-size: 0.88em; color: #444; }}
    .wc-legend-item {{ display: inline-flex; align-items: center; gap: 6px; }}
    .wc-dot {{ display: inline-block; width: 10px; height: 10px; border-radius: 50%; }}
    table {{ border-collapse: collapse; width: 100%; margin: 12px 0; }}
    th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
    th {{ background: #f0f0f0; }}
    details {{ margin: 12px 0 24px; padding: 8px; background: #fafafa; border-radius: 6px; }}
    #sec-ai-verify {{ margin-top: 40px; }}
    details.ai-verify-panel {{ margin: 16px 0; padding: 0; border: 1px solid #e5e7eb;
      background: #fff; }}
    details.ai-verify-panel > summary {{ padding: 12px 16px; font-size: 1.05em; font-weight: 600;
      cursor: pointer; list-style: none; background: #f8fafc; border-radius: 6px 6px 0 0; }}
    details.ai-verify-panel > summary::-webkit-details-marker {{ display: none; }}
    details.ai-verify-panel > summary::before {{ content: "▸ "; color: #2563eb; }}
    details.ai-verify-panel[open] > summary::before {{ content: "▾ "; }}
    details.ai-verify-panel[open] > summary {{ border-bottom: 1px solid #e5e7eb; }}
    .ai-verify-body {{ padding: 12px 16px 16px; }}
    .stage-h {{ margin-top: 32px; border-bottom: 1px solid #eee; padding-bottom: 8px; }}
    .wf-categories {{ margin: 24px 0; }}
    .wf-cat {{ margin-bottom: 20px; }}
    .wf-pager {{ margin: 16px 0; }}
    .wf-tab-bar {{ margin-bottom: 8px; }}
    .wf-tab {{ margin: 4px 8px 4px 0; padding: 6px 12px; cursor: pointer;
      border: 1px solid #ccc; background: #fff; border-radius: 4px; }}
    .wf-tab.active {{ background: #2563eb; color: #fff; border-color: #2563eb; }}
    .wf-nav {{ margin: 12px 0; display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }}
    .wf-table-wrap {{ max-height: 520px; overflow: auto; border: 1px solid #e5e7eb; border-radius: 8px; }}
    .wf-freq-table thead th {{ position: sticky; top: 0; z-index: 1; box-shadow: 0 1px 0 #ddd; }}
    .wf-freq-table tbody tr:nth-child(even) {{ background: #fafafa; }}
    .wf-freq-table tbody tr:hover {{ background: #f0f7ff; }}
    .wf-row {{ display: none; }}
    .wf-row.wf-visible {{ display: table-row; }}
    .wf-count {{ min-width: 120px; }}
    .wf-count .wf-bar {{ display: inline-block; height: 10px; margin-right: 8px;
      background: linear-gradient(90deg, #93c5fd, #2563eb); border-radius: 2px; vertical-align: middle; }}
    .wf-en {{ font-family: ui-monospace, monospace; font-size: 0.92em; }}
    #sec-wordfreq {{ margin-top: 40px; }}
    .struct-sample {{ margin: 24px 0; padding: 16px; border: 1px solid #e5e7eb;
      border-radius: 8px; background: #fafafa; }}
    .struct-sample h4 {{ margin: 0 0 12px; }}
    .struct-review {{ white-space: pre-wrap; word-break: break-word; margin: 8px 0 16px; }}
    .struct-json {{ margin: 0; padding: 12px; background: #fff; border: 1px solid #e5e7eb;
      border-radius: 6px; overflow-x: auto; font-size: 0.9em; line-height: 1.5; }}
  </style>
</head>
<body>
  <h1>{page_title}</h1>
  {body_sections}
  <script>
    (function() {{
      const el = document.getElementById('wordcloud-chart');
      if (el && typeof echarts !== 'undefined') {{
        const chart = echarts.init(el);
        chart.setOption({{
          tooltip: {{
            show: true,
            formatter: function(p) {{
              var d = p.data || {{}};
              var rank = d.rank ? ('#' + d.rank + ' ') : '';
              var cnt = d.count != null ? d.count : p.value;
              var zh = d.zh && d.zh !== d.name ? ('<br/>中文: ' + d.zh) : '';
              var cat = (d.categories && d.categories.length)
                ? ('<br/>分类: ' + d.categories.join('、'))
                : (d.category ? ('<br/>分类: ' + d.category) : '<br/>分类: 未分类');
              return rank + (d.name || p.name) + zh + cat + '<br/>次数: ' + cnt;
            }}
          }},
          series: [{{
            type: 'wordCloud',
            shape: 'circle',
            width: '95%',
            height: '95%',
            sizeRange: [{WORDCLOUD_SIZE_MIN}, {WORDCLOUD_SIZE_MAX}],
            rotationRange: [-15, 15],
            rotationStep: 15,
            gridSize: 8,
            drawOutOfBound: false,
            layoutAnimation: true,
            textStyle: {{ fontFamily: 'system-ui, -apple-system, sans-serif' }},
            emphasis: {{ focus: 'self', textStyle: {{ shadowBlur: 6, shadowColor: '#333' }} }},
            data: {data_json}
          }}]
        }});
        window.addEventListener('resize', function() {{ chart.resize(); }});
      }}
      const rows = Array.from(document.querySelectorAll('.wf-row'));
      const tabs = Array.from(document.querySelectorAll('.wf-tab'));
      const label = document.getElementById('wf-page-label');
      const pageCount = tabs.length || 1;
      let cur = 1;
      function showPage(n) {{
        cur = Math.max(1, Math.min(n, pageCount));
        rows.forEach(function(r) {{
          r.classList.toggle('wf-visible', parseInt(r.dataset.page, 10) === cur);
        }});
        tabs.forEach(function(t) {{
          t.classList.toggle('active', parseInt(t.dataset.page, 10) === cur);
        }});
        if (label) label.textContent = '第 ' + cur + ' / ' + pageCount + ' 页';
      }}
      tabs.forEach(function(t) {{
        t.addEventListener('click', function() {{
          showPage(parseInt(t.dataset.page, 10));
        }});
      }});
      var prev = document.getElementById('wf-prev');
      var next = document.getElementById('wf-next');
      if (prev) prev.addEventListener('click', function() {{ showPage(cur - 1); }});
      if (next) next.addEventListener('click', function() {{ showPage(cur + 1); }});
      showPage(1);
    }})();
  </script>
</body>
</html>
"""


def generate_report(
    *,
    product_name: str,
    industry: str,
    cleaned_csv: Path,
    cluster_db: Path,
    embed_db: Path,
    word_freq_csv: Path,
    output_html: Path,
    structured_db: Path = STRUCTURED_DB,
    min_cluster_review_ratio: float | None = None,
    save_llm_raw: bool = False,
    llm_raw_path: Path | None = None,
) -> dict:
    sync_voc_report_constants(sys.modules[__name__])
    api_key = _resolve_api_key()
    if not api_key:
        raise RuntimeError("缺少 DASHSCOPE_API_KEY 或 .dashscope_key")

    cleaned_csv = cleaned_csv.resolve()
    word_freq = _load_word_freq(
        word_freq_csv,
        max(WORDCLOUD_TOP_N, WORD_FREQ_TABLE_N, WORD_CATEGORY_CLASSIFY_N),
    )
    total_reviews, bundles = build_cluster_bundles(
        cluster_db=cluster_db,
        embed_db=embed_db,
        cleaned_csv=cleaned_csv,
        min_cluster_review_ratio=min_cluster_review_ratio,
    )
    top2_audience = _load_step2_top2_audience(cluster_db)
    if not top2_audience:
        top2_audience = _ordered_step2_audiences(bundles, [])[:2]
    logger.info("阶段二 top2 受众簇: %s", top2_audience)

    logger.info("调用 LLM 生成分析报告（含词频分类、簇命名、译文映射）…")
    system, user = _build_report_prompt(
        product_name=product_name,
        industry=industry,
        total_reviews=total_reviews,
        bundles=bundles,
        word_freq=word_freq,
        top2_audience_clusters=top2_audience,
    )
    raw = _fetch_report_llm_raw_with_retry(
        system=system,
        user=user,
        api_key=api_key,
        expect_cluster_names=len(bundles) > 0,
    )
    saved_raw_path: str | None = None
    if save_llm_raw:
        raw_path = (llm_raw_path or output_html.parent / "report_llm_raw.txt").resolve()
        raw_path.parent.mkdir(parents=True, exist_ok=True)
        raw_path.write_text(raw, encoding="utf-8")
        saved_raw_path = str(raw_path)
        logger.info("已保存 LLM 原文 %s", raw_path)

    report_html, word_zh, category_data, cluster_names = _parse_report_response(raw)
    category_data = _classify_remaining_words_in_pool(
        category_data,
        word_freq,
        industry=industry,
        product_name=product_name,
        api_key=api_key,
    )
    category_data = _fill_category_analysis(
        category_data,
        word_freq,
        industry=industry,
        product_name=product_name,
        api_key=api_key,
    )
    _apply_cluster_names(bundles, cluster_names)

    appendix_bundles = _bundles_for_phrase_appendix(bundles)
    all_phrases: List[str] = []
    for b in appendix_bundles:
        all_phrases.extend(b.embed_texts)
    zh_map = _translate_unique_phrases(all_phrases, api_key)
    _fill_phrase_translations(appendix_bundles, zh_map)

    wc_data = _wordcloud_data(word_freq, word_zh, category_data)
    wordfreq_sec = _render_wordfreq_section(word_freq, word_zh, category_data)
    phrases_sec = _render_phrase_sections(appendix_bundles)
    struct_samples = _build_structured_appendix_samples(
        structured_db=structured_db.resolve(),
        cleaned_csv=cleaned_csv,
    )
    structured_sec = _render_structured_validation_section(struct_samples)
    html = _assemble_html(
        product_name=product_name,
        wordfreq_html=wordfreq_sec,
        report_html=report_html,
        phrases_html=phrases_sec,
        structured_html=structured_sec,
        wordcloud_data=wc_data,
    )
    output_html.parent.mkdir(parents=True, exist_ok=True)
    output_html.write_text(html, encoding="utf-8")
    logger.info("已写入 %s", output_html)
    return {
        "report_html": str(output_html),
        "total_reviews": total_reviews,
        "cluster_groups": len(bundles),
        "min_cluster_review_ratio": min_cluster_review_ratio,
        "structured_samples": len(struct_samples),
        "llm_raw_path": saved_raw_path,
    }