"""
从 voc_embeddings.sqlite 读取向量，按五段流程 UMAP+HDBSCAN 聚类，结果写入 voc_clustering.sqlite。

流程：
  1. audience（LLM 自动调参 n_neighbors）
  2a. 前两 audience 簇各自独立：簇内 pain_point（LLM 自动调参）
  2b. 前两 audience 簇各自独立：簇内 aspect_opinion 按 Positive/Negative/Neutral 分桶聚类
  3a. 全量 pain_point（LLM 自动调参）
  3b. 全量 aspect_opinion 按 Positive/Negative/Neutral 分桶聚类（各自 LLM 自动调参）

规则：
  - 聚类单元：每行 embedding_items
  - n < 10：不跑 HDBSCAN，逐条独立簇标签（0..n-1）
  - 步骤 1/2 的 -1：写入 SQL，不参与后续（步骤 1 的 -1 不进入步骤 2）
  - 步骤 3 的 -1：写入 SQL，参与后续统计

用法::

  ./310py/bin/python 聚类.py
  # 默认自动使用 voc_structured.sqlite 中最新 analysis_jobs.id（须已向量化）
  ./310py/bin/python 聚类.py --job-id 4   # 可选：手动指定
"""
from __future__ import annotations

import argparse
import csv
import json
import logging
import os
import random
import re
import sqlite3
import struct
import sys
import time
import warnings
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Tuple

import hdbscan
import numpy as np
import umap
from openai import OpenAI
from sklearn.metrics import silhouette_score

from voc_llm import CHAT_MODEL, chat_extra_body, create_chat_client, require_chat_api_key

# UMAP 设 random_state 时的单线程提示；轮廓系数在 extmath 中的 matmul 数值告警
warnings.filterwarnings(
    "ignore",
    message=r"n_jobs value .* overridden .* by setting random_state",
    category=UserWarning,
    module=r"umap\.umap_",
)
warnings.filterwarnings(
    "ignore",
    category=RuntimeWarning,
    module=r"sklearn\.utils\.extmath",
)


logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    stream=sys.stderr,
)
logger = logging.getLogger("voc_cluster")

PROJECT_ROOT = Path(__file__).resolve().parent
STRUCTURED_DB = PROJECT_ROOT / "voc_structured.sqlite"
EMBED_DB = PROJECT_ROOT / "voc_embeddings.sqlite"
CLUSTER_DB = PROJECT_ROOT / "voc_clustering.sqlite"
LLM_MODEL = CHAT_MODEL

UMAP_N_COMPONENTS = 30
UMAP_MIN_DIST = 0.1
UMAP_METRIC = "cosine"
UMAP_RANDOM_STATE = 42
MIN_POINTS_FOR_HDBSCAN = 10
# HDBSCAN 目标：非离群簇数不超过该值（通过提高 min_cluster_size 达成）
MAX_CLUSTERS = 20

VALID_SENTIMENTS: Tuple[str, ...] = ("Positive", "Negative", "Neutral")
_SENTIMENT_STAGE_SUFFIX: Dict[str, str] = {
    "Positive": "positive",
    "Negative": "negative",
    "Neutral": "neutral",
}

HDBSCAN_BASE = dict(
    min_samples=1,
    cluster_selection_method="eom",
    prediction_data=True,
)

INITIAL_N_NEIGHBORS = 10
MAX_N_NEIGHBORS = 45
CROSS_SIMILAR_RATIO_THRESHOLD = 0.10
SILHOUETTE_STOP_THRESHOLD = 0.6
SILHOUETTE_DECLINE_WINDOW = 10  # 连续 N 个轮廓值：后 N-1 个均小于第 1 个则停止
SILHOUETTE_NEIGHBOR_MARGIN = 0.03  # 邻轮轮廓 ≥ 峰值−此值视为「接近」，参与离群数决胜
SILHOUETTE_NEIGHBOR_RADIUS = 3  # 峰值轮次前后各 3 轮
SAMPLE_CAP = 30
SAMPLE_RATIO = 0.6

# 步骤 2 未启用 LLM 调参时的保守默认（当前步骤 2 已启用自动调参）
STEP2_N_NEIGHBORS = 8

# 簇内去重评论数 / 本步骤参与聚类的去重评论总数 < 该比例则不写入库、不进入报告
CLUSTER_MIN_REVIEW_RATIO = 0.10


@dataclass
class EmbedRow:
    id: int
    job_id: int
    extraction_id: int
    source_row: int
    entity_type: str
    entity_index: int
    embed_text: str
    audience: str | None
    aspect: str | None
    opinion: str | None
    category: str | None
    sentiment: str | None
    content: str
    dimensions: int
    embedding: np.ndarray


def _unpack_embedding(blob: bytes, dimensions: int) -> np.ndarray:
    n = dimensions
    expected = n * 4
    if len(blob) != expected:
        raise ValueError(f"BLOB 长度 {len(blob)} != {expected}（dim={n}）")
    return np.array(struct.unpack(f"{n}f", blob), dtype=np.float32)


def _latest_job_id_from_structured(structured_db: Path) -> int | None:
    """与 向量化.py 一致：analysis_jobs 表 id 最大者为最新 job。"""
    if not structured_db.is_file():
        return None
    conn = sqlite3.connect(structured_db)
    try:
        row = conn.execute(
            "SELECT id FROM analysis_jobs ORDER BY id DESC LIMIT 1"
        ).fetchone()
        return int(row[0]) if row else None
    finally:
        conn.close()


def _latest_job_id_from_embeddings(conn: sqlite3.Connection) -> int | None:
    row = conn.execute(
        "SELECT job_id FROM embedding_items ORDER BY job_id DESC LIMIT 1"
    ).fetchone()
    return int(row[0]) if row else None


def _count_embeddings(conn: sqlite3.Connection, job_id: int) -> int:
    row = conn.execute(
        "SELECT COUNT(*) FROM embedding_items WHERE job_id = ?", (job_id,)
    ).fetchone()
    return int(row[0]) if row else 0


def _resolve_job_id(
    job_id: int | None,
    embed_conn: sqlite3.Connection,
    structured_db: Path = STRUCTURED_DB,
) -> int:
    """未传 job_id 时自动选最新结构化 job，并确认向量库中已有数据。"""
    if job_id is not None:
        jid = job_id
        if _count_embeddings(embed_conn, jid) == 0:
            raise RuntimeError(
                f"job_id={jid} 在 {EMBED_DB.name} 中无向量，请先运行 向量化.py"
            )
        logger.info("使用指定 job_id=%s（%s 条向量）", jid, _count_embeddings(embed_conn, jid))
        return jid

    jid = _latest_job_id_from_structured(structured_db)
    source = "voc_structured.analysis_jobs"
    if jid is None:
        jid = _latest_job_id_from_embeddings(embed_conn)
        source = "voc_embeddings.embedding_items"
    if jid is None:
        raise RuntimeError("无可用 job_id：请先运行 结构化_server.py 与 向量化.py")

    n = _count_embeddings(embed_conn, jid)
    if n == 0:
        latest_embed = _latest_job_id_from_embeddings(embed_conn)
        if latest_embed is not None and latest_embed != jid:
            logger.warning(
                "结构化最新 job_id=%s 尚无向量，改用向量库最新 job_id=%s",
                jid,
                latest_embed,
            )
            jid = latest_embed
            n = _count_embeddings(embed_conn, jid)
            source = "voc_embeddings.embedding_items（回退）"
        else:
            raise RuntimeError(
                f"最新 job_id={jid} 在向量库中无数据，请先对应该 job 运行 向量化.py"
            )

    logger.info("自动选用最新 job_id=%s（来源: %s，%s 条向量）", jid, source, n)
    return jid


def _load_embed_rows(
    conn: sqlite3.Connection, job_id: int, entity_type: str
) -> List[EmbedRow]:
    cur = conn.execute(
        """
        SELECT id, job_id, extraction_id, source_row, entity_type, entity_index,
               embed_text, audience, aspect, opinion, category, sentiment,
               content, dimensions, embedding
        FROM embedding_items
        WHERE job_id = ? AND entity_type = ?
        ORDER BY source_row, entity_index, id
        """,
        (job_id, entity_type),
    )
    out: List[EmbedRow] = []
    for r in cur.fetchall():
        dims = int(r[13])
        out.append(
            EmbedRow(
                id=int(r[0]),
                job_id=int(r[1]),
                extraction_id=int(r[2]),
                source_row=int(r[3]),
                entity_type=str(r[4]),
                entity_index=int(r[5]),
                embed_text=str(r[6]),
                audience=r[7],
                aspect=r[8],
                opinion=r[9],
                category=r[10],
                sentiment=r[11],
                content=str(r[12] or ""),
                dimensions=dims,
                embedding=_unpack_embedding(r[14], dims),
            )
        )
    return out


def _filter_by_source_rows(
    rows: List[EmbedRow], allowed: set[int]
) -> List[EmbedRow]:
    return [r for r in rows if r.source_row in allowed]


def _strict_sentiment(raw: str | None) -> str | None:
    """仅接受 Positive / Negative / Neutral，其余丢弃。"""
    if raw is None:
        return None
    s = str(raw).strip()
    if s in VALID_SENTIMENTS:
        return s
    return None


def _filter_by_sentiment(rows: List[EmbedRow], sentiment: str) -> List[EmbedRow]:
    return [r for r in rows if _strict_sentiment(r.sentiment) == sentiment]


def _stage_3b(sentiment: str) -> str:
    return f"3b_aspect_opinion_{_SENTIMENT_STAGE_SUFFIX[sentiment]}"


def _stage_2b(sentiment: str, audience_cluster: int) -> str:
    return f"2b_aspect_opinion_{_SENTIMENT_STAGE_SUFFIX[sentiment]}_audience_c{audience_cluster}"


def _cluster_aspect_opinion_sentiment_stages(
    cconn: sqlite3.Connection,
    run_id: int,
    ao_rows: List[EmbedRow],
    *,
    stage_for: Any,
    outlier_participates: bool,
    parent_audience_cluster: int | None,
    skipped_reason_prefix: str,
    client: OpenAI,
) -> Dict[str, int]:
    """对 aspect_opinion 按三档情感各跑一 stage，返回 stage -> 条数。"""
    counts: Dict[str, int] = {}
    for sentiment in VALID_SENTIMENTS:
        stage = stage_for(sentiment)
        sub = _filter_by_sentiment(ao_rows, sentiment)
        logger.info(
            "[%s] %s 条 aspect_opinion（情感=%s）",
            stage,
            len(sub),
            sentiment,
        )
        if not sub:
            _save_stage_meta(
                cconn,
                run_id,
                stage,
                {
                    "skipped": True,
                    "reason": f"{skipped_reason_prefix}_{_SENTIMENT_STAGE_SUFFIX[sentiment]}",
                    "sentiment": sentiment,
                },
            )
            counts[stage] = 0
            continue
        labels, meta = _cluster_stage(
            sub, stage=stage, use_llm_tune=True, client=client
        )
        parent_clusters = None
        if parent_audience_cluster is not None:
            parent_clusters = [parent_audience_cluster] * len(sub)
        filt = _save_assignments(
            cconn,
            run_id,
            stage,
            sub,
            labels,
            n_neighbors=meta.get("n_neighbors"),
            outlier_participates=outlier_participates,
            parent_clusters=parent_clusters,
        )
        meta["cluster_filter"] = filt
        meta["sentiment"] = sentiment
        _save_stage_meta(cconn, run_id, stage, meta)
        if meta.get("tuning_log"):
            _save_tuning_logs(cconn, run_id, stage, meta["tuning_log"])
        counts[stage] = len(sub)
    return counts


def _source_rows_for_audience_cluster(
    row_to_aud: Dict[int, int], audience_cluster: int
) -> set[int]:
    return {sr for sr, lab in row_to_aud.items() if lab == audience_cluster}


def _stage_name_step2_pain(audience_cluster: int) -> str:
    return f"2a_pain_audience_c{audience_cluster}"


def _cluster_step2_for_audience(
    cconn: sqlite3.Connection,
    run_id: int,
    *,
    audience_cluster: int,
    pain_rows: List[EmbedRow],
    ao_rows: List[EmbedRow],
    row_to_aud: Dict[int, int],
    client: OpenAI,
) -> Dict[str, int]:
    """对单个 audience 簇分别聚类 pain 与 aspect_opinion，返回各 stage 条数。"""
    allowed = _source_rows_for_audience_cluster(row_to_aud, audience_cluster)
    counts: Dict[str, int] = {}
    pain_sub = _filter_by_source_rows(pain_rows, allowed)
    ao_sub = _filter_by_source_rows(ao_rows, allowed)

    stage_pain = _stage_name_step2_pain(audience_cluster)
    logger.info(
        "[%s] audience 簇 %s：%s 条 pain（%s 条评论）",
        stage_pain,
        audience_cluster,
        len(pain_sub),
        len(allowed),
    )
    if pain_sub:
        labels_pain, meta_pain = _cluster_stage(
            pain_sub, stage=stage_pain, use_llm_tune=True, client=client
        )
        filt = _save_assignments(
            cconn,
            run_id,
            stage_pain,
            pain_sub,
            labels_pain,
            n_neighbors=meta_pain.get("n_neighbors"),
            outlier_participates=False,
            parent_clusters=[audience_cluster] * len(pain_sub),
        )
        meta_pain["cluster_filter"] = filt
        _save_stage_meta(cconn, run_id, stage_pain, meta_pain)
        if meta_pain.get("tuning_log"):
            _save_tuning_logs(cconn, run_id, stage_pain, meta_pain["tuning_log"])
        counts[stage_pain] = len(pain_sub)
    else:
        _save_stage_meta(
            cconn,
            run_id,
            stage_pain,
            {"skipped": True, "reason": "no_pain_points", "audience_cluster": audience_cluster},
        )
        counts[stage_pain] = 0

    ao_counts = _cluster_aspect_opinion_sentiment_stages(
        cconn,
        run_id,
        ao_sub,
        stage_for=lambda s: _stage_2b(s, audience_cluster),
        outlier_participates=False,
        parent_audience_cluster=audience_cluster,
        skipped_reason_prefix="no_aspect_opinion",
        client=client,
    )
    counts.update(ao_counts)

    return counts


def _adaptive_min_cluster_size(n: int) -> int:
    """初始 min_cluster_size = max(2, n // MAX_CLUSTERS)，无上限；簇数仍 > MAX_CLUSTERS 时再迭代增大。"""
    if n < MIN_POINTS_FOR_HDBSCAN:
        return 2
    return max(2, n // MAX_CLUSTERS)


def _n_clusters_from_labels(labels: np.ndarray) -> int:
    labs = {int(x) for x in labels.tolist()}
    labs.discard(-1)
    return len(labs)


def _stack_embeddings(rows: Sequence[EmbedRow]) -> np.ndarray:
    return np.vstack([r.embedding for r in rows]).astype(np.float32)


def _fit_umap(embeddings: np.ndarray, n_neighbors: int) -> np.ndarray:
    n = len(embeddings)
    n_neighbors = min(n_neighbors, max(2, n - 1))
    reducer = umap.UMAP(
        n_components=min(UMAP_N_COMPONENTS, max(2, n - 2)),
        n_neighbors=n_neighbors,
        min_dist=UMAP_MIN_DIST,
        metric=UMAP_METRIC,
        random_state=UMAP_RANDOM_STATE,
    )
    return reducer.fit_transform(embeddings)


def _fit_hdbscan(umap_emb: np.ndarray, min_cluster_size: int) -> np.ndarray:
    clusterer = hdbscan.HDBSCAN(
        min_cluster_size=min_cluster_size,
        **HDBSCAN_BASE,
    )
    return clusterer.fit_predict(umap_emb)


def _run_umap_hdbscan_capped(
    embeddings: np.ndarray,
    n_neighbors: int,
    *,
    max_clusters: int = MAX_CLUSTERS,
) -> Tuple[np.ndarray, np.ndarray, int]:
    """UMAP + HDBSCAN；若簇数 > max_clusters 则增大 min_cluster_size 直至满足或无法再增。"""
    n = len(embeddings)
    umap_emb = _fit_umap(embeddings, n_neighbors)
    min_cs = _adaptive_min_cluster_size(n)
    labels = _fit_hdbscan(umap_emb, min_cs)
    n_clusters = _n_clusters_from_labels(labels)

    while n_clusters > max_clusters and min_cs < n:
        step = max(1, (n_clusters - max_clusters + 1) // 2)
        next_cs = min(min_cs + step, n)
        if next_cs <= min_cs:
            break
        logger.info(
            "簇数 %s > %s，min_cluster_size %s -> %s",
            n_clusters,
            max_clusters,
            min_cs,
            next_cs,
        )
        min_cs = next_cs
        labels = _fit_hdbscan(umap_emb, min_cs)
        n_clusters = _n_clusters_from_labels(labels)

    if n_clusters > max_clusters:
        logger.warning(
            "簇数 %s 仍 > %s（min_cluster_size=%s, n=%s）",
            n_clusters,
            max_clusters,
            min_cs,
            n,
        )
    return labels, umap_emb, min_cs


def _labels_all_zero(n: int) -> np.ndarray:
    return np.zeros(n, dtype=int)


def _strip_think(text: str) -> str:
    if not text:
        return text
    text = re.sub(
        r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE
    )
    text = re.sub(r"</?think>", "", text, flags=re.IGNORECASE)
    return text.strip()


def _parse_json_from_llm(text: str) -> dict:
    text = _strip_think(text or "")
    text = text.replace("```json", "").replace("```JSON", "").replace("```", "").strip()
    start = text.find("{")
    end = text.rfind("}")
    if start == -1 or end <= start:
        raise ValueError("未找到 JSON 对象")
    return json.loads(text[start : end + 1])


def _sample_from_clusters(
    sentences: List[str], cluster_labels: np.ndarray, rng: random.Random
) -> Dict[int, List[str]]:
    groups: Dict[int, List[str]] = {}
    for sent, lab in zip(sentences, cluster_labels.tolist()):
        groups.setdefault(int(lab), []).append(sent)
    samples: Dict[int, List[str]] = {}
    for lab, sents in groups.items():
        if lab == -1:
            continue
        n = len(sents)
        k = min(SAMPLE_CAP, max(1, int(n * SAMPLE_RATIO)))
        k = min(k, n)
        samples[lab] = rng.sample(sents, k) if n > k else list(sents)
    return samples


def _ai_evaluate_cluster_samples(
    client: OpenAI, cluster_samples: Dict[int, List[str]], max_retries: int = 2
) -> dict:
    if not cluster_samples:
        return {"cross_similar_count": 0, "total_sentences": 0}
    lines = []
    total = 0
    for lab in sorted(cluster_samples.keys()):
        lines.append(f"【聚类{lab}】")
        for i, sent in enumerate(cluster_samples[lab], 1):
            lines.append(f"  {i}. {sent}")
            total += 1
    sample_text = "\n".join(lines)
    prompt = f"""你是 VOC 评论短语聚类质量评估助手。以下是多个聚类类别的抽样短语（英文为主）。

{sample_text}

任务：统计 cross_similar_count——**不同聚类类别之间**、语义相近的短语条数。

## 判断标准（从宽，不要漏判）
将两条短语判为「跨类相似」，只要它们表达的是**同一类用户意图/问题/反馈**，不要求措辞一致。以下情况**都应计入**：
- 同义改写：如 "doesn't work" 与 "not effective"
- 同一痛点不同说法：如 "cat pees on bed" 与 "urinates on sofa"
- 同一产品缺陷的不同表述：如 "strong smell" 与 "odor too strong"
- 核心对象相同、评价方向相同：如 "sprayer broken" 与 "nozzle stopped working"
- 一方是另一方的子集或概括：如 "joint pain" 与 "severe joint pain in elderly dog"

以下情况**不计入**：
- 仅在同一聚类类别内部相似（不算跨类）
- 明显不同方面：如 "fast shipping" 与 "bad smell"
- 褒贬相反：如 "works great" 与 "doesn't work at all"

## 计数规则
1. 逐条短语与其他聚类中的短语比对；只要与**任一**其他类的**任一**短语相近，该条计 1。
2. 每条短语最多计 1 次。
3. 宁可多计疑似相近，也不要漏掉明显同义/同主题的跨类重复。

只输出 JSON：
{{
  "cross_similar_count": 整数,
  "total_sentences": {total}
}}"""
    for attempt in range(max_retries):
        try:
            resp = client.chat.completions.create(
                model=LLM_MODEL,
                messages=[
                    {"role": "system", "content": "你是聚类质量评估助手。跨类相似判断从宽：同主题、同义改写、同一痛点/反馈的不同说法都应算相似。只输出合法 JSON。"},
                    {"role": "user", "content": prompt},
                ],
                max_tokens=200_000,
                temperature=0.0,
                response_format={"type": "json_object"},
                # 关闭思考，避免 token 耗在 reasoning_content 导致 content 为空且无 JSON
                extra_body=chat_extra_body(LLM_MODEL),
            )
            msg = resp.choices[0].message
            raw = msg.content or getattr(msg, "reasoning_content", None) or ""
            data = _parse_json_from_llm(raw)
            data["total_sentences"] = int(data.get("total_sentences", total) or total)
            data["cross_similar_count"] = int(data.get("cross_similar_count", 0))
            return data
        except Exception as e:
            logger.warning("AI 评估失败(第%s次): %s", attempt + 1, e)
            if attempt < max_retries - 1:
                time.sleep(1)
    return {"cross_similar_count": total, "total_sentences": total}


def _silhouette_decline_should_stop(scores: List[float]) -> bool:
    """最近 SILHOUETTE_DECLINE_WINDOW 个有效轮廓系数中，后 N-1 个是否都严格小于第一个。"""
    if len(scores) < SILHOUETTE_DECLINE_WINDOW:
        return False
    window = scores[-SILHOUETTE_DECLINE_WINDOW:]
    first = window[0]
    return all(s < first for s in window[1:])


def _best_silhouette_in_window(
    snapshots: List[Tuple[int, np.ndarray, float, int]],
) -> Tuple[int, np.ndarray, float, int] | None:
    """在最近轮廓窗口内取轮廓系数最高的一轮；无快照时返回 None。"""
    if not snapshots:
        return None
    pool = snapshots[-SILHOUETTE_DECLINE_WINDOW:]
    return max(pool, key=lambda x: x[2])


def _resolve_peak_with_neighbor_noise_tiebreak(
    snapshots: List[Tuple[int, np.ndarray, float, int]],
) -> Tuple[Tuple[int, np.ndarray, float, int], str | None]:
    """轮廓窗口早停：以窗口内峰值为中心，±2 邻轮若轮廓 ≥ 峰值−0.03 则与峰值一起按离群数择优。"""
    pool = snapshots[-SILHOUETTE_DECLINE_WINDOW:]
    center = max(pool, key=lambda x: x[2])
    center_nn, _, center_sil, center_noise = center
    threshold = center_sil - SILHOUETTE_NEIGHBOR_MARGIN
    by_nn = {s[0]: s for s in snapshots}
    candidates: Dict[int, Tuple[int, np.ndarray, float, int]] = {center_nn: center}
    for delta in range(-SILHOUETTE_NEIGHBOR_RADIUS, SILHOUETTE_NEIGHBOR_RADIUS + 1):
        if delta == 0:
            continue
        snap = by_nn.get(center_nn + delta)
        if snap is not None and snap[2] >= threshold:
            candidates[snap[0]] = snap
    if len(candidates) == 1:
        return center, None
    chosen = min(candidates.values(), key=lambda s: (s[3], -s[2]))
    note = (
        f"邻轮复核：峰值 n_neighbors={center_nn}（轮廓{center_sil:.4f}，离群{center_noise}），"
        f"候选 {sorted(candidates)} 中择离群最少 → n_neighbors={chosen[0]}"
        f"（轮廓{chosen[2]:.4f}，离群{chosen[3]}）"
    )
    return chosen, note


def _auto_tune_n_neighbors(
    embeddings: np.ndarray,
    sentences: List[str],
    client: OpenAI,
    stage: str,
) -> Tuple[np.ndarray, int, List[dict], Optional[float], int]:
    rng = random.Random(UMAP_RANDOM_STATE)
    n_neighbors = INITIAL_N_NEIGHBORS
    tuning_log: List[dict] = []
    labels: np.ndarray | None = None
    min_cs = 2
    silhouette_avg: float | None = None
    silhouette_history: List[float] = []
    # (n_neighbors, labels, silhouette, n_noise) 仅在有有效轮廓时入栈，供早停回退最优轮次
    silhouette_snapshots: List[Tuple[int, np.ndarray, float, int]] = []
    n = len(embeddings)

    while True:
        logger.info("[%s] 调参轮次 n_neighbors=%s", stage, n_neighbors)
        labels, umap_emb, min_cs = _run_umap_hdbscan_capped(embeddings, n_neighbors)
        n_clusters = _n_clusters_from_labels(labels)
        n_noise = int((labels == -1).sum())
        silhouette_avg = None
        if n_clusters > 1:
            mask = labels != -1
            if int(mask.sum()) > 1 and len(set(labels[mask].tolist())) > 1:
                silhouette_avg = float(
                    silhouette_score(umap_emb[mask], labels[mask])
                )
        logger.info(
            "[%s] 簇数=%s 离群=%s 轮廓=%s",
            stage,
            n_clusters,
            n_noise,
            f"{silhouette_avg:.4f}" if silhouette_avg is not None else "N/A",
        )

        samples = _sample_from_clusters(sentences, labels, rng)
        total_sample = sum(len(v) for v in samples.values())
        log_entry: dict = {
            "stage": stage,
            "n_neighbors": n_neighbors,
            "min_cluster_size": min_cs,
            "n_clusters": n_clusters,
            "n_noise": n_noise,
            "silhouette": silhouette_avg,
            "total_sample": total_sample,
        }

        if silhouette_avg is not None:
            silhouette_history.append(silhouette_avg)
            silhouette_snapshots.append(
                (n_neighbors, labels.copy(), silhouette_avg, n_noise)
            )
            if _silhouette_decline_should_stop(silhouette_history):
                window = silhouette_history[-SILHOUETTE_DECLINE_WINDOW:]
                best, tiebreak_note = _resolve_peak_with_neighbor_noise_tiebreak(
                    silhouette_snapshots
                )
                best_nn, best_labels, best_sil, best_noise = best
                n_neighbors = best_nn
                labels = best_labels
                silhouette_avg = best_sil
                log_entry["silhouette_window"] = [round(s, 4) for s in window]
                log_entry["n_noise"] = best_noise
                stop_tail = (
                    f"，回退至 n_neighbors={best_nn}（轮廓{best_sil:.4f}，离群{best_noise}）"
                )
                if tiebreak_note:
                    log_entry["neighbor_tiebreak"] = tiebreak_note
                    stop_tail = f"；{tiebreak_note}"
                log_entry["stop_reason"] = (
                    f"连续{SILHOUETTE_DECLINE_WINDOW}轮轮廓：后{SILHOUETTE_DECLINE_WINDOW - 1}个"
                    f"均低于窗口峰值{max(window):.4f}{stop_tail}"
                )
                tuning_log.append(log_entry)
                logger.info(
                    "[%s] 轮廓窗口 %s，早停并回退 n_neighbors=%s 轮廓=%.4f 离群=%s%s",
                    stage,
                    log_entry["silhouette_window"],
                    best_nn,
                    best_sil,
                    best_noise,
                    f"；{tiebreak_note}" if tiebreak_note else "",
                )
                break

        if n_neighbors > MAX_N_NEIGHBORS:
            best = _best_silhouette_in_window(silhouette_snapshots)
            if best is not None:
                best_nn, best_labels, best_sil, _ = best
                n_neighbors = best_nn
                labels = best_labels
                silhouette_avg = best_sil
                window = silhouette_history[-SILHOUETTE_DECLINE_WINDOW:]
                log_entry["silhouette_window"] = [round(s, 4) for s in window]
                log_entry["stop_reason"] = (
                    f"n_neighbors 超过上限 {MAX_N_NEIGHBORS}，"
                    f"回退至 n_neighbors={best_nn}（轮廓{best_sil:.4f}）"
                )
                logger.info(
                    "[%s] n_neighbors 超上限，回退 n_neighbors=%s 轮廓=%.4f",
                    stage,
                    best_nn,
                    best_sil,
                )
            else:
                log_entry["stop_reason"] = (
                    f"n_neighbors 超过上限 {MAX_N_NEIGHBORS}（无有效轮廓快照，保留当前轮）"
                )
            tuning_log.append(log_entry)
            break

        ai_result = _ai_evaluate_cluster_samples(client, samples)
        cross_count = ai_result["cross_similar_count"]
        total = ai_result["total_sentences"] or total_sample or 1
        ratio = cross_count / total
        log_entry.update(
            {
                "cross_similar_count": cross_count,
                "cross_similar_ratio": round(ratio, 4),
            }
        )
        sil_ok = silhouette_avg is not None and silhouette_avg > SILHOUETTE_STOP_THRESHOLD
        logger.info(
            "[%s] AI: 跨类相似 %s/%s (%.1f%%，阈值≤%.0f%%) 轮廓=%s (停止需>%.1f)",
            stage,
            cross_count,
            total,
            ratio * 100,
            CROSS_SIMILAR_RATIO_THRESHOLD * 100,
            f"{silhouette_avg:.4f}" if silhouette_avg is not None else "N/A",
            SILHOUETTE_STOP_THRESHOLD,
        )

        if ratio > CROSS_SIMILAR_RATIO_THRESHOLD:
            log_entry["stop_reason"] = "跨类相似率>10%，继续调参"
            tuning_log.append(log_entry)
            n_neighbors += 1
            time.sleep(0.3)
            continue

        if not sil_ok:
            reason = (
                "轮廓系数不可用，继续调参"
                if silhouette_avg is None
                else f"轮廓系数{silhouette_avg:.4f}≤{SILHOUETTE_STOP_THRESHOLD}，继续调参"
            )
            log_entry["stop_reason"] = reason
            tuning_log.append(log_entry)
            n_neighbors += 1
            time.sleep(0.3)
            continue

        log_entry["stop_reason"] = "跨类相似率≤10%且轮廓>0.6，停止调参"
        tuning_log.append(log_entry)
        break

    assert labels is not None
    return labels, n_neighbors, tuning_log, silhouette_avg, min_cs


def _cluster_fixed(
    embeddings: np.ndarray, n_neighbors: int = STEP2_N_NEIGHBORS
) -> Tuple[np.ndarray, int, int]:
    nn = min(n_neighbors, max(2, len(embeddings) - 1))
    labels, _, min_cs = _run_umap_hdbscan_capped(embeddings, nn)
    return labels, nn, min_cs


def _cluster_stage(
    rows: List[EmbedRow],
    *,
    stage: str,
    use_llm_tune: bool,
    client: OpenAI | None,
) -> Tuple[np.ndarray, dict]:
    n = len(rows)
    meta: dict = {
        "stage": stage,
        "n_items": n,
        "used_hdbscan": False,
        "n_neighbors": None,
        "tuning_log": [],
        "silhouette": None,
    }
    if n == 0:
        return np.array([], dtype=int), meta
    if n < MIN_POINTS_FOR_HDBSCAN:
        logger.info(
            "[%s] n=%s < %s，逐条独立簇标签，不跑 HDBSCAN",
            stage,
            n,
            MIN_POINTS_FOR_HDBSCAN,
        )
        meta["fallback"] = "per_item_labels_no_hdbscan"
        return np.arange(n, dtype=int), meta

    embeddings = _stack_embeddings(rows)
    sentences = [r.embed_text for r in rows]
    meta["used_hdbscan"] = True

    if use_llm_tune:
        if client is None:
            raise RuntimeError(f"{stage} 需要 LLM 客户端")
        labels, nn, tlog, sil, min_cs = _auto_tune_n_neighbors(
            embeddings, sentences, client, stage
        )
        meta["n_neighbors"] = nn
        meta["min_cluster_size"] = min_cs
        meta["max_clusters"] = MAX_CLUSTERS
        meta["tuning_log"] = tlog
        meta["silhouette"] = sil
    else:
        labels, nn, min_cs = _cluster_fixed(embeddings)
        meta["n_neighbors"] = nn
        meta["min_cluster_size"] = min_cs
        meta["max_clusters"] = MAX_CLUSTERS
        meta["fallback"] = "fixed_params"

    return labels, meta


def _init_cluster_db(conn: sqlite3.Connection) -> None:
    conn.executescript(
        """
        CREATE TABLE IF NOT EXISTS cluster_runs (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            created_at TEXT NOT NULL,
            job_id INTEGER NOT NULL,
            embed_db TEXT NOT NULL
        );
        CREATE TABLE IF NOT EXISTS cluster_assignments (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_id INTEGER NOT NULL,
            stage TEXT NOT NULL,
            embedding_item_id INTEGER NOT NULL,
            cluster_label INTEGER NOT NULL,
            is_outlier INTEGER NOT NULL DEFAULT 0,
            participates_downstream INTEGER NOT NULL DEFAULT 1,
            parent_audience_cluster INTEGER,
            n_neighbors INTEGER,
            embed_text TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            source_row INTEGER NOT NULL,
            extraction_id INTEGER NOT NULL,
            entity_index INTEGER NOT NULL,
            audience TEXT,
            aspect TEXT,
            opinion TEXT,
            category TEXT,
            sentiment TEXT,
            content TEXT NOT NULL,
            FOREIGN KEY (run_id) REFERENCES cluster_runs(id)
        );
        CREATE TABLE IF NOT EXISTS cluster_stage_meta (
            run_id INTEGER NOT NULL,
            stage TEXT NOT NULL,
            meta_json TEXT NOT NULL,
            PRIMARY KEY (run_id, stage)
        );
        CREATE TABLE IF NOT EXISTS cluster_tuning_log (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_id INTEGER NOT NULL,
            stage TEXT NOT NULL,
            round_index INTEGER NOT NULL,
            log_json TEXT NOT NULL,
            FOREIGN KEY (run_id) REFERENCES cluster_runs(id)
        );
        CREATE INDEX IF NOT EXISTS idx_assign_run_stage ON cluster_assignments(run_id, stage);
        CREATE INDEX IF NOT EXISTS idx_assign_cluster ON cluster_assignments(run_id, stage, cluster_label);
        CREATE INDEX IF NOT EXISTS idx_assign_source ON cluster_assignments(run_id, source_row);
        """
    )


def _resolve_source_csv(structured_db: Path, job_id: int) -> Path:
    conn = sqlite3.connect(structured_db)
    try:
        row = conn.execute(
            "SELECT source_file FROM analysis_jobs WHERE id = ?", (job_id,)
        ).fetchone()
    finally:
        conn.close()
    if not row:
        raise RuntimeError(f"analysis_jobs 无 job_id={job_id}")
    p = Path(str(row[0]))
    if p.is_file():
        return p.resolve()
    cand = PROJECT_ROOT / p
    if cand.is_file():
        return cand.resolve()
    raise FileNotFoundError(f"找不到结构化来源 CSV: {row[0]}")


def _count_csv_reviews(csv_path: Path) -> int:
    n = 0
    with csv_path.open(encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f)
        for row in reader:
            if (row.get("content") or "").strip():
                n += 1
    return n


def _keep_cluster_labels(
    labels: np.ndarray,
    source_rows: Sequence[int],
    total_reviews: int,
    *,
    min_ratio: float = CLUSTER_MIN_REVIEW_RATIO,
) -> Tuple[set[int], Dict[int, int]]:
    """返回 (保留的簇标签, 被丢弃簇标签 -> 去重评论数)。"""
    by_lab: Dict[int, set[int]] = defaultdict(set)
    for lab, sr in zip(labels.tolist(), source_rows):
        by_lab[int(lab)].add(int(sr))
    if total_reviews <= 0:
        return set(by_lab.keys()), {}
    kept: set[int] = set()
    dropped: Dict[int, int] = {}
    for lab, srs in by_lab.items():
        cnt = len(srs)
        if cnt / total_reviews >= min_ratio:
            kept.add(lab)
        else:
            dropped[lab] = cnt
    return kept, dropped


def _reset_cluster_db(path: Path, *, reset: bool = True) -> sqlite3.Connection:
    if reset and path.is_file():
        path.unlink()
    conn = sqlite3.connect(path)
    _init_cluster_db(conn)
    return conn


def _save_assignments(
    conn: sqlite3.Connection,
    run_id: int,
    stage: str,
    rows: List[EmbedRow],
    labels: np.ndarray,
    *,
    n_neighbors: int | None,
    outlier_participates: bool,
    parent_clusters: List[int | None] | None = None,
) -> dict:
    if parent_clusters is not None and len(parent_clusters) != len(rows):
        raise ValueError("parent_clusters 长度与 rows 不一致")
    source_rows = [r.source_row for r in rows]
    stage_total_reviews = len(set(source_rows))
    kept, dropped = _keep_cluster_labels(labels, source_rows, stage_total_reviews)
    if dropped:
        logger.info(
            "[%s] 过滤小簇（去重评论占比 < %.0f%%，本步骤评论总数 %s）：%s",
            stage,
            CLUSTER_MIN_REVIEW_RATIO * 100,
            stage_total_reviews,
            {
                lab: f"{cnt}条({cnt / stage_total_reviews * 100:.1f}%)"
                for lab, cnt in sorted(dropped.items())
                if stage_total_reviews > 0
            },
        )
    sql = """
        INSERT INTO cluster_assignments (
            run_id, stage, embedding_item_id, cluster_label, is_outlier,
            participates_downstream, parent_audience_cluster, n_neighbors,
            embed_text, entity_type, source_row, extraction_id, entity_index,
            audience, aspect, opinion, category, sentiment, content
        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
    """
    saved = 0
    for i, row in enumerate(rows):
        lab = int(labels[i])
        if lab not in kept:
            continue
        is_out = 1 if lab == -1 else 0
        if is_out:
            participates = 1 if outlier_participates else 0
        else:
            participates = 1
        parent = None
        if parent_clusters is not None:
            parent = parent_clusters[i]
        conn.execute(
            sql,
            (
                run_id,
                stage,
                row.id,
                lab,
                is_out,
                participates,
                parent,
                n_neighbors,
                row.embed_text,
                row.entity_type,
                row.source_row,
                row.extraction_id,
                row.entity_index,
                row.audience,
                row.aspect,
                row.opinion,
                row.category,
                row.sentiment,
                row.content,
            ),
        )
        saved += 1
    return {
        "kept_clusters": sorted(kept),
        "dropped_clusters": dropped,
        "saved_rows": saved,
        "min_review_ratio": CLUSTER_MIN_REVIEW_RATIO,
        "stage_total_reviews": stage_total_reviews,
    }


def _save_stage_meta(conn: sqlite3.Connection, run_id: int, stage: str, meta: dict) -> None:
    conn.execute(
        """
        INSERT OR REPLACE INTO cluster_stage_meta (run_id, stage, meta_json)
        VALUES (?, ?, ?)
        """,
        (run_id, stage, json.dumps(meta, ensure_ascii=False)),
    )


def _save_tuning_logs(
    conn: sqlite3.Connection, run_id: int, stage: str, tuning_log: List[dict]
) -> None:
    for i, entry in enumerate(tuning_log, start=1):
        conn.execute(
            """
            INSERT INTO cluster_tuning_log (run_id, stage, round_index, log_json)
            VALUES (?, ?, ?, ?)
            """,
            (run_id, stage, i, json.dumps(entry, ensure_ascii=False)),
        )


def _top2_audience_clusters(
    conn: sqlite3.Connection, run_id: int
) -> Tuple[List[int], Dict[int, int]]:
    """返回 (前两簇标签列表, source_row -> audience簇标签)。"""
    cur = conn.execute(
        """
        SELECT cluster_label, source_row
        FROM cluster_assignments
        WHERE run_id = ? AND stage = '1_audience'
          AND is_outlier = 0 AND participates_downstream = 1
        """,
        (run_id,),
    )
    row_to_cluster: Dict[int, int] = {}
    cluster_rows: Dict[int, set[int]] = {}
    for lab, sr in cur.fetchall():
        lab = int(lab)
        sr = int(sr)
        row_to_cluster[sr] = lab
        cluster_rows.setdefault(lab, set()).add(sr)
    ranked = sorted(
        cluster_rows.items(), key=lambda x: len(x[1]), reverse=True
    )
    top2 = [lab for lab, _ in ranked[:2]]
    return top2, row_to_cluster


def run_clustering(
    *,
    job_id: int | None = None,
    structured_db: Path = STRUCTURED_DB,
    embed_db: Path = EMBED_DB,
    cluster_db: Path = CLUSTER_DB,
    reset_db: bool = True,
) -> dict:
    require_chat_api_key()

    econn = sqlite3.connect(embed_db)
    try:
        jid = _resolve_job_id(job_id, econn, structured_db)
        audience_rows = _load_embed_rows(econn, jid, "audience")
        pain_rows = _load_embed_rows(econn, jid, "pain_point")
        ao_rows = _load_embed_rows(econn, jid, "aspect_opinion")
    finally:
        econn.close()

    client = create_chat_client()
    cconn = _reset_cluster_db(cluster_db, reset=reset_db)
    created = datetime.now(timezone.utc).isoformat()
    try:
        cur = cconn.execute(
            "INSERT INTO cluster_runs (created_at, job_id, embed_db) VALUES (?,?,?)",
            (created, jid, str(embed_db.resolve())),
        )
        run_id = int(cur.lastrowid)

        csv_path = _resolve_source_csv(structured_db, jid)
        total_reviews = _count_csv_reviews(csv_path)
        logger.info("清洗后评论总数: %s", total_reviews)

        # --- 1 audience ---
        labels1, meta1 = _cluster_stage(
            audience_rows, stage="1_audience", use_llm_tune=True, client=client
        )
        filt1 = _save_assignments(
            cconn,
            run_id,
            "1_audience",
            audience_rows,
            labels1,
            n_neighbors=meta1.get("n_neighbors"),
            outlier_participates=False,
        )
        meta1["cluster_filter"] = filt1
        _save_stage_meta(cconn, run_id, "1_audience", meta1)
        if meta1.get("tuning_log"):
            _save_tuning_logs(cconn, run_id, "1_audience", meta1["tuning_log"])

        top2, row_to_aud = _top2_audience_clusters(cconn, run_id)
        meta_top2 = {
            "top2_audience_clusters": top2,
            "per_cluster_source_rows": {
                str(lab): len(_source_rows_for_audience_cluster(row_to_aud, lab))
                for lab in top2
            },
            "step2_mode": "per_audience_cluster_llm_auto_tune",
        }
        _save_stage_meta(cconn, run_id, "step2_filter", meta_top2)
        logger.info("Audience 前两簇（将分别聚类）: %s", top2)

        step2_counts: Dict[str, int] = {}
        for aud_lab in top2:
            step2_counts.update(
                _cluster_step2_for_audience(
                    cconn,
                    run_id,
                    audience_cluster=aud_lab,
                    pain_rows=pain_rows,
                    ao_rows=ao_rows,
                    row_to_aud=row_to_aud,
                    client=client,
                )
            )

        # --- 3a pain global ---
        labels3a, meta3a = _cluster_stage(
            pain_rows, stage="3a_pain_global", use_llm_tune=True, client=client
        )
        filt3a = _save_assignments(
            cconn,
            run_id,
            "3a_pain_global",
            pain_rows,
            labels3a,
            n_neighbors=meta3a.get("n_neighbors"),
            outlier_participates=True,
        )
        meta3a["cluster_filter"] = filt3a
        _save_stage_meta(cconn, run_id, "3a_pain_global", meta3a)
        if meta3a.get("tuning_log"):
            _save_tuning_logs(cconn, run_id, "3a_pain_global", meta3a["tuning_log"])

        # --- 3b aspect_opinion by sentiment ---
        step3b_counts = _cluster_aspect_opinion_sentiment_stages(
            cconn,
            run_id,
            ao_rows,
            stage_for=_stage_3b,
            outlier_participates=True,
            parent_audience_cluster=None,
            skipped_reason_prefix="no_aspect_opinion",
            client=client,
        )

        cconn.commit()
        summary = {
            "run_id": run_id,
            "job_id": jid,
            "total_reviews": total_reviews,
            "min_cluster_review_ratio": CLUSTER_MIN_REVIEW_RATIO,
            "cluster_db": str(cluster_db.resolve()),
            "top2_audience_clusters": top2,
            "counts": {
                "1_audience": len(audience_rows),
                **step2_counts,
                "3a_pain_global": len(pain_rows),
                **step3b_counts,
            },
        }
        logger.info("聚类完成: %s", summary)
        return summary
    finally:
        cconn.close()


def main() -> None:
    parser = argparse.ArgumentParser(description="VOC 向量聚类")
    parser.add_argument(
        "--job-id",
        type=int,
        default=None,
        help="不指定则自动使用 voc_structured.sqlite 中最新 analysis_jobs.id",
    )
    parser.add_argument("--structured-db", type=Path, default=STRUCTURED_DB)
    parser.add_argument("--embed-db", type=Path, default=EMBED_DB)
    parser.add_argument("--cluster-db", type=Path, default=CLUSTER_DB)
    args = parser.parse_args()
    summary = run_clustering(
        job_id=args.job_id,
        structured_db=args.structured_db,
        embed_db=args.embed_db,
        cluster_db=args.cluster_db,
    )
    print(json.dumps(summary, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()