cb6692c0c0
包含 Persona 锚定聚类、LLM 报告生成、方法论文档及 .gitignore 更新，便于在 Gitee 独立部署。 Co-authored-by: Cursor <cursoragent@cursor.com>
1647 lines
55 KiB
Python
1647 lines
55 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
对「推理预测_明细_业务.csv」按词库指标列做分位数筛选。
|
||
|
||
示例：
|
||
# 查看 22 个可筛列的序号对照表
|
||
python3 特征工程_建模/filter_infer_business.py --list-cols
|
||
|
||
# 分位数：前 30%（默认 -p 30 --cols 1,3,9 --html-report）
|
||
-p 30 --cols 1，3，9
|
||
|
||
# 仅指定输入即可（使用上述默认参数）
|
||
-i .../asin_推理预测_明细_业务.csv
|
||
|
||
# 临界值：与 --cols 一一对应（max 列 >= 临界值，min 列 <= 临界值，含边界）
|
||
--cols 1，3，9 --thresholds 500,0.05,1.2
|
||
|
||
# cols 分隔符：英文逗号、中文逗号、空格均可混用
|
||
--cols "1，8 9"
|
||
--cols "1:min,9:max" # 覆盖默认方向
|
||
|
||
# 逐列串联（更严）
|
||
--logic seq
|
||
|
||
# 强制保留核心词（与分位数结果取并集）
|
||
python3 特征工程_建模/filter_infer_business.py -i ...csv -p 20 --cols 1,8,9 \\
|
||
--keep-keywords "turkey tail"
|
||
|
||
python3 特征工程_建模/filter_infer_business.py \
|
||
-i "/Users/onesvmwhoops/Cursor_Project/选品:爬数据/特征工程_建模/输出结果/B0G528HRJM/模型结果/B0G528HRJM_推理预测_明细_业务.xlsx"\
|
||
-p 30 --cols 1,3,9 \
|
||
--html-report
|
||
|
||
# 筛选后生成词频 HTML（词频基于输入明细_业务表；默认 stem/lemma 并族，加 --freq-synonyms 启用 WordNet 同义词）
|
||
python3 特征工程_建模/filter_infer_business.py -i '.../B0G528HRJM_推理预测_明细_业务.xlsx' -p 30 --cols 1，3，9 --html-report
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import html
|
||
import io
|
||
import json
|
||
import math
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
from typing import Literal
|
||
|
||
import pandas as pd
|
||
|
||
Direction = Literal["max", "min"]
|
||
Logic = Literal["and", "seq"]
|
||
|
||
# (列名, 默认方向)：越大越好 max，越小越好 min
|
||
FILTERABLE_COLUMNS: list[tuple[str, Direction]] = [
|
||
("周搜索量", "max"),
|
||
("周点击量", "max"),
|
||
("周点击率", "max"),
|
||
("曝光点击率", "max"),
|
||
("周销售量", "max"),
|
||
("搜索转化率", "max"),
|
||
("点击转化率", "max"),
|
||
("CPC竞价-最低", "min"),
|
||
("CPC竞价-平均", "min"),
|
||
("CPC竞价-最高", "min"),
|
||
("目标排位建议-90%", "min"),
|
||
("目标排位建议-50%", "min"),
|
||
("CPR", "min"),
|
||
("ABA排名", "min"),
|
||
("ABA排名涨跌幅", "max"),
|
||
("关键词产品数", "min"),
|
||
("推广CPA", "min"),
|
||
("推广成本", "min"),
|
||
("广告点击转化率", "max"),
|
||
("TOP产品的均价", "min"),
|
||
("ABA TOP 3 ASIN点击占比", "min"),
|
||
("ABA TOP 3 ASIN转化占比", "min"),
|
||
]
|
||
|
||
_COLS_SPLIT_RE = re.compile(r"[,，\s]+")
|
||
_KEYWORD_COL_CANDIDATES = ("keyword", "关键词")
|
||
|
||
|
||
def _pip_install(package: str) -> None:
|
||
print(f"[依赖] 正在安装 {package} …", flush=True)
|
||
subprocess.check_call(
|
||
[sys.executable, "-m", "pip", "install", package, "-q"],
|
||
stdout=subprocess.DEVNULL,
|
||
)
|
||
|
||
|
||
def _ensure_nltk():
|
||
try:
|
||
import nltk
|
||
except ImportError:
|
||
_pip_install("nltk")
|
||
import nltk # noqa: F401
|
||
import nltk
|
||
from nltk.corpus import wordnet as wn
|
||
from nltk.stem import PorterStemmer, WordNetLemmatizer
|
||
from nltk.tag import pos_tag
|
||
from nltk.tokenize import word_tokenize
|
||
|
||
for resource, pkg in (
|
||
("tokenizers/punkt", "punkt"),
|
||
("tokenizers/punkt_tab", "punkt_tab"),
|
||
("corpora/wordnet", "wordnet"),
|
||
("corpora/omw-1.4", "omw-1.4"),
|
||
("taggers/averaged_perceptron_tagger", "averaged_perceptron_tagger"),
|
||
("taggers/averaged_perceptron_tagger_eng", "averaged_perceptron_tagger_eng"),
|
||
("corpora/stopwords", "stopwords"),
|
||
):
|
||
try:
|
||
nltk.data.find(resource)
|
||
except LookupError:
|
||
print(f"[依赖] 正在下载 NLTK 数据 {pkg} …", flush=True)
|
||
nltk.download(pkg, quiet=True)
|
||
|
||
return word_tokenize, pos_tag, WordNetLemmatizer(), PorterStemmer(), wn
|
||
|
||
|
||
def _penn_to_wn_pos(tag: str, wn) -> str:
|
||
if tag.startswith("J"):
|
||
return wn.ADJ
|
||
if tag.startswith("V"):
|
||
return wn.VERB
|
||
if tag.startswith("N"):
|
||
return wn.NOUN
|
||
if tag.startswith("R"):
|
||
return wn.ADV
|
||
return wn.NOUN
|
||
|
||
|
||
def parse_keep_keywords(spec: str) -> list[str]:
|
||
text = (spec or "").strip()
|
||
if not text:
|
||
return []
|
||
seen: set[str] = set()
|
||
out: list[str] = []
|
||
for part in _COLS_SPLIT_RE.split(text):
|
||
w = part.strip().lower()
|
||
if w and w not in seen:
|
||
seen.add(w)
|
||
out.append(w)
|
||
return out
|
||
|
||
|
||
def resolve_keyword_column(df: pd.DataFrame) -> str:
|
||
for col in _KEYWORD_COL_CANDIDATES:
|
||
if col in df.columns:
|
||
return col
|
||
raise ValueError(
|
||
f"输入表缺少关键词列（需要其一：{_KEYWORD_COL_CANDIDATES}）"
|
||
)
|
||
|
||
|
||
_CLICKS_COL = "周点击量"
|
||
DEFAULT_FILTER_PCT = 30.0
|
||
DEFAULT_FILTER_COLS = "1,3,9"
|
||
_HTML_TOP_DEFAULT = 100
|
||
# 输入明细_业务表：全量词库相对排序前 20%（与 train_eval 业务明细一致）
|
||
_HTML_LIBRARY_RANK_PCT = 20.0
|
||
# 相对排序表展示：在前 10% 池中再取 rank_pred 前 10%
|
||
_HTML_RANK_PRED_PCT = 10.0
|
||
|
||
# 英文停用词：NLTK english + 常见无分析价值 token
|
||
_EXTRA_STOP_WORDS = frozenset(
|
||
{
|
||
"www",
|
||
"http",
|
||
"https",
|
||
"com",
|
||
"amazon",
|
||
"asin",
|
||
"sku",
|
||
"oz",
|
||
"lb",
|
||
"lbs",
|
||
"inch",
|
||
"inches",
|
||
"ft",
|
||
"mm",
|
||
"cm",
|
||
"ml",
|
||
"kg",
|
||
"pcs",
|
||
"pc",
|
||
}
|
||
)
|
||
|
||
|
||
def load_stop_words() -> set[str]:
|
||
_ensure_nltk()
|
||
import nltk
|
||
from nltk.corpus import stopwords
|
||
|
||
try:
|
||
words = set(stopwords.words("english"))
|
||
except LookupError:
|
||
nltk.download("stopwords", quiet=True)
|
||
words = set(stopwords.words("english"))
|
||
words |= _EXTRA_STOP_WORDS
|
||
return words
|
||
|
||
|
||
class WordVariantEngine:
|
||
"""NLTK 分词 + 词形变体（stem / POS+lemma / 可选 WordNet 同义词）。"""
|
||
|
||
def __init__(self) -> None:
|
||
word_tokenize, pos_tag, lemmatizer, stemmer, wn = _ensure_nltk()
|
||
self._word_tokenize = word_tokenize
|
||
self._pos_tag = pos_tag
|
||
self._lemmatizer = lemmatizer
|
||
self._stemmer = stemmer
|
||
self._wn = wn
|
||
self._synonym_pool_cache: dict[str, set[str]] = {}
|
||
|
||
def variants_for_word(self, word: str, pos: str | None = None) -> set[str]:
|
||
w = (word or "").lower().strip()
|
||
if not w:
|
||
return set()
|
||
out: set[str] = {w, self._stemmer.stem(w)}
|
||
if pos is not None:
|
||
wn_pos = _penn_to_wn_pos(pos, self._wn)
|
||
out.add(self._lemmatizer.lemmatize(w, pos=wn_pos))
|
||
for p in (self._wn.NOUN, self._wn.VERB, self._wn.ADJ, self._wn.ADV):
|
||
out.add(self._lemmatizer.lemmatize(w, pos=p))
|
||
return {x for x in out if x}
|
||
|
||
def _synonym_variant_pool(self, word: str) -> set[str]:
|
||
w = (word or "").lower().strip()
|
||
if not w:
|
||
return set()
|
||
if w in self._synonym_pool_cache:
|
||
return self._synonym_pool_cache[w]
|
||
extra: set[str] = set()
|
||
for syn in self._wn.synsets(w):
|
||
for lemma in syn.lemmas():
|
||
name = lemma.name().replace("_", " ").lower()
|
||
if name:
|
||
extra |= self.variants_for_word(name, pos=None)
|
||
self._synonym_pool_cache[w] = extra
|
||
return extra
|
||
|
||
def variant_pool(
|
||
self, word: str, pos: str | None = None, *, include_synonyms: bool = False
|
||
) -> set[str]:
|
||
"""stem + lemma；include_synonyms 时再并入 WordNet 同义词 lemma/stem。"""
|
||
pool = self.variants_for_word(word, pos=pos)
|
||
if include_synonyms:
|
||
pool |= self._synonym_variant_pool(word)
|
||
return pool
|
||
|
||
def tokenize_tagged(self, text: str) -> list[tuple[str, str]]:
|
||
try:
|
||
tokens = self._word_tokenize(str(text).lower())
|
||
return self._pos_tag(tokens)
|
||
except Exception:
|
||
tokens = re.findall(r"[a-z0-9']+", str(text).lower())
|
||
return [(t, "NN") for t in tokens]
|
||
|
||
def text_variant_set(self, text: str) -> set[str]:
|
||
variants: set[str] = set()
|
||
for tok, tag in self.tokenize_tagged(text):
|
||
if tok:
|
||
variants |= self.variants_for_word(tok, pos=tag)
|
||
return variants
|
||
|
||
|
||
class _UnionFind:
|
||
def __init__(self) -> None:
|
||
self._parent: dict[str, str] = {}
|
||
|
||
def add(self, x: str) -> None:
|
||
if x not in self._parent:
|
||
self._parent[x] = x
|
||
|
||
def find(self, x: str) -> str:
|
||
self.add(x)
|
||
while self._parent[x] != x:
|
||
self._parent[x] = self._parent[self._parent[x]]
|
||
x = self._parent[x]
|
||
return x
|
||
|
||
def union(self, a: str, b: str) -> None:
|
||
ra, rb = self.find(a), self.find(b)
|
||
if ra != rb:
|
||
self._parent[rb] = ra
|
||
|
||
def groups(self) -> dict[str, list[str]]:
|
||
out: dict[str, list[str]] = defaultdict(list)
|
||
for x in self._parent:
|
||
out[self.find(x)].append(x)
|
||
return dict(out)
|
||
|
||
|
||
def build_word_freq_groups(
|
||
df: pd.DataFrame,
|
||
*,
|
||
top_n: int = _HTML_TOP_DEFAULT,
|
||
stop_words: set[str] | None = None,
|
||
include_synonyms: bool = False,
|
||
) -> list[dict]:
|
||
"""拆词 → 停用词过滤 → Union-Find 合并词族（stem/lemma/可选同义词）→ 按周点击量 Top N。"""
|
||
if df.empty:
|
||
return []
|
||
|
||
kw_col = resolve_keyword_column(df)
|
||
if _CLICKS_COL not in df.columns:
|
||
raise ValueError(f"输入表缺少列 {_CLICKS_COL!r}，无法生成词频报告")
|
||
|
||
stops = stop_words if stop_words is not None else load_stop_words()
|
||
engine = WordVariantEngine()
|
||
uf = _UnionFind()
|
||
surface_clicks: dict[str, float] = defaultdict(float)
|
||
variant_index: dict[str, set[str]] = defaultdict(set)
|
||
|
||
for _, row in df.iterrows():
|
||
kw = row[kw_col]
|
||
if pd.isna(kw) or not str(kw).strip():
|
||
continue
|
||
clicks = pd.to_numeric(row[_CLICKS_COL], errors="coerce")
|
||
if pd.isna(clicks):
|
||
clicks = 0.0
|
||
clicks_f = float(clicks)
|
||
|
||
seen_in_row: set[str] = set()
|
||
for tok, tag in engine.tokenize_tagged(str(kw)):
|
||
if not tok or len(tok) < 1 or tok in stops or not re.search(r"[a-z]", tok):
|
||
continue
|
||
if tok in seen_in_row:
|
||
continue
|
||
seen_in_row.add(tok)
|
||
uf.add(tok)
|
||
surface_clicks[tok] += clicks_f
|
||
variants = engine.variant_pool(tok, pos=tag, include_synonyms=include_synonyms)
|
||
related: set[str] = set()
|
||
for v in variants:
|
||
related |= variant_index[v]
|
||
for other in related:
|
||
uf.union(tok, other)
|
||
for v in variants:
|
||
variant_index[v].add(tok)
|
||
|
||
raw_groups = uf.groups()
|
||
families: list[dict] = []
|
||
for members in raw_groups.values():
|
||
members_sorted = sorted(members)
|
||
total = sum(surface_clicks[m] for m in members_sorted)
|
||
member_details = [
|
||
{"surface": m, "clicks": round(surface_clicks[m], 2)}
|
||
for m in sorted(members_sorted, key=lambda x: (-surface_clicks[x], x))
|
||
]
|
||
label = member_details[0]["surface"] if member_details else members_sorted[0]
|
||
families.append(
|
||
{
|
||
"label": label,
|
||
"total": round(total, 2),
|
||
"members": member_details,
|
||
}
|
||
)
|
||
|
||
families.sort(key=lambda x: (-x["total"], x["label"]))
|
||
return families[: max(1, int(top_n))]
|
||
|
||
|
||
def default_html_report_path(output_table_path: Path, *paths: Path | None) -> Path:
|
||
title = infer_html_report_title(*paths, output_table_path)
|
||
return output_table_path.with_name(f"{title}.html")
|
||
|
||
|
||
_ASIN_STEM_RE = re.compile(r"^(B0[A-Z0-9]{8,})", re.I)
|
||
|
||
|
||
def infer_html_report_title(*paths: Path | None) -> str:
|
||
"""从输出/输入路径提取 ASIN，生成如 B0G528HRJM_广告词推荐。"""
|
||
for path in paths:
|
||
if path is None:
|
||
continue
|
||
stem = path.stem.replace("_筛选", "").replace("_词频", "").replace("_广告词推荐", "")
|
||
m = _ASIN_STEM_RE.match(stem)
|
||
if m:
|
||
return f"{m.group(1).upper()}_广告词推荐"
|
||
return "asin_广告词推荐"
|
||
|
||
|
||
def filter_top_rank_pred(
|
||
df: pd.DataFrame,
|
||
top_pct: float,
|
||
*,
|
||
by_model: bool = True,
|
||
) -> pd.DataFrame:
|
||
"""按 model（若有）保留 rank_pred 最小（预测最优）的前 top_pct% 行。"""
|
||
if df.empty or "rank_pred" not in df.columns:
|
||
return df.iloc[0:0].copy()
|
||
pct = max(0.0, min(100.0, float(top_pct)))
|
||
if by_model and "model" in df.columns:
|
||
parts: list[pd.DataFrame] = []
|
||
for _, group in df.groupby("model", sort=False):
|
||
k = max(1, int(math.ceil(len(group) * pct / 100.0)))
|
||
parts.append(group.nsmallest(k, "rank_pred"))
|
||
return pd.concat(parts, ignore_index=True)
|
||
k = max(1, int(math.ceil(len(df) * pct / 100.0)))
|
||
return df.nsmallest(k, "rank_pred").copy()
|
||
|
||
|
||
def build_filter_tooltip(
|
||
col_specs: list[tuple[str, Direction]],
|
||
*,
|
||
pct: float | None = None,
|
||
thresholds: list[float] | None = None,
|
||
logic: str = "and",
|
||
) -> str:
|
||
pool = f"全量词库相对排序前 {_HTML_LIBRARY_RANK_PCT:g}% 的关键词"
|
||
if thresholds is not None:
|
||
spec = _format_col_threshold_specs(col_specs, thresholds)
|
||
return f"在{pool}中，按临界值筛选：{spec}（logic={logic}）"
|
||
parts: list[str] = []
|
||
p = pct if pct is not None else 20.0
|
||
for name, direction in col_specs:
|
||
hint = "越大越好" if direction == "max" else "越小越好"
|
||
parts.append(f"{name}（{hint}，前 {p:g}%）")
|
||
joiner = " 且 " if logic == "and" else " → "
|
||
return f"1️⃣在{pool}中，按指标筛选：{joiner.join(parts)}"
|
||
|
||
|
||
def _dataframe_to_download_bytes(df: pd.DataFrame, suffix: str) -> bytes:
|
||
ext = suffix.lower()
|
||
if ext in (".xlsx", ".xls"):
|
||
buf = io.BytesIO()
|
||
df.to_excel(buf, index=False, engine="openpyxl")
|
||
return buf.getvalue()
|
||
return df.to_csv(index=False, encoding="utf-8-sig").encode("utf-8-sig")
|
||
|
||
|
||
def _download_mime(suffix: str) -> str:
|
||
ext = suffix.lower()
|
||
if ext == ".xlsx":
|
||
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||
if ext == ".xls":
|
||
return "application/vnd.ms-excel"
|
||
return "text/csv"
|
||
|
||
|
||
def _serialize_table_for_html(df: pd.DataFrame) -> str:
|
||
"""将 DataFrame 序列化为 HTML 内嵌 JSON（供可排序表格渲染）。"""
|
||
out = df.copy()
|
||
for col in out.columns:
|
||
if pd.api.types.is_datetime64_any_dtype(out[col]):
|
||
out[col] = out[col].astype(str)
|
||
out = out.where(pd.notna(out), None)
|
||
payload = {
|
||
"columns": [str(c) for c in out.columns],
|
||
"rows": out.values.tolist(),
|
||
}
|
||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||
|
||
|
||
def render_word_freq_html(
|
||
groups: list[dict],
|
||
*,
|
||
embed_path: Path,
|
||
title: str,
|
||
row_count: int,
|
||
filtered_df: pd.DataFrame | None = None,
|
||
original_rank_df: pd.DataFrame | None = None,
|
||
input_download_df: pd.DataFrame | None = None,
|
||
filter_tooltip: str = "",
|
||
) -> str:
|
||
"""单文件 HTML：词频图 + 词频/原始/筛选表格 + 底部双下载。"""
|
||
suffix = embed_path.suffix.lower()
|
||
out_df = filtered_df if filtered_df is not None else pd.DataFrame()
|
||
out_bytes = _dataframe_to_download_bytes(out_df, suffix)
|
||
out_mime = _download_mime(suffix)
|
||
out_b64 = base64.b64encode(out_bytes).decode("ascii")
|
||
out_download_name = html.escape(embed_path.name)
|
||
|
||
in_df = input_download_df if input_download_df is not None else pd.DataFrame()
|
||
in_bytes = _dataframe_to_download_bytes(in_df, suffix)
|
||
in_b64 = base64.b64encode(in_bytes).decode("ascii")
|
||
in_stem = embed_path.stem.replace("_筛选", "")
|
||
in_download_name = html.escape(
|
||
f"{in_stem}_原始相对排序前20%{embed_path.suffix}"
|
||
)
|
||
|
||
input_tooltip = html.escape(
|
||
f"全量词库相对排序前 {_HTML_RANK_PRED_PCT:g}%（，"
|
||
f"取 相对排序 最优前 {_HTML_RANK_PRED_PCT:g}%）"
|
||
)
|
||
filter_tooltip_esc = html.escape(filter_tooltip or "按指定指标筛选")
|
||
|
||
chart_data = json.dumps(groups, ensure_ascii=False)
|
||
labels = [g["label"] for g in groups]
|
||
totals = [g["total"] for g in groups]
|
||
displayed = len(groups)
|
||
title_esc = html.escape(title)
|
||
filtered_json = (
|
||
_serialize_table_for_html(filtered_df)
|
||
if filtered_df is not None
|
||
else '{"columns":[],"rows":[]}'
|
||
)
|
||
original_json = (
|
||
_serialize_table_for_html(original_rank_df)
|
||
if original_rank_df is not None
|
||
else '{"columns":[],"rows":[]}'
|
||
)
|
||
return f"""<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="utf-8"/>
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<title>{title_esc}</title>
|
||
<script src="https://cdn.jsdelivr.net/npm/echarts@5/dist/echarts.min.js"></script>
|
||
<style>
|
||
:root {{
|
||
--bg: #f0f4f8;
|
||
--card: #ffffff;
|
||
--text: #1f2937;
|
||
--muted: #6b7280;
|
||
--border: #e5e7eb;
|
||
--primary: #2563eb;
|
||
--primary-dark: #1d4ed8;
|
||
--primary-soft: #eff6ff;
|
||
--accent-freq: #7c3aed;
|
||
--accent-freq-soft: #f5f3ff;
|
||
--accent-rank: #d97706;
|
||
--accent-rank-soft: #fffbeb;
|
||
--success: #059669;
|
||
--shadow: 0 4px 24px rgba(15, 23, 42, 0.06);
|
||
--radius: 12px;
|
||
}}
|
||
* {{ box-sizing: border-box; }}
|
||
body {{
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", sans-serif;
|
||
margin: 0;
|
||
padding: 24px 16px 48px;
|
||
color: var(--text);
|
||
background: linear-gradient(160deg, #eef2ff 0%, var(--bg) 40%, #f8fafc 100%);
|
||
line-height: 1.5;
|
||
}}
|
||
.page {{ max-width: 1200px; margin: 0 auto; }}
|
||
.page-header {{
|
||
background: linear-gradient(135deg, #1e40af 0%, #2563eb 55%, #3b82f6 100%);
|
||
color: #fff;
|
||
padding: 28px 32px;
|
||
border-radius: var(--radius);
|
||
box-shadow: var(--shadow);
|
||
margin-bottom: 24px;
|
||
}}
|
||
.page-header h1 {{
|
||
font-size: 1.5rem;
|
||
font-weight: 700;
|
||
margin: 0 0 6px;
|
||
letter-spacing: 0.02em;
|
||
}}
|
||
.page-header .subtitle {{
|
||
font-size: 0.9rem;
|
||
opacity: 0.88;
|
||
margin: 0;
|
||
}}
|
||
.section-card {{
|
||
background: var(--card);
|
||
border-radius: var(--radius);
|
||
box-shadow: var(--shadow);
|
||
padding: 24px 28px;
|
||
margin-bottom: 24px;
|
||
border: 1px solid var(--border);
|
||
border-left: 4px solid var(--section-accent, var(--primary));
|
||
}}
|
||
.section-filtered {{ --section-accent: var(--primary); }}
|
||
.section-freq {{ --section-accent: var(--accent-freq); }}
|
||
.section-rank {{ --section-accent: var(--accent-rank); }}
|
||
h2 {{
|
||
font-size: 1.08rem;
|
||
font-weight: 600;
|
||
margin: 0 0 12px;
|
||
color: var(--text);
|
||
display: flex;
|
||
align-items: center;
|
||
flex-wrap: wrap;
|
||
gap: 8px;
|
||
}}
|
||
h2 .sub {{
|
||
font-size: 0.95rem;
|
||
font-weight: 500;
|
||
color: var(--muted);
|
||
}}
|
||
.badge {{
|
||
display: inline-block;
|
||
font-size: 0.75rem;
|
||
font-weight: 600;
|
||
padding: 2px 10px;
|
||
border-radius: 999px;
|
||
background: var(--primary-soft);
|
||
color: var(--primary);
|
||
}}
|
||
.section-freq .badge {{ background: var(--accent-freq-soft); color: var(--accent-freq); }}
|
||
.section-rank .badge {{ background: var(--accent-rank-soft); color: var(--accent-rank); }}
|
||
.chart-wrap {{
|
||
margin: 16px 0 8px;
|
||
padding: 16px;
|
||
background: linear-gradient(180deg, var(--accent-freq-soft) 0%, #fff 100%);
|
||
border-radius: 10px;
|
||
border: 1px solid #ede9fe;
|
||
overflow: visible;
|
||
}}
|
||
#chart {{ width: 100%; height: 520px; min-height: 520px; }}
|
||
.note {{
|
||
color: var(--muted);
|
||
font-size: 0.84rem;
|
||
margin: 8px 0 0;
|
||
}}
|
||
.note.filter-spec {{
|
||
color: #374151;
|
||
font-size: 0.88rem;
|
||
margin: 6px 0;
|
||
padding: 10px 14px;
|
||
background: #f9fafb;
|
||
border-radius: 8px;
|
||
border-left: 3px solid #d1d5db;
|
||
}}
|
||
.section-filtered .note.filter-spec {{ border-left-color: var(--primary); background: var(--primary-soft); }}
|
||
.section-freq .note.filter-spec {{ border-left-color: var(--accent-freq); background: var(--accent-freq-soft); }}
|
||
.section-rank .note.filter-spec {{
|
||
border-left-color: var(--accent-rank);
|
||
background: var(--accent-rank-soft);
|
||
}}
|
||
.section-rank .note.filter-spec .warn-icon {{
|
||
color: #ea580c;
|
||
font-family: "Apple Color Emoji", "Segoe UI Emoji", sans-serif;
|
||
}}
|
||
.table-wrap {{
|
||
max-height: 480px;
|
||
overflow: auto;
|
||
border: 1px solid var(--border);
|
||
border-radius: 10px;
|
||
background: #fff;
|
||
margin-top: 12px;
|
||
}}
|
||
table.data-table {{
|
||
width: max-content;
|
||
min-width: 100%;
|
||
border-collapse: collapse;
|
||
font-size: 0.82rem;
|
||
}}
|
||
table.data-table thead th {{
|
||
position: sticky;
|
||
top: 0;
|
||
z-index: 2;
|
||
background: linear-gradient(180deg, #f8fafc 0%, #f1f5f9 100%);
|
||
border-bottom: 2px solid var(--border);
|
||
padding: 10px 12px;
|
||
text-align: left;
|
||
white-space: nowrap;
|
||
cursor: pointer;
|
||
user-select: none;
|
||
color: #334155;
|
||
font-weight: 600;
|
||
}}
|
||
table.data-table thead th:hover {{ background: #e2e8f0; }}
|
||
table.data-table thead th .sort-icon {{ color: var(--primary); margin-left: 4px; font-size: 0.75rem; }}
|
||
table.data-table tbody td {{
|
||
border-bottom: 1px solid #f1f5f9;
|
||
padding: 8px 12px;
|
||
white-space: nowrap;
|
||
max-width: 320px;
|
||
overflow: hidden;
|
||
text-overflow: ellipsis;
|
||
}}
|
||
table.data-table tbody tr:hover {{ background: #eff6ff !important; }}
|
||
table.data-table tbody tr:nth-child(even) {{ background: #fafbfc; }}
|
||
table.freq-table thead th {{ cursor: default; background: linear-gradient(180deg, #f5f3ff 0%, #ede9fe 100%); }}
|
||
table.freq-table thead th:hover {{ background: #ede9fe; }}
|
||
table.freq-table td.num {{ text-align: right; font-variant-numeric: tabular-nums; font-weight: 500; }}
|
||
table.freq-table td.members {{ max-width: 480px; white-space: normal; color: #4b5563; }}
|
||
table.freq-table tbody tr:nth-child(1) td:first-child {{ color: #ca8a04; font-weight: 700; }}
|
||
table.freq-table tbody tr:nth-child(2) td:first-child {{ color: #94a3b8; font-weight: 700; }}
|
||
table.freq-table tbody tr:nth-child(3) td:first-child {{ color: #b45309; font-weight: 700; }}
|
||
.download-panel {{
|
||
background: var(--card);
|
||
border-radius: var(--radius);
|
||
box-shadow: var(--shadow);
|
||
padding: 24px 28px;
|
||
border: 1px solid var(--border);
|
||
margin-top: 8px;
|
||
}}
|
||
.download-panel h3 {{
|
||
margin: 0 0 14px;
|
||
font-size: 1rem;
|
||
color: var(--text);
|
||
}}
|
||
.downloads {{ display: flex; flex-wrap: wrap; gap: 12px; }}
|
||
.downloads a {{
|
||
display: inline-flex;
|
||
align-items: center;
|
||
gap: 6px;
|
||
padding: 11px 20px;
|
||
text-decoration: none;
|
||
border-radius: 8px;
|
||
font-size: 0.9rem;
|
||
font-weight: 500;
|
||
transition: transform 0.15s, box-shadow 0.15s;
|
||
}}
|
||
.downloads a:hover {{ transform: translateY(-1px); box-shadow: 0 4px 12px rgba(37, 99, 235, 0.25); }}
|
||
.downloads a.btn-primary {{
|
||
background: linear-gradient(135deg, var(--primary) 0%, #3b82f6 100%);
|
||
color: #fff;
|
||
}}
|
||
.downloads a.btn-secondary {{
|
||
background: #fff;
|
||
color: var(--primary);
|
||
border: 1.5px solid #93c5fd;
|
||
}}
|
||
.downloads a.btn-secondary:hover {{ background: var(--primary-soft); }}
|
||
.footer-note {{ text-align: center; margin-top: 16px; font-size: 0.8rem; color: var(--muted); }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="page">
|
||
<header class="page-header">
|
||
<h1>{title_esc}</h1>
|
||
<p class="subtitle">广告关键词筛选 · 词频分析 · 相对排序推荐</p>
|
||
</header>
|
||
<section class="section-card section-filtered" id="section-filtered">
|
||
<h2>按指定指标筛选结果 <span class="badge">{row_count} 行</span></h2>
|
||
<p class="note filter-spec">{filter_tooltip_esc}</p>
|
||
<p class="note filter-spec">2️⃣「搜索量大、点击率高、平均 CPC 低、相对排序靠前」，分别说明「曝光量高、点击量高、市场竞争小、转化率高」的可能性较大。</p>
|
||
<p class="note">点击表头可排序；表格区域可上下滚动浏览全部行。</p>
|
||
<div class="table-wrap">
|
||
<table class="data-table" id="filtered-table">
|
||
<thead id="filtered-thead"></thead>
|
||
<tbody id="filtered-tbody"></tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
<section class="section-card section-freq" id="section-freq">
|
||
<h2>词频统计</h2>
|
||
<p class="note filter-spec">统计对象：全量词库相对排序前 {_HTML_LIBRARY_RANK_PCT:g}% 的关键词（按周点击量汇总词频，可查看词族内各原词形及对应周点击量，高点击的词族更值得关注）</p>
|
||
<div class="chart-wrap">
|
||
<div id="chart"></div>
|
||
</div>
|
||
<p class="note">柱图可左右拖动/滚轮缩放；悬停柱体可查看词族内各原词形及对应周点击量。</p>
|
||
<h2>词频明细 <span class="badge">{displayed} 个词族</span></h2>
|
||
<p class="note">按周点击量合计降序；表格区域可上下滚动浏览全部词族。</p>
|
||
<div class="table-wrap">
|
||
<table class="data-table freq-table" id="freq-table">
|
||
<thead>
|
||
<tr>
|
||
<th>#</th>
|
||
<th>词族</th>
|
||
<th>周点击量合计</th>
|
||
<th>词形数</th>
|
||
<th>词形明细</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody id="freq-tbody"></tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
<section class="section-card section-rank" id="section-rank">
|
||
<h2>全量词库相对排序前 {_HTML_RANK_PRED_PCT:g}% 关键词展示</h2>
|
||
<p class="note filter-spec">在全量词库相对排序里前 {_HTML_RANK_PRED_PCT:g}% 的关键词，这些关键词的点击转化率较高的可能性较大。<span class="warn-icon">⚠️</span>防止遗漏不满足筛选条件的高转化小词</p>
|
||
<p class="note">点击表头可排序，可上下滚动。</p>
|
||
<div class="table-wrap">
|
||
<table class="data-table" id="original-table">
|
||
<thead id="original-thead"></thead>
|
||
<tbody id="original-tbody"></tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
<div class="download-panel">
|
||
<h3>数据下载</h3>
|
||
<div class="downloads">
|
||
<a id="dl-input" class="btn-secondary" download="{in_download_name}" title="{input_tooltip}">📥 原始相对排序前20%结果</a>
|
||
<a id="dl-output" class="btn-primary" download="{out_download_name}" title="{filter_tooltip_esc}">📥 按指定指标筛选结果</a>
|
||
</div>
|
||
<p class="footer-note">悬停下载按钮可查看筛选规则；表格数据与下载文件已内嵌于本 HTML，可离线打开。</p>
|
||
</div>
|
||
</div>
|
||
<script>
|
||
const chartData = {chart_data};
|
||
const labels = {json.dumps(labels, ensure_ascii=False)};
|
||
const totals = {json.dumps(totals)};
|
||
const originalTablePayload = {original_json};
|
||
const filteredTablePayload = {filtered_json};
|
||
const inB64 = "{in_b64}";
|
||
const outB64 = "{out_b64}";
|
||
const inMime = "{out_mime}";
|
||
const outMime = "{out_mime}";
|
||
document.getElementById("dl-input").href = "data:" + inMime + ";base64," + inB64;
|
||
document.getElementById("dl-output").href = "data:" + outMime + ";base64," + outB64;
|
||
|
||
function tooltipHtml(idx) {{
|
||
const g = chartData[idx];
|
||
if (!g) return "";
|
||
let rows = g.members.map(m =>
|
||
`<tr><td>${{m.surface}}</td><td style="text-align:right">${{m.clicks.toLocaleString()}}</td></tr>`
|
||
).join("");
|
||
return `<div style="font-size:13px;line-height:1.5">
|
||
<b>${{g.label}}</b> 合计 <b>${{g.total.toLocaleString()}}</b><br/>
|
||
<table style="margin-top:6px;border-collapse:collapse">
|
||
<tr><th style="text-align:left;padding-right:12px">词形</th><th>周点击量</th></tr>${{rows}}
|
||
</table></div>`;
|
||
}}
|
||
|
||
const chart = echarts.init(document.getElementById("chart"));
|
||
const initialEnd = labels.length <= 25 ? 100 : Math.round(25 / labels.length * 100);
|
||
chart.setOption({{
|
||
tooltip: {{
|
||
trigger: "axis",
|
||
axisPointer: {{ type: "shadow" }},
|
||
formatter: params => {{
|
||
const p = params[0];
|
||
return tooltipHtml(p.dataIndex);
|
||
}}
|
||
}},
|
||
grid: {{ left: 16, right: 24, bottom: 96, top: 48, containLabel: true }},
|
||
dataZoom: labels.length > 25 ? [
|
||
{{ type: "slider", xAxisIndex: 0, start: 0, end: initialEnd, height: 22, bottom: 8,
|
||
borderColor: "#c4b5fd", fillerColor: "rgba(124, 58, 237, 0.15)", handleStyle: {{ color: "#7c3aed" }} }},
|
||
{{ type: "inside", xAxisIndex: 0, start: 0, end: initialEnd }}
|
||
] : [],
|
||
xAxis: {{
|
||
type: "category",
|
||
data: labels,
|
||
axisLabel: {{ rotate: 35, interval: 0, color: "#64748b", fontSize: 11 }},
|
||
axisLine: {{ lineStyle: {{ color: "#e2e8f0" }} }}
|
||
}},
|
||
yAxis: {{
|
||
type: "value",
|
||
name: "周点击量合计",
|
||
nameLocation: "end",
|
||
nameGap: 10,
|
||
nameTextStyle: {{ fontSize: 12, align: "left", color: "#64748b" }},
|
||
splitLine: {{ lineStyle: {{ color: "#f1f5f9", type: "dashed" }} }},
|
||
axisLabel: {{ color: "#64748b" }}
|
||
}},
|
||
series: [{{
|
||
type: "bar",
|
||
data: totals,
|
||
itemStyle: {{
|
||
borderRadius: [4, 4, 0, 0],
|
||
color: {{
|
||
type: "linear",
|
||
x: 0, y: 0, x2: 0, y2: 1,
|
||
colorStops: [
|
||
{{ offset: 0, color: "#8b5cf6" }},
|
||
{{ offset: 1, color: "#6366f1" }}
|
||
]
|
||
}}
|
||
}},
|
||
emphasis: {{
|
||
itemStyle: {{
|
||
color: {{
|
||
type: "linear",
|
||
x: 0, y: 0, x2: 0, y2: 1,
|
||
colorStops: [
|
||
{{ offset: 0, color: "#a78bfa" }},
|
||
{{ offset: 1, color: "#818cf8" }}
|
||
]
|
||
}}
|
||
}}
|
||
}}
|
||
}}]
|
||
}});
|
||
window.addEventListener("resize", () => chart.resize());
|
||
setTimeout(() => chart.resize(), 0);
|
||
|
||
function escapeHtml(s) {{
|
||
return String(s)
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/"/g, """);
|
||
}}
|
||
|
||
(function renderFreqTable() {{
|
||
const tbody = document.getElementById("freq-tbody");
|
||
if (!tbody || !chartData.length) return;
|
||
tbody.innerHTML = chartData.map((g, i) => {{
|
||
const members = (g.members || [])
|
||
.map(m => `${{m.surface}} (${{Number(m.clicks).toLocaleString()}})`)
|
||
.join("；");
|
||
return `<tr>
|
||
<td class="num">${{i + 1}}</td>
|
||
<td>${{escapeHtml(g.label)}}</td>
|
||
<td class="num">${{Number(g.total).toLocaleString()}}</td>
|
||
<td class="num">${{(g.members || []).length}}</td>
|
||
<td class="members" title="${{escapeHtml(members)}}">${{escapeHtml(members)}}</td>
|
||
</tr>`;
|
||
}}).join("");
|
||
}})();
|
||
|
||
function initSortableTable(theadId, tbodyId, payload) {{
|
||
const columns = payload.columns || [];
|
||
const rows = (payload.rows || []).map(r => r.slice());
|
||
const thead = document.getElementById(theadId);
|
||
const tbody = document.getElementById(tbodyId);
|
||
if (!thead || !tbody || !columns.length) return;
|
||
|
||
let sortCol = -1;
|
||
let sortAsc = true;
|
||
|
||
function cellText(v) {{
|
||
if (v === null || v === undefined) return "";
|
||
return String(v);
|
||
}}
|
||
|
||
function compareValues(a, b) {{
|
||
const sa = cellText(a);
|
||
const sb = cellText(b);
|
||
if (sa === "" && sb === "") return 0;
|
||
if (sa === "") return 1;
|
||
if (sb === "") return -1;
|
||
const na = Number(sa);
|
||
const nb = Number(sb);
|
||
if (!Number.isNaN(na) && !Number.isNaN(nb)) return na - nb;
|
||
return sa.localeCompare(sb, undefined, {{ numeric: true, sensitivity: "base" }});
|
||
}}
|
||
|
||
function renderHead() {{
|
||
thead.innerHTML = "<tr>" + columns.map((col, i) => {{
|
||
let icon = "↕";
|
||
if (sortCol === i) icon = sortAsc ? "↑" : "↓";
|
||
return `<th data-col="${{i}}">${{col}}<span class="sort-icon">${{icon}}</span></th>`;
|
||
}}).join("") + "</tr>";
|
||
thead.querySelectorAll("th").forEach(th => {{
|
||
th.addEventListener("click", () => {{
|
||
const col = Number(th.dataset.col);
|
||
if (sortCol === col) sortAsc = !sortAsc;
|
||
else {{ sortCol = col; sortAsc = true; }}
|
||
rows.sort((ra, rb) => {{
|
||
const cmp = compareValues(ra[col], rb[col]);
|
||
return sortAsc ? cmp : -cmp;
|
||
}});
|
||
renderBody();
|
||
renderHead();
|
||
}});
|
||
}});
|
||
}}
|
||
|
||
function renderBody() {{
|
||
tbody.innerHTML = rows.map(r =>
|
||
"<tr>" + r.map(v => `<td title="${{cellText(v).replace(/"/g, """)}}">${{cellText(v)}}</td>`).join("") + "</tr>"
|
||
).join("");
|
||
}}
|
||
|
||
renderHead();
|
||
renderBody();
|
||
}}
|
||
|
||
initSortableTable("original-thead", "original-tbody", originalTablePayload);
|
||
initSortableTable("filtered-thead", "filtered-tbody", filteredTablePayload);
|
||
</script>
|
||
</body>
|
||
</html>"""
|
||
|
||
|
||
def write_word_freq_html_report(
|
||
filtered_df: pd.DataFrame,
|
||
input_df: pd.DataFrame,
|
||
embed_path: Path,
|
||
html_path: Path,
|
||
*,
|
||
input_path: Path | None = None,
|
||
top_n: int = _HTML_TOP_DEFAULT,
|
||
include_synonyms: bool = False,
|
||
filter_tooltip: str = "",
|
||
) -> tuple[Path, list[str]]:
|
||
groups = build_word_freq_groups(
|
||
input_df, top_n=top_n, include_synonyms=include_synonyms
|
||
)
|
||
displayed = len(groups)
|
||
original_rank_df = filter_top_rank_pred(
|
||
input_df, _HTML_RANK_PRED_PCT, by_model=True
|
||
)
|
||
input_download_df = input_df.copy()
|
||
title = infer_html_report_title(input_path, embed_path)
|
||
content = render_word_freq_html(
|
||
groups,
|
||
embed_path=embed_path,
|
||
title=title,
|
||
row_count=len(filtered_df),
|
||
filtered_df=filtered_df,
|
||
original_rank_df=original_rank_df,
|
||
input_download_df=input_download_df,
|
||
filter_tooltip=filter_tooltip,
|
||
)
|
||
html_path.parent.mkdir(parents=True, exist_ok=True)
|
||
html_path.write_text(content, encoding="utf-8")
|
||
merge_desc = "stem/lemma/WordNet 同义词" if include_synonyms else "stem/lemma"
|
||
cap_note = f"Top {top_n}" if displayed >= top_n else f"共 {displayed} 个（不足 {top_n}，已全部展示）"
|
||
logs = [
|
||
f"[词频] 基于输入明细_业务表 {len(input_df)} 行 · 词族 {displayed} 个（{cap_note}）· {merge_desc} 并族",
|
||
f" 相对排序前 {_HTML_RANK_PRED_PCT:g}% 关键词表 {len(original_rank_df)} 行 · 已内嵌下载",
|
||
f" HTML 含词频/原始/筛选三表 + 底部双下载 · 已内嵌 {embed_path.name}",
|
||
]
|
||
return html_path, logs
|
||
|
||
|
||
class KeepKeywordMatcher:
|
||
"""核心词强制保留：子串 | POS+lemma | stem | WordNet 同义词（OR，无 embedding）。"""
|
||
|
||
def __init__(self, terms: list[str]) -> None:
|
||
if not terms:
|
||
raise ValueError("核心词列表为空")
|
||
self.terms = terms
|
||
self._engine = WordVariantEngine()
|
||
|
||
self._core_pools: list[set[str]] = []
|
||
for term in terms:
|
||
self._core_pools.append(
|
||
self._engine.variant_pool(term, pos=None, include_synonyms=True)
|
||
)
|
||
|
||
def matches(self, text: str) -> bool:
|
||
if not text or not str(text).strip():
|
||
return False
|
||
lowered = str(text).lower()
|
||
for term in self.terms:
|
||
if term in lowered:
|
||
return True
|
||
text_vars = self._engine.text_variant_set(lowered)
|
||
return any(text_vars & pool for pool in self._core_pools)
|
||
|
||
def mask(self, series: pd.Series) -> pd.Series:
|
||
return series.map(self.matches)
|
||
|
||
|
||
def apply_keep_keywords(
|
||
df: pd.DataFrame,
|
||
filtered: pd.DataFrame,
|
||
terms: list[str],
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
kw_col = resolve_keyword_column(df)
|
||
matcher = KeepKeywordMatcher(terms)
|
||
pin_mask = matcher.mask(df[kw_col])
|
||
pinned = df[pin_mask]
|
||
filtered_idx = set(filtered.index)
|
||
rescued = int(sum(1 for idx in pinned.index if idx not in filtered_idx))
|
||
merged = pd.concat([filtered, pinned]).drop_duplicates()
|
||
logs = [
|
||
"[保留] 核心词 "
|
||
f"{terms!r}（子串 / POS+lemma / stem / WordNet 同义词）",
|
||
f" 命中 {len(pinned)} 行，救回 {rescued} 行",
|
||
f" 筛选结果 {len(filtered)} 行 → 合并后 {len(merged)} 行",
|
||
]
|
||
return merged, logs
|
||
|
||
|
||
def print_column_catalog() -> None:
|
||
print("可筛选列序号对照表（--cols 填序号；可选 序号:max 或 序号:min 覆盖默认方向）：\n")
|
||
print(f"{'序号':>4} {'方向':<4} 列名")
|
||
print("-" * 48)
|
||
for i, (name, direction) in enumerate(FILTERABLE_COLUMNS, start=1):
|
||
hint = "越大越好" if direction == "max" else "越小越好"
|
||
print(f"{i:>4} {direction:<4} {name} ({hint})")
|
||
print("\n--cols 示例：1,8,9 | 1 8 9 | 1，8，9 | 1:min,9:max")
|
||
print("--thresholds 示例（与 --cols 个数、顺序一致）：500,0.05,1.2 | 500 0.05 1.2")
|
||
|
||
|
||
def resolve_column(index: int, direction_override: Direction | None) -> tuple[str, Direction]:
|
||
if index < 1 or index > len(FILTERABLE_COLUMNS):
|
||
raise ValueError(
|
||
f"无效序号 {index}，允许 1–{len(FILTERABLE_COLUMNS)}（用 --list-cols 查看）"
|
||
)
|
||
name, default_dir = FILTERABLE_COLUMNS[index - 1]
|
||
direction = direction_override or default_dir
|
||
if direction not in ("max", "min"):
|
||
raise ValueError(f"无效方向 {direction!r}，仅支持 max / min")
|
||
return name, direction
|
||
|
||
|
||
def parse_cols_spec(spec: str) -> list[tuple[str, Direction]]:
|
||
"""解析 --cols：支持逗号、中文逗号、空格分隔；项可为 序号 或 序号:max/min。"""
|
||
text = (spec or "").strip()
|
||
if not text:
|
||
raise ValueError("--cols 不能为空")
|
||
parts = [p for p in _COLS_SPLIT_RE.split(text) if p.strip()]
|
||
if not parts:
|
||
raise ValueError("--cols 解析后为空")
|
||
|
||
out: list[tuple[str, Direction]] = []
|
||
seen: set[str] = set()
|
||
for part in parts:
|
||
direction_override: Direction | None = None
|
||
if ":" in part:
|
||
idx_str, dir_str = part.split(":", 1)
|
||
direction_override = dir_str.strip().lower() # type: ignore[assignment]
|
||
if direction_override not in ("max", "min"):
|
||
raise ValueError(f"无效方向 {dir_str!r}（在 {part!r} 中）")
|
||
else:
|
||
idx_str = part
|
||
try:
|
||
index = int(idx_str.strip())
|
||
except ValueError as e:
|
||
raise ValueError(f"无法解析列序号：{part!r}") from e
|
||
name, direction = resolve_column(index, direction_override)
|
||
if name in seen:
|
||
continue
|
||
seen.add(name)
|
||
out.append((name, direction))
|
||
return out
|
||
|
||
|
||
def mask_top_pct(series: pd.Series, direction: Direction, pct: float) -> pd.Series:
|
||
"""保留该列「最好的前 pct%」行（max：大值；min：小值）。"""
|
||
s = pd.to_numeric(series, errors="coerce")
|
||
valid = s.notna()
|
||
if not valid.any():
|
||
return pd.Series(False, index=series.index)
|
||
|
||
q = max(0.0, min(100.0, float(pct))) / 100.0
|
||
if direction == "max":
|
||
thr = s[valid].quantile(1.0 - q)
|
||
return valid & (s >= thr)
|
||
thr = s[valid].quantile(q)
|
||
return valid & (s <= thr)
|
||
|
||
|
||
def threshold_for_column(series: pd.Series, direction: Direction, pct: float) -> float | None:
|
||
s = pd.to_numeric(series, errors="coerce").dropna()
|
||
if s.empty:
|
||
return None
|
||
q = max(0.0, min(100.0, float(pct))) / 100.0
|
||
if direction == "max":
|
||
return float(s.quantile(1.0 - q))
|
||
return float(s.quantile(q))
|
||
|
||
|
||
def filter_group(
|
||
df: pd.DataFrame,
|
||
cols: list[tuple[str, Direction]],
|
||
pct: float,
|
||
logic: Logic,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
"""对单个 DataFrame 筛选，返回 (结果, 日志行)。"""
|
||
if df.empty:
|
||
return df, []
|
||
|
||
logs: list[str] = []
|
||
if logic == "and":
|
||
mask = pd.Series(True, index=df.index)
|
||
for col, direction in cols:
|
||
if col not in df.columns:
|
||
raise ValueError(f"输入 CSV 缺少列 {col!r}，请确认词库已 join 或换 --cols")
|
||
col_mask = mask_top_pct(df[col], direction, pct)
|
||
thr = threshold_for_column(df[col], direction, pct)
|
||
op = ">=" if direction == "max" else "<="
|
||
thr_s = f"{thr:.6g}" if thr is not None else "N/A"
|
||
logs.append(
|
||
f" - {col} ({direction} top {pct:g}%): {op} {thr_s}，保留 {int(col_mask.sum())} 行"
|
||
)
|
||
mask &= col_mask
|
||
return df[mask].copy(), logs
|
||
|
||
# logic == "seq"
|
||
current = df
|
||
for col, direction in cols:
|
||
if col not in current.columns:
|
||
raise ValueError(f"输入 CSV 缺少列 {col!r}，请确认词库已 join 或换 --cols")
|
||
col_mask = mask_top_pct(current[col], direction, pct)
|
||
thr = threshold_for_column(current[col], direction, pct)
|
||
op = ">=" if direction == "max" else "<="
|
||
thr_s = f"{thr:.6g}" if thr is not None else "N/A"
|
||
kept = int(col_mask.sum())
|
||
logs.append(
|
||
f" - {col} ({direction} top {pct:g}%): {op} {thr_s}，"
|
||
f"本步 {len(current)} → {kept} 行"
|
||
)
|
||
current = current[col_mask].copy()
|
||
return current, logs
|
||
|
||
|
||
def filter_infer_business(
|
||
df: pd.DataFrame,
|
||
cols: list[tuple[str, Direction]],
|
||
pct: float,
|
||
*,
|
||
logic: Logic = "and",
|
||
by_model: bool = False,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
logs: list[str] = []
|
||
if by_model and "model" in df.columns:
|
||
parts: list[pd.DataFrame] = []
|
||
for model_name, group in df.groupby("model", sort=False):
|
||
sub, sub_logs = filter_group(group, cols, pct, logic)
|
||
logs.append(f"[model={model_name}] {len(group)} → {len(sub)} 行")
|
||
logs.extend(sub_logs)
|
||
parts.append(sub)
|
||
out = pd.concat(parts, ignore_index=True) if parts else df.iloc[0:0].copy()
|
||
return out, logs
|
||
|
||
out, sub_logs = filter_group(df, cols, pct, logic)
|
||
logs.extend(sub_logs)
|
||
return out, logs
|
||
|
||
|
||
def parse_thresholds(spec: str) -> list[float]:
|
||
"""解析 --thresholds，分隔符与 --cols 相同。"""
|
||
text = (spec or "").strip()
|
||
if not text:
|
||
return []
|
||
parts = [p for p in _COLS_SPLIT_RE.split(text) if p.strip()]
|
||
if not parts:
|
||
raise ValueError("--thresholds 解析后为空")
|
||
out: list[float] = []
|
||
for part in parts:
|
||
try:
|
||
out.append(float(part.strip()))
|
||
except ValueError as e:
|
||
raise ValueError(f"无法解析临界值：{part!r}") from e
|
||
return out
|
||
|
||
|
||
def mask_by_threshold(
|
||
series: pd.Series, direction: Direction, threshold: float
|
||
) -> pd.Series:
|
||
"""max：>= 临界值；min：<= 临界值（含边界）；NaN 不保留。"""
|
||
s = pd.to_numeric(series, errors="coerce")
|
||
valid = s.notna()
|
||
if direction == "max":
|
||
return valid & (s >= threshold)
|
||
return valid & (s <= threshold)
|
||
|
||
|
||
def filter_group_threshold(
|
||
df: pd.DataFrame,
|
||
col_specs: list[tuple[str, Direction]],
|
||
thresholds: list[float],
|
||
logic: Logic,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
if df.empty:
|
||
return df, []
|
||
if len(col_specs) != len(thresholds):
|
||
raise ValueError(
|
||
f"--cols 与 --thresholds 数量不一致：{len(col_specs)} 列 vs {len(thresholds)} 个临界值"
|
||
)
|
||
|
||
logs: list[str] = []
|
||
if logic == "and":
|
||
mask = pd.Series(True, index=df.index)
|
||
for (col, direction), thr in zip(col_specs, thresholds):
|
||
if col not in df.columns:
|
||
raise ValueError(f"输入表缺少列 {col!r}，请确认词库已 join 或换 --cols")
|
||
col_mask = mask_by_threshold(df[col], direction, thr)
|
||
op = ">=" if direction == "max" else "<="
|
||
logs.append(
|
||
f" - {col} ({direction} {op} {thr:g})：满足 {int(col_mask.sum())} 行"
|
||
)
|
||
mask &= col_mask
|
||
kept = int(mask.sum())
|
||
logs.append(f" - 同时满足：{kept} 行")
|
||
return df[mask].copy(), logs
|
||
|
||
current = df
|
||
for (col, direction), thr in zip(col_specs, thresholds):
|
||
if col not in current.columns:
|
||
raise ValueError(f"输入表缺少列 {col!r}，请确认词库已 join 或换 --cols")
|
||
col_mask = mask_by_threshold(current[col], direction, thr)
|
||
op = ">=" if direction == "max" else "<="
|
||
kept = int(col_mask.sum())
|
||
logs.append(
|
||
f" - {col} ({direction} {op} {thr:g})："
|
||
f"本步 {len(current)} → {kept} 行"
|
||
)
|
||
current = current[col_mask].copy()
|
||
return current, logs
|
||
|
||
|
||
def filter_infer_business_threshold(
|
||
df: pd.DataFrame,
|
||
col_specs: list[tuple[str, Direction]],
|
||
thresholds: list[float],
|
||
*,
|
||
logic: Logic = "and",
|
||
by_model: bool = False,
|
||
) -> tuple[pd.DataFrame, list[str]]:
|
||
logs: list[str] = []
|
||
if by_model and "model" in df.columns:
|
||
parts: list[pd.DataFrame] = []
|
||
for model_name, group in df.groupby("model", sort=False):
|
||
sub, sub_logs = filter_group_threshold(
|
||
group, col_specs, thresholds, logic
|
||
)
|
||
logs.append(f"[model={model_name}] {len(group)} → {len(sub)} 行")
|
||
logs.extend(sub_logs)
|
||
parts.append(sub)
|
||
out = pd.concat(parts, ignore_index=True) if parts else df.iloc[0:0].copy()
|
||
return out, logs
|
||
|
||
out, sub_logs = filter_group_threshold(df, col_specs, thresholds, logic)
|
||
logs.extend(sub_logs)
|
||
return out, logs
|
||
|
||
|
||
def default_output_path(input_path: Path) -> Path:
|
||
return input_path.with_name(f"{input_path.stem}_筛选{input_path.suffix}")
|
||
|
||
|
||
def read_business_table(path: Path) -> pd.DataFrame:
|
||
"""读取业务明细表（.csv / .xlsx / .xls）。"""
|
||
suffix = path.suffix.lower()
|
||
if suffix in (".csv", ".txt", ".tsv"):
|
||
return pd.read_csv(path, encoding="utf-8-sig")
|
||
if suffix in (".xlsx", ".xls"):
|
||
return pd.read_excel(path, engine="openpyxl")
|
||
raise ValueError(f"不支持的输入格式：{suffix}（请用 .csv / .xlsx / .xls）")
|
||
|
||
|
||
def write_business_table(df: pd.DataFrame, path: Path) -> None:
|
||
suffix = path.suffix.lower()
|
||
if suffix in (".csv", ".txt", ".tsv"):
|
||
df.to_csv(path, index=False, encoding="utf-8-sig")
|
||
return
|
||
if suffix in (".xlsx", ".xls"):
|
||
df.to_excel(path, index=False, engine="openpyxl")
|
||
return
|
||
raise ValueError(f"不支持的输出格式：{suffix}（请用 .csv / .xlsx / .xls）")
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
p = argparse.ArgumentParser(
|
||
description="对推理预测_明细_业务表筛选：分位数（-p）或固定临界值（--thresholds）"
|
||
)
|
||
p.add_argument(
|
||
"-i",
|
||
"--input",
|
||
type=Path,
|
||
default=None,
|
||
help="业务 CSV/xlsx 路径",
|
||
)
|
||
p.add_argument(
|
||
"-o",
|
||
"--output",
|
||
type=Path,
|
||
default=None,
|
||
help="输出路径；默认在同目录生成 *_筛选（扩展名与输入一致）",
|
||
)
|
||
p.add_argument(
|
||
"-p",
|
||
"--pct",
|
||
type=float,
|
||
default=DEFAULT_FILTER_PCT,
|
||
help=f"分位数模式：保留每列最好的前 a%%；与 --thresholds 二选一，默认 {DEFAULT_FILTER_PCT:g}",
|
||
)
|
||
p.add_argument(
|
||
"-c",
|
||
"--cols",
|
||
default=DEFAULT_FILTER_COLS,
|
||
help=f"列序号，逗号/中文逗号/空格分隔；可写 1:max,9:min 覆盖方向（默认 {DEFAULT_FILTER_COLS}）",
|
||
)
|
||
p.add_argument(
|
||
"-t",
|
||
"--thresholds",
|
||
default="",
|
||
help="临界值模式：与 --cols 个数、顺序一致；max 列>=临界值，min 列<=临界值",
|
||
)
|
||
p.add_argument(
|
||
"--logic",
|
||
choices=("and", "seq"),
|
||
default="and",
|
||
help="and：各列阈值同时满足；seq：逐列串联筛选（默认 and）",
|
||
)
|
||
p.add_argument(
|
||
"--by-model",
|
||
action="store_true",
|
||
help="若存在 model 列，则每个 model 内分别筛选再合并",
|
||
)
|
||
p.add_argument(
|
||
"--list-cols",
|
||
action="store_true",
|
||
help="打印可筛选列序号对照表后退出",
|
||
)
|
||
p.add_argument(
|
||
"-k",
|
||
"--keep-keywords",
|
||
default="",
|
||
help="核心词强制保留（逗号/中文逗号/空格分隔），与筛选结果取并集",
|
||
)
|
||
p.add_argument(
|
||
"--html-report",
|
||
action=argparse.BooleanOptionalAction,
|
||
default=True,
|
||
help="筛选完成后生成词频 HTML（默认开启；--no-html-report 关闭）",
|
||
)
|
||
p.add_argument(
|
||
"--html-top",
|
||
type=int,
|
||
default=_HTML_TOP_DEFAULT,
|
||
help=f"词频 HTML 展示词族数量（默认 {_HTML_TOP_DEFAULT}）",
|
||
)
|
||
p.add_argument(
|
||
"--html-output",
|
||
type=Path,
|
||
default=None,
|
||
help="词频 HTML 输出路径；默认 {ASIN}_广告词推荐.html",
|
||
)
|
||
p.add_argument(
|
||
"--freq-synonyms",
|
||
action="store_true",
|
||
help="词频并族时启用 WordNet 同义词（默认仅 stem/lemma 并族）",
|
||
)
|
||
return p.parse_args()
|
||
|
||
|
||
def _format_col_threshold_specs(
|
||
col_specs: list[tuple[str, Direction]],
|
||
thresholds: list[float] | None = None,
|
||
) -> str:
|
||
parts: list[str] = []
|
||
for i, (name, direction) in enumerate(col_specs):
|
||
if thresholds is not None:
|
||
thr = thresholds[i]
|
||
op = ">=" if direction == "max" else "<="
|
||
parts.append(f"{name}({direction} {op} {thr:g})")
|
||
else:
|
||
parts.append(f"{name}({direction})")
|
||
return ", ".join(parts)
|
||
|
||
|
||
def run_filter_infer_business(
|
||
input_path: Path | str,
|
||
*,
|
||
pct: float | None = DEFAULT_FILTER_PCT,
|
||
cols: str = DEFAULT_FILTER_COLS,
|
||
thresholds: str = "",
|
||
logic: Logic = "and",
|
||
by_model: bool = False,
|
||
keep_keywords: str = "",
|
||
html_report: bool = True,
|
||
html_top: int = _HTML_TOP_DEFAULT,
|
||
output: Path | None = None,
|
||
html_output: Path | None = None,
|
||
freq_synonyms: bool = False,
|
||
skip_if_missing_vocab_cols: bool = True,
|
||
) -> tuple[int, Path | None, Path | None]:
|
||
"""
|
||
对「推理预测_明细_业务」筛选并可选生成词频 HTML。
|
||
|
||
默认等价于：-p 30 --cols 1,3,9 --html-report
|
||
返回 (exit_code, 筛选表路径, HTML路径)；跳过时后两者为 None。
|
||
"""
|
||
input_path = Path(input_path).expanduser().resolve()
|
||
if not input_path.is_file():
|
||
print(f"[失败] 找不到文件：{input_path}", file=sys.stderr)
|
||
return 1, None, None
|
||
|
||
use_thresholds = bool((thresholds or "").strip())
|
||
if use_thresholds and pct is not None:
|
||
print("[提示] 已指定 --thresholds，忽略 --pct，使用临界值模式。", file=sys.stderr)
|
||
|
||
try:
|
||
col_specs = parse_cols_spec(cols)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
print("提示：运行 --list-cols 查看序号。", file=sys.stderr)
|
||
return 1, None, None
|
||
|
||
threshold_values: list[float] = []
|
||
if use_thresholds:
|
||
try:
|
||
threshold_values = parse_thresholds(thresholds)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
return 1, None, None
|
||
if len(threshold_values) != len(col_specs):
|
||
print(
|
||
f"[失败] --cols 有 {len(col_specs)} 列，--thresholds 有 {len(threshold_values)} 个值，"
|
||
"须一一对应。",
|
||
file=sys.stderr,
|
||
)
|
||
return 1, None, None
|
||
else:
|
||
pct_val = DEFAULT_FILTER_PCT if pct is None else float(pct)
|
||
if not (0 < pct_val <= 100):
|
||
print("[失败] --pct 须在 (0, 100] 内，或使用 --thresholds。", file=sys.stderr)
|
||
return 1, None, None
|
||
pct = pct_val
|
||
|
||
try:
|
||
df = read_business_table(input_path)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
return 1, None, None
|
||
except ImportError:
|
||
print(
|
||
"[失败] 读取 xlsx 需要 openpyxl：pip install openpyxl",
|
||
file=sys.stderr,
|
||
)
|
||
return 1, None, None
|
||
|
||
if skip_if_missing_vocab_cols:
|
||
missing = [name for name, _ in col_specs if name not in df.columns]
|
||
if missing:
|
||
print(
|
||
f"[业务筛选] 跳过：输入表缺少词库列 {missing}，"
|
||
"请确认词库已 join 后再筛选。",
|
||
file=sys.stderr,
|
||
)
|
||
return 0, None, None
|
||
|
||
before = len(df)
|
||
|
||
try:
|
||
if use_thresholds:
|
||
filtered, logs = filter_infer_business_threshold(
|
||
df,
|
||
col_specs,
|
||
threshold_values,
|
||
logic=logic,
|
||
by_model=by_model,
|
||
)
|
||
else:
|
||
filtered, logs = filter_infer_business(
|
||
df,
|
||
col_specs,
|
||
float(pct), # type: ignore[arg-type]
|
||
logic=logic,
|
||
by_model=by_model,
|
||
)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
return 1, None, None
|
||
|
||
keep_terms = parse_keep_keywords(keep_keywords)
|
||
keep_logs: list[str] = []
|
||
if keep_terms:
|
||
try:
|
||
filtered, keep_logs = apply_keep_keywords(df, filtered, keep_terms)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
return 1, None, None
|
||
except Exception as e:
|
||
print(f"[失败] 核心词匹配：{e}", file=sys.stderr)
|
||
return 1, None, None
|
||
|
||
output_path = (
|
||
output.expanduser().resolve()
|
||
if output is not None
|
||
else default_output_path(input_path)
|
||
)
|
||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
write_business_table(filtered, output_path)
|
||
except ValueError as e:
|
||
print(f"[失败] {e}", file=sys.stderr)
|
||
return 1, None, None
|
||
except ImportError:
|
||
print(
|
||
"[失败] 写入 xlsx 需要 openpyxl：pip install openpyxl",
|
||
file=sys.stderr,
|
||
)
|
||
return 1, None, None
|
||
|
||
selected = _format_col_threshold_specs(
|
||
col_specs, threshold_values if use_thresholds else None
|
||
)
|
||
if use_thresholds:
|
||
mode_desc = f"临界值, logic={logic}, by_model={by_model}"
|
||
else:
|
||
mode_desc = f"pct={float(pct):g}%, logic={logic}, by_model={by_model}"
|
||
print(f"[筛选] 输入 {before} 行 → 输出 {len(filtered)} 行 ({mode_desc})")
|
||
print(f"[筛选] 列：{selected}")
|
||
for line in logs:
|
||
print(line)
|
||
for line in keep_logs:
|
||
print(line)
|
||
print(f"已写入：{output_path}")
|
||
|
||
html_path: Path | None = None
|
||
if html_report:
|
||
html_path = (
|
||
html_output.expanduser().resolve()
|
||
if html_output is not None
|
||
else default_html_report_path(output_path, input_path)
|
||
)
|
||
html_top_n = max(1, int(html_top))
|
||
if use_thresholds:
|
||
filter_tooltip = build_filter_tooltip(
|
||
col_specs,
|
||
thresholds=threshold_values,
|
||
logic=logic,
|
||
)
|
||
else:
|
||
filter_tooltip = build_filter_tooltip(
|
||
col_specs,
|
||
pct=float(pct), # type: ignore[arg-type]
|
||
logic=logic,
|
||
)
|
||
try:
|
||
html_path, html_logs = write_word_freq_html_report(
|
||
filtered,
|
||
df,
|
||
embed_path=output_path,
|
||
html_path=html_path,
|
||
input_path=input_path,
|
||
top_n=html_top_n,
|
||
include_synonyms=freq_synonyms,
|
||
filter_tooltip=filter_tooltip,
|
||
)
|
||
except ValueError as e:
|
||
print(f"[失败] 词频 HTML：{e}", file=sys.stderr)
|
||
return 1, output_path, None
|
||
except Exception as e:
|
||
print(f"[失败] 词频 HTML：{e}", file=sys.stderr)
|
||
return 1, output_path, None
|
||
for line in html_logs:
|
||
print(line)
|
||
print(f"已写入：{html_path}")
|
||
|
||
return 0, output_path, html_path
|
||
|
||
|
||
def main() -> int:
|
||
args = parse_args()
|
||
if args.list_cols:
|
||
print_column_catalog()
|
||
return 0
|
||
|
||
if args.input is None:
|
||
print("[失败] 请指定 --input，或使用 --list-cols 查看列序号。", file=sys.stderr)
|
||
return 1
|
||
|
||
code, _, _ = run_filter_infer_business(
|
||
args.input,
|
||
pct=args.pct,
|
||
cols=args.cols,
|
||
thresholds=args.thresholds,
|
||
logic=args.logic, # type: ignore[arg-type]
|
||
by_model=bool(args.by_model),
|
||
keep_keywords=args.keep_keywords,
|
||
html_report=bool(args.html_report),
|
||
html_top=int(args.html_top),
|
||
output=args.output,
|
||
html_output=args.html_output,
|
||
freq_synonyms=bool(args.freq_synonyms),
|
||
skip_if_missing_vocab_cols=True,
|
||
)
|
||
return code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|