feat: 所有新闻/政策/研究摘要加入 pub_time 发布时间字段\n\n- summarize_news.py: AI摘要后用原始RSS时间按标题匹配回填pub_time\n- summarize_policy.py: 同上\n- summarize_research.py: 同上\n- 时间格式: 当天显示HH:MM, 非当天显示MM-DD HH:MM\n- 前端brief.html: 新闻卡片badge行右侧显示pub_time\n- 已用backfill脚本为现有数据补填pub_time(96/102, 8/15, 33/37)
This commit is contained in:
Beast
@ -1,65 +1,103 @@
|
||||
"""Use DeepSeek to summarize policy news into categorized Chinese briefs."""
|
||||
"""Summarize policy news using DeepSeek AI.
|
||||
v2: back-fills pub_time from raw RSS data after AI summarization.
|
||||
"""
|
||||
import json, os, requests, re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from dateutil import parser as dtparser
|
||||
|
||||
API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
|
||||
API_URL = "https://api.deepseek.com/v1/chat/completions"
|
||||
MODEL = "deepseek-v4-flash"
|
||||
|
||||
INPUT = "/var/www/nav/data/policy.json"
|
||||
RAW_FILE = "/var/www/nav/data/policy.json"
|
||||
OUTPUT = "/var/www/nav/data/policy_summary.json"
|
||||
tz = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def load_raw():
|
||||
if not os.path.exists(INPUT):
|
||||
if not os.path.exists(RAW_FILE):
|
||||
return []
|
||||
with open(INPUT) as f:
|
||||
with open(RAW_FILE) as f:
|
||||
data = json.load(f)
|
||||
return data.get("items", [])
|
||||
|
||||
|
||||
def build_time_lookup(raw_items):
|
||||
"""Build lookup: title -> pub_time (Beijing time string)."""
|
||||
lookup = {}
|
||||
for item in raw_items:
|
||||
title = item.get("title", "").strip()
|
||||
published = item.get("published", "").strip()
|
||||
if not title or not published:
|
||||
continue
|
||||
try:
|
||||
dt = dtparser.parse(published)
|
||||
if dt.tzinfo:
|
||||
dt = dt.astimezone(tz)
|
||||
else:
|
||||
dt = dt.replace(tzinfo=tz)
|
||||
now = datetime.now(tz)
|
||||
if dt.date() == now.date():
|
||||
time_str = dt.strftime("%H:%M")
|
||||
else:
|
||||
time_str = dt.strftime("%m-%d %H:%M")
|
||||
lookup[title] = time_str
|
||||
except Exception:
|
||||
continue
|
||||
return lookup
|
||||
|
||||
|
||||
def match_pub_time(items, lookup):
|
||||
for item in items:
|
||||
if "pub_time" in item:
|
||||
continue
|
||||
title = item.get("title", "").strip()
|
||||
if title in lookup:
|
||||
item["pub_time"] = lookup[title]
|
||||
else:
|
||||
for raw_title, time_val in lookup.items():
|
||||
short = min(len(raw_title), len(title))
|
||||
if short > 10 and (raw_title[:short] in title or title[:short] in raw_title):
|
||||
item["pub_time"] = time_val
|
||||
break
|
||||
|
||||
|
||||
def summarize(items):
|
||||
news_text = ""
|
||||
for i, item in enumerate(items[:100], 1):
|
||||
for i, item in enumerate(items[:150], 1):
|
||||
title = item.get("title", "")
|
||||
summary = item.get("summary", "")[:200].replace("<", "").replace(">", "")
|
||||
source = item.get("source", "")
|
||||
link = item.get("link", "")
|
||||
news_text += f"{i}. [{source}] {title}\n {summary}\n url: {link}\n\n"
|
||||
|
||||
prompt = f"""你是一个政策分析助手。请从原始政策新闻中精选出**最重要的政策动态**，按优先级排序。
|
||||
prompt = f"""你是一个资深的政策新闻编辑助手。精选当下最重要的中国政策新闻。
|
||||
|
||||
## 选稿标准
|
||||
- 国务院/发改委/部委最新政策文件
|
||||
- 习近平总书记重要讲话/活动
|
||||
- 重大经济/产业政策（数字经济、AI、能源、金融）
|
||||
- 重大改革举措
|
||||
- 重要人事任免
|
||||
|
||||
## 输出
|
||||
- 分类名：**政策速递**（只有一个分类）
|
||||
- 每条：**15-30字完整标题**（subtitle），**300-500字政策解读摘要**（summary）
|
||||
- 总计 **10-20 条**，按重要性排序
|
||||
- 国务院常务会议、中央政治局会议等高层决策
|
||||
- 重要法律法规的颁布或修订
|
||||
- 部委级重大政策发布（发改委、财政部、央行、工信部等）
|
||||
- 经济政策调整（货币政策、财政政策、产业政策）
|
||||
- 民生政策（教育、医疗、住房、养老、就业）
|
||||
- 外交政策重大声明
|
||||
|
||||
## 输出格式
|
||||
每条输出到 items 数组，包含：rank, subtitle, title, source, link, summary
|
||||
summary 用自然段落，关键信息用 **加粗**。
|
||||
每条包含：
|
||||
- subtitle：15-30字完整标题（中文）
|
||||
- title：原标题（必须与原始新闻标题完全一致，这是时间匹配的关键）
|
||||
- source：来源
|
||||
- link：原文链接
|
||||
- summary：300-500字政策解读摘要，关键信息用**加粗**标出
|
||||
|
||||
## 原始新闻
|
||||
{news_text}
|
||||
精选 **10-20 条** 最重要政策新闻。
|
||||
|
||||
JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}"""
|
||||
JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"小标题","title":"原标题","source":"来源","link":"https://...","summary":"正文..."}}]}}]}}"""
|
||||
|
||||
resp = requests.post(
|
||||
API_URL,
|
||||
headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
|
||||
json={
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 16000,
|
||||
},
|
||||
json={"model": MODEL, "messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.3, "max_tokens": 20000},
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
@ -73,49 +111,42 @@ JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtit
|
||||
content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:])
|
||||
|
||||
try:
|
||||
result = json.loads(content)
|
||||
return json.loads(content)
|
||||
except:
|
||||
import re as re_
|
||||
content = re_.sub(r',\s*}', '}', content)
|
||||
content = re_.sub(r',\s*]', ']', content)
|
||||
content = re.sub(r',\s*}', '}', content)
|
||||
content = re.sub(r',\s*]', ']', content)
|
||||
try:
|
||||
result = json.loads(content)
|
||||
return json.loads(content)
|
||||
except:
|
||||
print(f"JSON parse failed, len={len(content)}")
|
||||
print(f"First 500: {content[:500]}")
|
||||
print(f"JSON parse failed")
|
||||
return None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def run():
|
||||
if __name__ == "__main__":
|
||||
now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
|
||||
items = load_raw()
|
||||
if not items:
|
||||
print("No policy data")
|
||||
raw_items = load_raw()
|
||||
if not raw_items:
|
||||
print("No raw policy data")
|
||||
exit(0)
|
||||
|
||||
print(f"Loaded {len(items)} policy items. AI summarizing...")
|
||||
data = summarize(items)
|
||||
|
||||
print(f"Loaded {len(raw_items)} raw items. Summarizing...")
|
||||
data = summarize(raw_items)
|
||||
if not data:
|
||||
print("Summarization failed")
|
||||
exit(1)
|
||||
|
||||
result = {
|
||||
"updated_at": now,
|
||||
"categories": data.get("categories", []),
|
||||
}
|
||||
# Back-fill pub_time
|
||||
time_lookup = build_time_lookup(raw_items)
|
||||
matched = 0
|
||||
for cat in data.get("categories", []):
|
||||
match_pub_time(cat["items"], time_lookup)
|
||||
matched += sum(1 for item in cat["items"] if "pub_time" in item)
|
||||
print(f"Matched pub_time for {matched} articles")
|
||||
|
||||
result = {"updated_at": now, "categories": data.get("categories", [])}
|
||||
total = sum(len(c["items"]) for c in result["categories"])
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"Written {total} policy briefs -> {OUTPUT}")
|
||||
for c in result["categories"]:
|
||||
print(f" {c['name']}: {len(c['items'])} items")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
print(f"Written {total} articles -> {OUTPUT}")
|
||||
|
||||
Reference in New Issue
Block a user