diff --git a/scripts/summarize_news.py b/scripts/summarize_news.py index 943ca25..9df038e 100644 --- a/scripts/summarize_news.py +++ b/scripts/summarize_news.py @@ -1,6 +1,9 @@ -"""Use DeepSeek to summarize top news into categorized Chinese briefs (5 per category).""" -import json, os, requests +"""Use DeepSeek to summarize top news into categorized Chinese briefs (5 per category). +v2: back-fills pub_time from raw RSS data after AI summarization. +""" +import json, os, requests, re from datetime import datetime, timezone, timedelta +from dateutil import parser as dtparser API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" API_URL = "https://api.deepseek.com/v1/chat/completions" @@ -12,6 +15,7 @@ tz = timezone(timedelta(hours=8)) def load_raw_news(): + """Load raw news and build title->published lookup.""" if not os.path.exists(NEWS_FILE): return [] with open(NEWS_FILE) as f: @@ -19,52 +23,101 @@ def load_raw_news(): return data.get("items", []) +def build_time_lookup(raw_items): + """Build lookup: title -> pub_time (Beijing time string like '06-15 10:30').""" + lookup = {} + for item in raw_items: + title = item.get("title", "").strip() + published = item.get("published", "").strip() + if not title or not published: + continue + try: + dt = dtparser.parse(published) + # Convert to Beijing time if not already + if dt.tzinfo: + dt = dt.astimezone(tz) + else: + dt = dt.replace(tzinfo=tz) + # Format: MM-DD HH:MM for today, MM-DD for older + now = datetime.now(tz) + if dt.date() == now.date(): + time_str = dt.strftime("%H:%M") + else: + time_str = dt.strftime("%m-%d %H:%M") + lookup[title] = time_str + except Exception: + continue + return lookup + + +def match_pub_time(items, lookup): + """Match AI output items back to raw RSS pub_time by title similarity.""" + for item in items: + if "pub_time" in item: + continue + title = item.get("title", "").strip() + subtitle = item.get("subtitle", "").strip() + matched = None + # Exact match on original title first + if title in lookup: + matched = lookup[title] + else: + # Fuzzy: check if subtitle contains original title keywords + for raw_title, time_val in lookup.items(): + # Try partial match + short = min(len(raw_title), len(title)) + if short > 10 and raw_title[:short] in title or title[:short] in raw_title: + matched = time_val + break + # Check if first 20 chars match + if raw_title[:20].strip() == title[:20].strip(): + matched = time_val + break + if matched: + item["pub_time"] = matched + + def summarize(items): news_text = "" - for i, item in enumerate(items[:30], 1): + for i, item in enumerate(items[:200], 1): title = item.get("title", "") summary = item.get("summary", "")[:150].replace("<", "").replace(">", "") source = item.get("source", "") - news_text += f"{i}. [{source}] {title}\n {summary}\n\n" + link = item.get("link", "") + news_text += f"{i}. [{source}] {title}\n {summary}\n url: {link}\n\n" - prompt = f"""你是一个新闻整理助手。以下是来自BBC世界新闻、BBC中文、新浪新闻、涌报、Hacker News的最新资讯。 + prompt = f"""你是一个资深的新闻编辑助手。你的任务是精选出当下最重要的新闻，按分类组织。 -请从以上内容中精选出最重要的新闻，按以下分类组织（每个分类最多5条）： +## 选稿标准（满足任一即可） +- 地缘政治新闻（战争、条约、制裁、外交、选举） +- 宏观经济决策（央行、通胀、贸易、就业） +- 科技重大突破（AI、航天、医药、能源） +- 重大自然灾害/公共卫生事件 +- **中国官方媒体头条**（新华社、人民日报的置顶/头条报道——这些非常重要，必须收录） +- 中国国内引发广泛讨论的社会事件 +- 任何具有长期影响的政策变化 -分类要求： -- 国际要闻：全球政治、外交、军事 -- 科技前沿：AI、软件、硬件、互联网 -- 财经动向：市场、经济、商业 -- 中国热点：中国相关新闻 -- 社会生活：健康、环境、文化、民生 +## 分类（每个分类**至少15条，最多30条**） +- 国际要闻 +- 科技前沿 +- 财经动向 +- 中国热点（必须包含新华社、人民日报的重要报道） +- 社会生活 -每条新闻格式： -## 小标题（10字以内概括核心） +## 输出格式 +每条输出到 items 数组，包含以下字段： +- subtitle：**15-30字完整标题**（不要缩写） +- title：原标题（必须与原始新闻标题完全一致，这是时间匹配的关键） +- source：来源 +- link：原文链接（如不可用则留空） +- summary：**300-500字完整新闻摘要**，用自然段落叙述，关键信息用**加粗**标出。不在前面写"核心事实""背景分析"等前缀，直接写内容，段落清晰流畅。 -正文（200-300字中文精炼摘要，包含核心事实和背景） +## 原始新闻 +{news_text} -输出格式为JSON： -{{ - "categories": [ - {{ - "name": "国际要闻", - "items": [ - {{"rank": 1, "subtitle": "小标题", "title": "原标题", "source": "BBC World", "summary": "正文..."}}, - ... - ] - }}, - ... - ] -}} +请输出JSON格式。注意：宁多勿少，确保每个分类**至少15条，最多30条**。 -注意： -- 每个分类最多5条 -- 小标题要精炼抓人 -- 摘要要有信息量，不要空话 -- 只输出JSON，不要其他内容 - -原始新闻： -{news_text}""" +JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle":"小标题","title":"原标题","source":"来源","link":"https://...","summary":"正文..."}}]}}]}}""" resp = requests.post( API_URL, @@ -72,8 +125,8 @@ def summarize(items): json={ "model": MODEL, "messages": [{"role": "user", "content": prompt}], - "temperature": 0.4, - "max_tokens": 8000, + "temperature": 0.3, + "max_tokens": 40000, }, timeout=180, ) @@ -91,36 +144,56 @@ def summarize(items): try: result = json.loads(content) except: - # Try to fix common JSON issues content = content.strip() - # Remove trailing commas before closing brackets - import re content = re.sub(r',\s*}', '}', content) content = re.sub(r',\s*]', ']', content) - try: - result = json.loads(content) - except: - print(f"JSON parse failed, len={len(content)}") - print(f"First 800 chars: {content[:800]}") - return None + for attempt in range(3): + try: + result = json.loads(content) + break + except: + if attempt == 0: + idx = content.rfind('{"rank"') + if idx > 0: + content = content[:content.rfind('}', 0, idx)] + '}]}]}' + elif attempt == 1: + lines = content.split('\n') + fixed = [] + for line in lines: + if line.strip().endswith(',') and not any(x in line for x in ['"', ']', '}']): + line = line.rstrip(',') + fixed.append(line) + content = '\n'.join(fixed) + else: + print(f"JSON parse failed, len={len(content)}") + print(f"First 800 chars: {content[:800]}") + return None return result if __name__ == "__main__": now = datetime.now(tz).strftime("%Y-%m-%d %H:%M") - items = load_raw_news() - if not items: + raw_items = load_raw_news() + if not raw_items: print("No news data") exit(0) - print(f"Loaded {len(items)} raw news items. AI summarizing by category...") - data = summarize(items) + print(f"Loaded {len(raw_items)} raw news items. AI summarizing by category...") + data = summarize(raw_items) if not data: print("Summarization failed") exit(1) + # Back-fill pub_time from raw RSS data + time_lookup = build_time_lookup(raw_items) + matched = 0 + for cat in data.get("categories", []): + match_pub_time(cat["items"], time_lookup) + matched += sum(1 for item in cat["items"] if "pub_time" in item) + print(f"Matched pub_time for {matched} articles") + result = { "updated_at": now, "categories": data.get("categories", []), diff --git a/scripts/summarize_policy.py b/scripts/summarize_policy.py index 77da4d0..a4997dc 100644 --- a/scripts/summarize_policy.py +++ b/scripts/summarize_policy.py @@ -1,65 +1,103 @@ -"""Use DeepSeek to summarize policy news into categorized Chinese briefs.""" +"""Summarize policy news using DeepSeek AI. +v2: back-fills pub_time from raw RSS data after AI summarization. +""" import json, os, requests, re from datetime import datetime, timezone, timedelta +from dateutil import parser as dtparser API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" API_URL = "https://api.deepseek.com/v1/chat/completions" MODEL = "deepseek-v4-flash" -INPUT = "/var/www/nav/data/policy.json" +RAW_FILE = "/var/www/nav/data/policy.json" OUTPUT = "/var/www/nav/data/policy_summary.json" tz = timezone(timedelta(hours=8)) def load_raw(): - if not os.path.exists(INPUT): + if not os.path.exists(RAW_FILE): return [] - with open(INPUT) as f: + with open(RAW_FILE) as f: data = json.load(f) return data.get("items", []) +def build_time_lookup(raw_items): + """Build lookup: title -> pub_time (Beijing time string).""" + lookup = {} + for item in raw_items: + title = item.get("title", "").strip() + published = item.get("published", "").strip() + if not title or not published: + continue + try: + dt = dtparser.parse(published) + if dt.tzinfo: + dt = dt.astimezone(tz) + else: + dt = dt.replace(tzinfo=tz) + now = datetime.now(tz) + if dt.date() == now.date(): + time_str = dt.strftime("%H:%M") + else: + time_str = dt.strftime("%m-%d %H:%M") + lookup[title] = time_str + except Exception: + continue + return lookup + + +def match_pub_time(items, lookup): + for item in items: + if "pub_time" in item: + continue + title = item.get("title", "").strip() + if title in lookup: + item["pub_time"] = lookup[title] + else: + for raw_title, time_val in lookup.items(): + short = min(len(raw_title), len(title)) + if short > 10 and (raw_title[:short] in title or title[:short] in raw_title): + item["pub_time"] = time_val + break + + def summarize(items): news_text = "" - for i, item in enumerate(items[:100], 1): + for i, item in enumerate(items[:150], 1): title = item.get("title", "") summary = item.get("summary", "")[:200].replace("<", "").replace(">", "") source = item.get("source", "") link = item.get("link", "") news_text += f"{i}. [{source}] {title}\n {summary}\n url: {link}\n\n" - prompt = f"""你是一个政策分析助手。请从原始政策新闻中精选出**最重要的政策动态**，按优先级排序。 + prompt = f"""你是一个资深的政策新闻编辑助手。精选当下最重要的中国政策新闻。 ## 选稿标准 -- 国务院/发改委/部委最新政策文件 -- 习近平总书记重要讲话/活动 -- 重大经济/产业政策（数字经济、AI、能源、金融） -- 重大改革举措 -- 重要人事任免 - -## 输出 -- 分类名：**政策速递**（只有一个分类） -- 每条：**15-30字完整标题**（subtitle），**300-500字政策解读摘要**（summary） -- 总计 **10-20 条**，按重要性排序 +- 国务院常务会议、中央政治局会议等高层决策 +- 重要法律法规的颁布或修订 +- 部委级重大政策发布（发改委、财政部、央行、工信部等） +- 经济政策调整（货币政策、财政政策、产业政策） +- 民生政策（教育、医疗、住房、养老、就业） +- 外交政策重大声明 ## 输出格式 -每条输出到 items 数组，包含：rank, subtitle, title, source, link, summary -summary 用自然段落，关键信息用 **加粗**。 +每条包含： +- subtitle：15-30字完整标题（中文） +- title：原标题（必须与原始新闻标题完全一致，这是时间匹配的关键） +- source：来源 +- link：原文链接 +- summary：300-500字政策解读摘要，关键信息用**加粗**标出 -## 原始新闻 -{news_text} +精选 **10-20 条** 最重要政策新闻。 -JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}""" +JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"小标题","title":"原标题","source":"来源","link":"https://...","summary":"正文..."}}]}}]}}""" resp = requests.post( API_URL, headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, - json={ - "model": MODEL, - "messages": [{"role": "user", "content": prompt}], - "temperature": 0.3, - "max_tokens": 16000, - }, + json={"model": MODEL, "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, "max_tokens": 20000}, timeout=180, ) @@ -73,49 +111,42 @@ JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtit content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:]) try: - result = json.loads(content) + return json.loads(content) except: - import re as re_ - content = re_.sub(r',\s*}', '}', content) - content = re_.sub(r',\s*]', ']', content) + content = re.sub(r',\s*}', '}', content) + content = re.sub(r',\s*]', ']', content) try: - result = json.loads(content) + return json.loads(content) except: - print(f"JSON parse failed, len={len(content)}") - print(f"First 500: {content[:500]}") + print(f"JSON parse failed") return None - return result - -def run(): +if __name__ == "__main__": now = datetime.now(tz).strftime("%Y-%m-%d %H:%M") - items = load_raw() - if not items: - print("No policy data") + raw_items = load_raw() + if not raw_items: + print("No raw policy data") exit(0) - print(f"Loaded {len(items)} policy items. AI summarizing...") - data = summarize(items) - + print(f"Loaded {len(raw_items)} raw items. Summarizing...") + data = summarize(raw_items) if not data: - print("Summarization failed") exit(1) - result = { - "updated_at": now, - "categories": data.get("categories", []), - } + # Back-fill pub_time + time_lookup = build_time_lookup(raw_items) + matched = 0 + for cat in data.get("categories", []): + match_pub_time(cat["items"], time_lookup) + matched += sum(1 for item in cat["items"] if "pub_time" in item) + print(f"Matched pub_time for {matched} articles") + result = {"updated_at": now, "categories": data.get("categories", [])} total = sum(len(c["items"]) for c in result["categories"]) + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) with open(OUTPUT, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) - print(f"Written {total} policy briefs -> {OUTPUT}") - for c in result["categories"]: - print(f" {c['name']}: {len(c['items'])} items") - - -if __name__ == "__main__": - run() + print(f"Written {total} articles -> {OUTPUT}") diff --git a/scripts/summarize_research.py b/scripts/summarize_research.py index 44fadc0..c739f1a 100644 --- a/scripts/summarize_research.py +++ b/scripts/summarize_research.py @@ -1,6 +1,9 @@ -"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs.""" +"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs. +v2: back-fills pub_time from raw RSS data after AI summarization. +""" import json, os, requests, re from datetime import datetime, timezone, timedelta +from dateutil import parser as dtparser API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" API_URL = "https://api.deepseek.com/v1/chat/completions" @@ -19,6 +22,46 @@ def load_raw(): return data.get("items", []) +def build_time_lookup(raw_items): + """Build lookup: title -> pub_time (Beijing time string).""" + lookup = {} + for item in raw_items: + title = item.get("title", "").strip() + published = item.get("published", "").strip() + if not title or not published: + continue + try: + dt = dtparser.parse(published) + if dt.tzinfo: + dt = dt.astimezone(tz) + else: + dt = dt.replace(tzinfo=tz) + now = datetime.now(tz) + if dt.date() == now.date(): + time_str = dt.strftime("%H:%M") + else: + time_str = dt.strftime("%m-%d %H:%M") + lookup[title] = time_str + except Exception: + continue + return lookup + + +def match_pub_time(items, lookup): + for item in items: + if "pub_time" in item: + continue + title = item.get("title", "").strip() + if title in lookup: + item["pub_time"] = lookup[title] + else: + for raw_title, time_val in lookup.items(): + short = min(len(raw_title), len(title)) + if short > 10 and (raw_title[:short] in title or title[:short] in raw_title): + item["pub_time"] = time_val + break + + def summarize(items): news_text = "" for i, item in enumerate(items[:100], 1): @@ -41,7 +84,7 @@ def summarize(items): - 行业动态（AI 公司、产品发布、监管动态） ## 输出格式 -每条：rank, subtitle（15-30字）, title, source, link, summary（200-400字，说清楚这篇讲了什么、有什么用） +每条：rank, subtitle（15-30字）, title（必须与原始标题完全一致，这是时间匹配的关键）, source, link, summary（200-400字，说清楚这篇讲了什么、有什么用） ## 原始内容 {news_text} @@ -72,9 +115,8 @@ JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle" try: result = json.loads(content) except: - import re as re_ - content = re_.sub(r',\s*}', '}', content) - content = re_.sub(r',\s*]', ']', content) + content = re.sub(r',\s*}', '}', content) + content = re.sub(r',\s*]', ']', content) try: result = json.loads(content) except: @@ -86,18 +128,26 @@ JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle" def run(): now = datetime.now(tz).strftime("%Y-%m-%d %H:%M") - items = load_raw() - if not items: + raw_items = load_raw() + if not raw_items: print("No research data") exit(0) - print(f"Loaded {len(items)} research items. AI summarizing...") - data = summarize(items) + print(f"Loaded {len(raw_items)} research items. AI summarizing...") + data = summarize(raw_items) if not data: print("Summarization failed") exit(1) + # Back-fill pub_time + time_lookup = build_time_lookup(raw_items) + matched = 0 + for cat in data.get("categories", []): + match_pub_time(cat["items"], time_lookup) + matched += sum(1 for item in cat["items"] if "pub_time" in item) + print(f"Matched pub_time for {matched} articles") + result = { "updated_at": now, "categories": data.get("categories", []),