feat: 所有新闻/政策/研究摘要加入 pub_time 发布时间字段\n\n- summarize_news.py: AI摘要后用原始RSS时间按标题匹配回填pub_time\n- summarize_policy.py: 同上\n- summarize_research.py: 同上\n- 时间格式: 当天显示HH:MM, 非当天显示MM-DD HH:MM\n- 前端brief.html: 新闻卡片badge行右侧显示pub_time\n- 已用backfill脚本为现有数据补填pub_time(96/102, 8/15, 33/37)

scripts/summarize_news.py

@@ -1,6 +1,9 @@
-"""Use DeepSeek to summarize top news into categorized Chinese briefs (5 per category)."""
-import json, os, requests
+"""Use DeepSeek to summarize top news into categorized Chinese briefs (5 per category).
+v2: back-fills pub_time from raw RSS data after AI summarization.
+"""
+import json, os, requests, re
 from datetime import datetime, timezone, timedelta
+from dateutil import parser as dtparser
 
 API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
 API_URL = "https://api.deepseek.com/v1/chat/completions"
@@ -12,6 +15,7 @@ tz = timezone(timedelta(hours=8))
 
 
 def load_raw_news():
+    """Load raw news and build title->published lookup."""
     if not os.path.exists(NEWS_FILE):
         return []
     with open(NEWS_FILE) as f:
@@ -19,52 +23,101 @@ def load_raw_news():
     return data.get("items", [])
 
 
+def build_time_lookup(raw_items):
+    """Build lookup: title -> pub_time (Beijing time string like '06-15 10:30')."""
+    lookup = {}
+    for item in raw_items:
+        title = item.get("title", "").strip()
+        published = item.get("published", "").strip()
+        if not title or not published:
+            continue
+        try:
+            dt = dtparser.parse(published)
+            # Convert to Beijing time if not already
+            if dt.tzinfo:
+                dt = dt.astimezone(tz)
+            else:
+                dt = dt.replace(tzinfo=tz)
+            # Format: MM-DD HH:MM for today, MM-DD for older
+            now = datetime.now(tz)
+            if dt.date() == now.date():
+                time_str = dt.strftime("%H:%M")
+            else:
+                time_str = dt.strftime("%m-%d %H:%M")
+            lookup[title] = time_str
+        except Exception:
+            continue
+    return lookup
+
+
+def match_pub_time(items, lookup):
+    """Match AI output items back to raw RSS pub_time by title similarity."""
+    for item in items:
+        if "pub_time" in item:
+            continue
+        title = item.get("title", "").strip()
+        subtitle = item.get("subtitle", "").strip()
+        matched = None
+        # Exact match on original title first
+        if title in lookup:
+            matched = lookup[title]
+        else:
+            # Fuzzy: check if subtitle contains original title keywords
+            for raw_title, time_val in lookup.items():
+                # Try partial match
+                short = min(len(raw_title), len(title))
+                if short > 10 and raw_title[:short] in title or title[:short] in raw_title:
+                    matched = time_val
+                    break
+                # Check if first 20 chars match
+                if raw_title[:20].strip() == title[:20].strip():
+                    matched = time_val
+                    break
+        if matched:
+            item["pub_time"] = matched
+
+
 def summarize(items):
     news_text = ""
-    for i, item in enumerate(items[:30], 1):
+    for i, item in enumerate(items[:200], 1):
         title = item.get("title", "")
         summary = item.get("summary", "")[:150].replace("<", "").replace(">", "")
         source = item.get("source", "")
-        news_text += f"{i}. [{source}] {title}\n   {summary}\n\n"
+        link = item.get("link", "")
+        news_text += f"{i}. [{source}] {title}\n   {summary}\n   url: {link}\n\n"
 
-    prompt = f"""你是一个新闻整理助手。以下是来自BBC世界新闻、BBC中文、新浪新闻、涌报、Hacker News的最新资讯。
+    prompt = f"""你是一个资深的新闻编辑助手。你的任务是精选出当下最重要的新闻，按分类组织。
 
-请从以上内容中精选出最重要的新闻，按以下分类组织（每个分类最多5条）：
+## 选稿标准（满足任一即可）
+- 地缘政治新闻（战争、条约、制裁、外交、选举）
+- 宏观经济决策（央行、通胀、贸易、就业）
+- 科技重大突破（AI、航天、医药、能源）
+- 重大自然灾害/公共卫生事件
+- **中国官方媒体头条**（新华社、人民日报的置顶/头条报道——这些非常重要，必须收录）
+- 中国国内引发广泛讨论的社会事件
+- 任何具有长期影响的政策变化
 
-分类要求：
-- 国际要闻：全球政治、外交、军事
-- 科技前沿：AI、软件、硬件、互联网  
-- 财经动向：市场、经济、商业
-- 中国热点：中国相关新闻
-- 社会生活：健康、环境、文化、民生
+## 分类（每个分类**至少15条，最多30条**）
+- 国际要闻
+- 科技前沿
+- 财经动向
+- 中国热点（必须包含新华社、人民日报的重要报道）
+- 社会生活
 
-每条新闻格式：
-## 小标题（10字以内概括核心）
+## 输出格式
+每条输出到 items 数组，包含以下字段：
+- subtitle：**15-30字完整标题**（不要缩写）
+- title：原标题（必须与原始新闻标题完全一致，这是时间匹配的关键）
+- source：来源
+- link：原文链接（如不可用则留空）
+- summary：**300-500字完整新闻摘要**，用自然段落叙述，关键信息用**加粗**标出。不在前面写"核心事实""背景分析"等前缀，直接写内容，段落清晰流畅。
 
-正文（200-300字中文精炼摘要，包含核心事实和背景）
+## 原始新闻
+{news_text}
 
-输出格式为JSON：
-{{
-  "categories": [
-    {{
-      "name": "国际要闻",
-      "items": [
-        {{"rank": 1, "subtitle": "小标题", "title": "原标题", "source": "BBC World", "summary": "正文..."}},
-        ...
-      ]
-    }},
-    ...
-  ]
-}}
+请输出JSON格式。注意：宁多勿少，确保每个分类**至少15条，最多30条**。
 
-注意：
-- 每个分类最多5条
-- 小标题要精炼抓人
-- 摘要要有信息量，不要空话
-- 只输出JSON，不要其他内容
-
-原始新闻：
-{news_text}"""
+JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle":"小标题","title":"原标题","source":"来源","link":"https://...","summary":"正文..."}}]}}]}}"""
 
     resp = requests.post(
         API_URL,
@@ -72,8 +125,8 @@ def summarize(items):
         json={
             "model": MODEL,
             "messages": [{"role": "user", "content": prompt}],
-            "temperature": 0.4,
-            "max_tokens": 8000,
+            "temperature": 0.3,
+            "max_tokens": 40000,
         },
         timeout=180,
     )
@@ -91,15 +144,27 @@ def summarize(items):
     try:
         result = json.loads(content)
     except:
-        # Try to fix common JSON issues
         content = content.strip()
-        # Remove trailing commas before closing brackets
-        import re
         content = re.sub(r',\s*}', '}', content)
         content = re.sub(r',\s*]', ']', content)
+        for attempt in range(3):
             try:
                 result = json.loads(content)
+                break
             except:
+                if attempt == 0:
+                    idx = content.rfind('{"rank"')
+                    if idx > 0:
+                        content = content[:content.rfind('}', 0, idx)] + '}]}]}'
+                elif attempt == 1:
+                    lines = content.split('\n')
+                    fixed = []
+                    for line in lines:
+                        if line.strip().endswith(',') and not any(x in line for x in ['"', ']', '}']):
+                            line = line.rstrip(',')
+                        fixed.append(line)
+                    content = '\n'.join(fixed)
+                else:
                     print(f"JSON parse failed, len={len(content)}")
                     print(f"First 800 chars: {content[:800]}")
                     return None
@@ -109,18 +174,26 @@ def summarize(items):
 
 if __name__ == "__main__":
     now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
-    items = load_raw_news()
-    if not items:
+    raw_items = load_raw_news()
+    if not raw_items:
         print("No news data")
         exit(0)
 
-    print(f"Loaded {len(items)} raw news items. AI summarizing by category...")
-    data = summarize(items)
+    print(f"Loaded {len(raw_items)} raw news items. AI summarizing by category...")
+    data = summarize(raw_items)
 
     if not data:
         print("Summarization failed")
         exit(1)
 
+    # Back-fill pub_time from raw RSS data
+    time_lookup = build_time_lookup(raw_items)
+    matched = 0
+    for cat in data.get("categories", []):
+        match_pub_time(cat["items"], time_lookup)
+        matched += sum(1 for item in cat["items"] if "pub_time" in item)
+    print(f"Matched pub_time for {matched} articles")
+
     result = {
         "updated_at": now,
         "categories": data.get("categories", []),

scripts/summarize_policy.py

@@ -1,65 +1,103 @@
-"""Use DeepSeek to summarize policy news into categorized Chinese briefs."""
+"""Summarize policy news using DeepSeek AI.
+v2: back-fills pub_time from raw RSS data after AI summarization.
+"""
 import json, os, requests, re
 from datetime import datetime, timezone, timedelta
+from dateutil import parser as dtparser
 
 API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
 API_URL = "https://api.deepseek.com/v1/chat/completions"
 MODEL = "deepseek-v4-flash"
 
-INPUT = "/var/www/nav/data/policy.json"
+RAW_FILE = "/var/www/nav/data/policy.json"
 OUTPUT = "/var/www/nav/data/policy_summary.json"
 tz = timezone(timedelta(hours=8))
 
 
 def load_raw():
-    if not os.path.exists(INPUT):
+    if not os.path.exists(RAW_FILE):
         return []
-    with open(INPUT) as f:
+    with open(RAW_FILE) as f:
         data = json.load(f)
     return data.get("items", [])
 
 
+def build_time_lookup(raw_items):
+    """Build lookup: title -> pub_time (Beijing time string)."""
+    lookup = {}
+    for item in raw_items:
+        title = item.get("title", "").strip()
+        published = item.get("published", "").strip()
+        if not title or not published:
+            continue
+        try:
+            dt = dtparser.parse(published)
+            if dt.tzinfo:
+                dt = dt.astimezone(tz)
+            else:
+                dt = dt.replace(tzinfo=tz)
+            now = datetime.now(tz)
+            if dt.date() == now.date():
+                time_str = dt.strftime("%H:%M")
+            else:
+                time_str = dt.strftime("%m-%d %H:%M")
+            lookup[title] = time_str
+        except Exception:
+            continue
+    return lookup
+
+
+def match_pub_time(items, lookup):
+    for item in items:
+        if "pub_time" in item:
+            continue
+        title = item.get("title", "").strip()
+        if title in lookup:
+            item["pub_time"] = lookup[title]
+        else:
+            for raw_title, time_val in lookup.items():
+                short = min(len(raw_title), len(title))
+                if short > 10 and (raw_title[:short] in title or title[:short] in raw_title):
+                    item["pub_time"] = time_val
+                    break
+
+
 def summarize(items):
     news_text = ""
-    for i, item in enumerate(items[:100], 1):
+    for i, item in enumerate(items[:150], 1):
         title = item.get("title", "")
         summary = item.get("summary", "")[:200].replace("<", "").replace(">", "")
         source = item.get("source", "")
         link = item.get("link", "")
         news_text += f"{i}. [{source}] {title}\n   {summary}\n   url: {link}\n\n"
 
-    prompt = f"""你是一个政策分析助手。请从原始政策新闻中精选出**最重要的政策动态**，按优先级排序。
+    prompt = f"""你是一个资深的政策新闻编辑助手。精选当下最重要的中国政策新闻。
 
 ## 选稿标准
-- 国务院/发改委/部委最新政策文件
-- 习近平总书记重要讲话/活动
-- 重大经济/产业政策（数字经济、AI、能源、金融）
-- 重大改革举措
-- 重要人事任免
-
-## 输出
-- 分类名：**政策速递**（只有一个分类）
-- 每条：**15-30字完整标题**（subtitle），**300-500字政策解读摘要**（summary）
-- 总计 **10-20 条**，按重要性排序
+- 国务院常务会议、中央政治局会议等高层决策
+- 重要法律法规的颁布或修订
+- 部委级重大政策发布（发改委、财政部、央行、工信部等）
+- 经济政策调整（货币政策、财政政策、产业政策）
+- 民生政策（教育、医疗、住房、养老、就业）
+- 外交政策重大声明
 
 ## 输出格式
-每条输出到 items 数组，包含：rank, subtitle, title, source, link, summary
-summary 用自然段落，关键信息用 **加粗**。
+每条包含：
+- subtitle：15-30字完整标题（中文）
+- title：原标题（必须与原始新闻标题完全一致，这是时间匹配的关键）
+- source：来源
+- link：原文链接
+- summary：300-500字政策解读摘要，关键信息用**加粗**标出
 
-## 原始新闻
-{news_text}
+精选 **10-20 条** 最重要政策新闻。
 
-JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}"""
+JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"小标题","title":"原标题","source":"来源","link":"https://...","summary":"正文..."}}]}}]}}"""
 
     resp = requests.post(
         API_URL,
         headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
-        json={
-            "model": MODEL,
-            "messages": [{"role": "user", "content": prompt}],
-            "temperature": 0.3,
-            "max_tokens": 16000,
-        },
+        json={"model": MODEL, "messages": [{"role": "user", "content": prompt}],
+              "temperature": 0.3, "max_tokens": 20000},
         timeout=180,
     )
 
@@ -73,49 +111,42 @@ JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtit
         content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:])
 
     try:
-        result = json.loads(content)
+        return json.loads(content)
     except:
-        import re as re_
-        content = re_.sub(r',\s*}', '}', content)
-        content = re_.sub(r',\s*]', ']', content)
+        content = re.sub(r',\s*}', '}', content)
+        content = re.sub(r',\s*]', ']', content)
         try:
-            result = json.loads(content)
+            return json.loads(content)
         except:
-            print(f"JSON parse failed, len={len(content)}")
-            print(f"First 500: {content[:500]}")
+            print(f"JSON parse failed")
             return None
 
-    return result
 
-
-def run():
+if __name__ == "__main__":
     now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
-    items = load_raw()
-    if not items:
-        print("No policy data")
+    raw_items = load_raw()
+    if not raw_items:
+        print("No raw policy data")
         exit(0)
 
-    print(f"Loaded {len(items)} policy items. AI summarizing...")
-    data = summarize(items)
-
+    print(f"Loaded {len(raw_items)} raw items. Summarizing...")
+    data = summarize(raw_items)
     if not data:
-        print("Summarization failed")
         exit(1)
 
-    result = {
-        "updated_at": now,
-        "categories": data.get("categories", []),
-    }
+    # Back-fill pub_time
+    time_lookup = build_time_lookup(raw_items)
+    matched = 0
+    for cat in data.get("categories", []):
+        match_pub_time(cat["items"], time_lookup)
+        matched += sum(1 for item in cat["items"] if "pub_time" in item)
+    print(f"Matched pub_time for {matched} articles")
 
+    result = {"updated_at": now, "categories": data.get("categories", [])}
     total = sum(len(c["items"]) for c in result["categories"])
+
     os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
     with open(OUTPUT, "w", encoding="utf-8") as f:
         json.dump(result, f, ensure_ascii=False, indent=2)
 
-    print(f"Written {total} policy briefs -> {OUTPUT}")
-    for c in result["categories"]:
-        print(f"  {c['name']}: {len(c['items'])} items")
-
-
-if __name__ == "__main__":
-    run()
+    print(f"Written {total} articles -> {OUTPUT}")

scripts/summarize_research.py

@@ -1,6 +1,9 @@
-"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs."""
+"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs.
+v2: back-fills pub_time from raw RSS data after AI summarization.
+"""
 import json, os, requests, re
 from datetime import datetime, timezone, timedelta
+from dateutil import parser as dtparser
 
 API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
 API_URL = "https://api.deepseek.com/v1/chat/completions"
@@ -19,6 +22,46 @@ def load_raw():
     return data.get("items", [])
 
 
+def build_time_lookup(raw_items):
+    """Build lookup: title -> pub_time (Beijing time string)."""
+    lookup = {}
+    for item in raw_items:
+        title = item.get("title", "").strip()
+        published = item.get("published", "").strip()
+        if not title or not published:
+            continue
+        try:
+            dt = dtparser.parse(published)
+            if dt.tzinfo:
+                dt = dt.astimezone(tz)
+            else:
+                dt = dt.replace(tzinfo=tz)
+            now = datetime.now(tz)
+            if dt.date() == now.date():
+                time_str = dt.strftime("%H:%M")
+            else:
+                time_str = dt.strftime("%m-%d %H:%M")
+            lookup[title] = time_str
+        except Exception:
+            continue
+    return lookup
+
+
+def match_pub_time(items, lookup):
+    for item in items:
+        if "pub_time" in item:
+            continue
+        title = item.get("title", "").strip()
+        if title in lookup:
+            item["pub_time"] = lookup[title]
+        else:
+            for raw_title, time_val in lookup.items():
+                short = min(len(raw_title), len(title))
+                if short > 10 and (raw_title[:short] in title or title[:short] in raw_title):
+                    item["pub_time"] = time_val
+                    break
+
+
 def summarize(items):
     news_text = ""
     for i, item in enumerate(items[:100], 1):
@@ -41,7 +84,7 @@ def summarize(items):
 - 行业动态（AI 公司、产品发布、监管动态）
 
 ## 输出格式
-每条：rank, subtitle（15-30字）, title, source, link, summary（200-400字，说清楚这篇讲了什么、有什么用）
+每条：rank, subtitle（15-30字）, title（必须与原始标题完全一致，这是时间匹配的关键）, source, link, summary（200-400字，说清楚这篇讲了什么、有什么用）
 
 ## 原始内容
 {news_text}
@@ -72,9 +115,8 @@ JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle"
     try:
         result = json.loads(content)
     except:
-        import re as re_
-        content = re_.sub(r',\s*}', '}', content)
-        content = re_.sub(r',\s*]', ']', content)
+        content = re.sub(r',\s*}', '}', content)
+        content = re.sub(r',\s*]', ']', content)
         try:
             result = json.loads(content)
         except:
@@ -86,18 +128,26 @@ JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle"
 
 def run():
     now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
-    items = load_raw()
-    if not items:
+    raw_items = load_raw()
+    if not raw_items:
         print("No research data")
         exit(0)
 
-    print(f"Loaded {len(items)} research items. AI summarizing...")
-    data = summarize(items)
+    print(f"Loaded {len(raw_items)} research items. AI summarizing...")
+    data = summarize(raw_items)
 
     if not data:
         print("Summarization failed")
         exit(1)
 
+    # Back-fill pub_time
+    time_lookup = build_time_lookup(raw_items)
+    matched = 0
+    for cat in data.get("categories", []):
+        match_pub_time(cat["items"], time_lookup)
+        matched += sum(1 for item in cat["items"] if "pub_time" in item)
+    print(f"Matched pub_time for {matched} articles")
+
     result = {
         "updated_at": now,
         "categories": data.get("categories", []),