feat: add policy/research/market/deep-dive modules for daily brief expansion

scripts/fetch_policy.py

@@ -0,0 +1,80 @@
+"""Fetch policy news from Chinese official sources RSS."""
+import json, os, feedparser
+from datetime import datetime, timezone, timedelta
+
+OUTPUT = "/var/www/nav/data/policy.json"
+tz = timezone(timedelta(hours=8))
+
+FEEDS = [
+    # 新华社
+    {"name": "新华网·时政", "url": "http://www.xinhuanet.com/politics/news_politics.xml", "lang": "zh"},
+    {"name": "新华网·财经", "url": "http://www.xinhuanet.com/fortune/news_fortune.xml", "lang": "zh"},
+    {"name": "新华网·国际", "url": "http://www.xinhuanet.com/world/news_world.xml", "lang": "zh"},
+    # 人民网
+    {"name": "人民网·时政", "url": "http://www.people.com.cn/rss/politics.xml", "lang": "zh"},
+    # 中国政府网（国务院）
+    {"name": "中国政府网", "url": "http://www.gov.cn/xinwen/yaowen.htm", "lang": "zh"},
+    # 发改委（尝试 RSS，若无则用新华网发改委专题）
+    {"name": "新华网·财经政策", "url": "https://www.xinhuanet.com/fortune/policy.xml", "lang": "zh"},
+    # 财新（需确认 RSS 可用性）
+    {"name": "财新网", "url": "https://rss.caixin.com/conf/rss/valsart.xml", "lang": "zh"},
+    # 观察者网（政策解读强）
+    {"name": "观察者网", "url": "https://www.guancha.cn/index.rss", "lang": "zh"},
+]
+
+MAX_PER_FEED = 30
+MAX_TOTAL = 200
+
+
+def fetch_feed(feed_info):
+    try:
+        feed = feedparser.parse(feed_info["url"])
+        items = []
+        for entry in feed.entries[:MAX_PER_FEED]:
+            items.append({
+                "title": entry.get("title", ""),
+                "link": entry.get("link", ""),
+                "summary": entry.get("summary", "")[:300],
+                "published": entry.get("published", ""),
+                "source": feed_info["name"],
+                "lang": feed_info["lang"],
+            })
+        print(f"  {feed_info['name']}: {len(items)} items")
+        return items
+    except Exception as e:
+        print(f"  {feed_info['name']}: ERROR - {e}")
+        return []
+
+
+def fetch_policy():
+    items = []
+    for feed_info in FEEDS:
+        items.extend(fetch_feed(feed_info))
+
+    # Deduplicate by title
+    seen = set()
+    unique = []
+    for item in items:
+        key = item["title"][:30]
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "total": len(unique),
+        "items": unique[:MAX_TOTAL],
+        "sources": [f["name"] for f in FEEDS],
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"\nTotal: {len(unique)} unique items -> {OUTPUT}")
+    return result
+
+
+if __name__ == "__main__":
+    print(f"Fetching policy news at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
+    fetch_policy()