feat: add policy/research/market/deep-dive modules for daily brief expansion

scripts/fetch_research.py

@@ -0,0 +1,77 @@
+"""Fetch AI/quant research RSS: ArXiv cs.AI + quant blogs."""
+import json, os, feedparser
+from datetime import datetime, timezone, timedelta
+
+OUTPUT = "/var/www/nav/data/research.json"
+tz = timezone(timedelta(hours=8))
+
+FEEDS = [
+    # ArXiv AI (via arxiv-rss.appspot.com or direct)
+    {"name": "ArXiv cs.AI", "url": "https://rss.arxiv.org/rss/cs.AI", "lang": "en"},
+    {"name": "ArXiv q-fin", "url": "https://rss.arxiv.org/rss/q-fin", "lang": "en"},
+    # Quant blogs
+    {"name": "Quantitative Finance Stack Exchange", "url": "https://quant.stackexchange.com/feeds", "lang": "en"},
+    {"name": "Alpha Vantage Blog", "url": "https://blog.alphavantage.co/rss.xml", "lang": "en"},
+    # 机器之心（中文 AI 资讯）
+    {"name": "机器之心", "url": "https://www.jiqizhixin.com/rss", "lang": "zh"},
+    # Towards Data Science
+    {"name": "Towards Data Science", "url": "https://towardsdatascience.com/feed", "lang": "en"},
+    # AI Alignment Forum
+    {"name": "AI Alignment Forum", "url": "https://www.alignmentforum.org/feed.xml", "lang": "en"},
+]
+
+MAX_PER_FEED = 20
+MAX_TOTAL = 150
+
+
+def fetch_feed(feed_info):
+    try:
+        feed = feedparser.parse(feed_info["url"])
+        items = []
+        for entry in feed.entries[:MAX_PER_FEED]:
+            items.append({
+                "title": entry.get("title", ""),
+                "link": entry.get("link", ""),
+                "summary": entry.get("summary", "")[:400],
+                "published": entry.get("published", ""),
+                "source": feed_info["name"],
+                "lang": feed_info["lang"],
+            })
+        print(f"  {feed_info['name']}: {len(items)} items")
+        return items
+    except Exception as e:
+        print(f"  {feed_info['name']}: ERROR - {e}")
+        return []
+
+
+def fetch_research():
+    items = []
+    for feed_info in FEEDS:
+        items.extend(fetch_feed(feed_info))
+
+    seen = set()
+    unique = []
+    for item in items:
+        key = item["title"][:30]
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "total": len(unique),
+        "items": unique[:MAX_TOTAL],
+        "sources": [f["name"] for f in FEEDS],
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"\nTotal: {len(unique)} unique research items -> {OUTPUT}")
+    return result
+
+
+if __name__ == "__main__":
+    print(f"Fetching research RSS at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
+    fetch_research()