feat: add policy/research/market/deep-dive modules for daily brief expansion

2026-06-15 12:36:43 +08:00
parent 36a507e1d3
commit f31c741abd
6 changed files with 587 additions and 0 deletions
--- a/scripts/deep_dive.py
+++ b/scripts/deep_dive.py
@ -0,0 +1,105 @@
+"""Deep dive: pick top news and generate in-depth analysis via DeepSeek."""
+import json, os, requests
+from datetime import datetime, timezone, timedelta
+
+API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
+API_URL = "https://api.deepseek.com/v1/chat/completions"
+MODEL = "deepseek-v4-flash"
+
+NEWS_SUMMARY = "/var/www/nav/data/news_summary.json"
+OUTPUT = "/var/www/nav/data/deep_dive.json"
+tz = timezone(timedelta(hours=8))
+
+
+def load_top_news():
+    if not os.path.exists(NEWS_SUMMARY):
+        return None
+    with open(NEWS_SUMMARY) as f:
+        data = json.load(f)
+    # Pick the #1 item from the first category (usually 国际要闻 or most important)
+    for cat in data.get("categories", []):
+        if cat.get("items"):
+            return cat["items"][0], cat["name"]
+    return None, None
+
+
+def generate_deep_dive(top_item, category):
+    title = top_item.get("title", "")
+    subtitle = top_item.get("subtitle", "")
+    summary = top_item.get("summary", "")
+    source = top_item.get("source", "")
+    link = top_item.get("link", "")
+
+    prompt = f"""你是一位资深的国际事务/财经分析员。用户看到了一条新闻的简要摘要，请你写一篇深度解读，帮助用户理解这条新闻的**完整背景、深层含义和未来走向**。
+
+## 新闻信息
+- 分类：{category}
+- 标题：{subtitle}
+- 原文标题：{title}
+- 来源：{source}
+- 原文链接：{link}
+- 简要摘要：{summary}
+
+## 写作要求
+1. **800-1200 字**，中文，客观专业
+2. **结构清晰**：事件背景 → 关键事实 → 各方立场 → 深层分析 → 可能走向 → 对中国/市场的影响
+3. **不重复摘要内容**，要在摘要基础上做增量分析
+4. **引用具体数据/事件**支撑论点，不要空泛议论
+5. **结尾一句话总结**这条新闻对用户（中国中层干部、量化交易者）的核心启示
+6. 关键信息用 **加粗** 标出
+
+请直接输出正文，不要输出 JSON。"""
+
+    resp = requests.post(
+        API_URL,
+        headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
+        json={
+            "model": MODEL,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0.5,
+            "max_tokens": 4000,
+        },
+        timeout=120,
+    )
+
+    if resp.status_code != 200:
+        print(f"API error: {resp.status_code}")
+        return None
+
+    content = resp.json()["choices"][0]["message"]["content"]
+    return content.strip()
+
+
+def run():
+    top_item, category = load_top_news()
+    if not top_item:
+        print("No news data available for deep dive")
+        return
+
+    print(f"Generating deep dive for: {top_item.get('subtitle', '')[:50]}...")
+
+    analysis = generate_deep_dive(top_item, category)
+    if not analysis:
+        print("Deep dive generation failed")
+        return
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "source_news": {
+            "subtitle": top_item.get("subtitle", ""),
+            "title": top_item.get("title", ""),
+            "link": top_item.get("link", ""),
+            "category": category,
+        },
+        "analysis": analysis,
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"Deep dive written -> {OUTPUT}")
+
+
+if __name__ == "__main__":
+    run()
--- a/scripts/fetch_market.py
+++ b/scripts/fetch_market.py
@ -0,0 +1,87 @@
+"""Fetch market data: major indices, crypto, forex via public APIs (no API key needed)."""
+import json, os, requests
+from datetime import datetime, timezone, timedelta
+
+OUTPUT = "/var/www/nav/data/market.json"
+tz = timezone(timedelta(hours=8))
+
+# Public endpoints (no API key)
+COINGECKO_URL = "https://api.coingecko.com/api/v3/simple/price?ids=bitcoin,ethereum,solana,binancecoin,ripple&vs_currencies=usd&include_24hr_change=true"
+# A-share indices via eastmoney (free, no key)
+EM_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get?fltt=2&secids=1.000001,0.399001,0.399006&fields=f2,f3,f4,f12,f14"
+
+
+def fetch_crypto():
+    try:
+        resp = requests.get(COINGECKO_URL, timeout=10)
+        data = resp.json()
+        items = []
+        icons = {"bitcoin": "₿", "ethereum": "Ξ", "solana": "◎", "binancecoin": "BNB", "ripple": "XRP"}
+        names = {"bitcoin": "BTC", "ethereum": "ETH", "solana": "SOL", "binancecoin": "BNB", "ripple": "XRP"}
+        for coin_id, icon in icons.items():
+            if coin_id not in data:
+                continue
+            price = data[coin_id]["usd"]
+            change = data[coin_id].get("usd_24h_change", 0) or 0
+            direction = "▲" if change >= 0 else "▼"
+            items.append({
+                "type": "crypto",
+                "symbol": names[coin_id],
+                "icon": icon,
+                "price": f"${price:,.2f}",
+                "change_pct": f"{direction} {abs(change):.2f}%",
+                "is_up": change >= 0,
+            })
+        return items
+    except Exception as e:
+        print(f"  Crypto error: {e}")
+        return []
+
+
+def fetch_cn_indices():
+    """Fetch A-share indices via Eastmoney API (no key)."""
+    try:
+        resp = requests.get(EM_URL, timeout=10)
+        data = resp.json()
+        items = []
+        for diff in data.get("data", {}).get("diff", []):
+            name = diff.get("f14", "")
+            price = diff.get("f2", 0)
+            change = diff.get("f4", 0)
+            pct = diff.get("f3", 0)
+            direction = "▲" if change >= 0 else "▼"
+            items.append({
+                "type": "cn_index",
+                "symbol": name,
+                "price": f"{price:.2f}",
+                "change": f"{direction} {abs(change):.2f}",
+                "change_pct": f"{direction} {abs(pct):.2f}%",
+                "is_up": change >= 0,
+            })
+        return items
+    except Exception as e:
+        print(f"  CN indices error: {e}")
+        return []
+
+
+def fetch_market():
+    crypto = fetch_crypto()
+    cn_indices = fetch_cn_indices()
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "crypto": crypto,
+        "cn_indices": cn_indices,
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"Crypto: {len(crypto)} | CN indices: {len(cn_indices)} -> {OUTPUT}")
+    return result
+
+
+if __name__ == "__main__":
+    print(f"Fetching market data at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
+    fetch_market()
--- a/scripts/fetch_policy.py
+++ b/scripts/fetch_policy.py
@ -0,0 +1,80 @@
+"""Fetch policy news from Chinese official sources RSS."""
+import json, os, feedparser
+from datetime import datetime, timezone, timedelta
+
+OUTPUT = "/var/www/nav/data/policy.json"
+tz = timezone(timedelta(hours=8))
+
+FEEDS = [
+    # 新华社
+    {"name": "新华网·时政", "url": "http://www.xinhuanet.com/politics/news_politics.xml", "lang": "zh"},
+    {"name": "新华网·财经", "url": "http://www.xinhuanet.com/fortune/news_fortune.xml", "lang": "zh"},
+    {"name": "新华网·国际", "url": "http://www.xinhuanet.com/world/news_world.xml", "lang": "zh"},
+    # 人民网
+    {"name": "人民网·时政", "url": "http://www.people.com.cn/rss/politics.xml", "lang": "zh"},
+    # 中国政府网（国务院）
+    {"name": "中国政府网", "url": "http://www.gov.cn/xinwen/yaowen.htm", "lang": "zh"},
+    # 发改委（尝试 RSS，若无则用新华网发改委专题）
+    {"name": "新华网·财经政策", "url": "https://www.xinhuanet.com/fortune/policy.xml", "lang": "zh"},
+    # 财新（需确认 RSS 可用性）
+    {"name": "财新网", "url": "https://rss.caixin.com/conf/rss/valsart.xml", "lang": "zh"},
+    # 观察者网（政策解读强）
+    {"name": "观察者网", "url": "https://www.guancha.cn/index.rss", "lang": "zh"},
+]
+
+MAX_PER_FEED = 30
+MAX_TOTAL = 200
+
+
+def fetch_feed(feed_info):
+    try:
+        feed = feedparser.parse(feed_info["url"])
+        items = []
+        for entry in feed.entries[:MAX_PER_FEED]:
+            items.append({
+                "title": entry.get("title", ""),
+                "link": entry.get("link", ""),
+                "summary": entry.get("summary", "")[:300],
+                "published": entry.get("published", ""),
+                "source": feed_info["name"],
+                "lang": feed_info["lang"],
+            })
+        print(f"  {feed_info['name']}: {len(items)} items")
+        return items
+    except Exception as e:
+        print(f"  {feed_info['name']}: ERROR - {e}")
+        return []
+
+
+def fetch_policy():
+    items = []
+    for feed_info in FEEDS:
+        items.extend(fetch_feed(feed_info))
+
+    # Deduplicate by title
+    seen = set()
+    unique = []
+    for item in items:
+        key = item["title"][:30]
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "total": len(unique),
+        "items": unique[:MAX_TOTAL],
+        "sources": [f["name"] for f in FEEDS],
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"\nTotal: {len(unique)} unique items -> {OUTPUT}")
+    return result
+
+
+if __name__ == "__main__":
+    print(f"Fetching policy news at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
+    fetch_policy()
--- a/scripts/fetch_research.py
+++ b/scripts/fetch_research.py
@ -0,0 +1,77 @@
+"""Fetch AI/quant research RSS: ArXiv cs.AI + quant blogs."""
+import json, os, feedparser
+from datetime import datetime, timezone, timedelta
+
+OUTPUT = "/var/www/nav/data/research.json"
+tz = timezone(timedelta(hours=8))
+
+FEEDS = [
+    # ArXiv AI (via arxiv-rss.appspot.com or direct)
+    {"name": "ArXiv cs.AI", "url": "https://rss.arxiv.org/rss/cs.AI", "lang": "en"},
+    {"name": "ArXiv q-fin", "url": "https://rss.arxiv.org/rss/q-fin", "lang": "en"},
+    # Quant blogs
+    {"name": "Quantitative Finance Stack Exchange", "url": "https://quant.stackexchange.com/feeds", "lang": "en"},
+    {"name": "Alpha Vantage Blog", "url": "https://blog.alphavantage.co/rss.xml", "lang": "en"},
+    # 机器之心（中文 AI 资讯）
+    {"name": "机器之心", "url": "https://www.jiqizhixin.com/rss", "lang": "zh"},
+    # Towards Data Science
+    {"name": "Towards Data Science", "url": "https://towardsdatascience.com/feed", "lang": "en"},
+    # AI Alignment Forum
+    {"name": "AI Alignment Forum", "url": "https://www.alignmentforum.org/feed.xml", "lang": "en"},
+]
+
+MAX_PER_FEED = 20
+MAX_TOTAL = 150
+
+
+def fetch_feed(feed_info):
+    try:
+        feed = feedparser.parse(feed_info["url"])
+        items = []
+        for entry in feed.entries[:MAX_PER_FEED]:
+            items.append({
+                "title": entry.get("title", ""),
+                "link": entry.get("link", ""),
+                "summary": entry.get("summary", "")[:400],
+                "published": entry.get("published", ""),
+                "source": feed_info["name"],
+                "lang": feed_info["lang"],
+            })
+        print(f"  {feed_info['name']}: {len(items)} items")
+        return items
+    except Exception as e:
+        print(f"  {feed_info['name']}: ERROR - {e}")
+        return []
+
+
+def fetch_research():
+    items = []
+    for feed_info in FEEDS:
+        items.extend(fetch_feed(feed_info))
+
+    seen = set()
+    unique = []
+    for item in items:
+        key = item["title"][:30]
+        if key not in seen:
+            seen.add(key)
+            unique.append(item)
+
+    result = {
+        "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
+        "total": len(unique),
+        "items": unique[:MAX_TOTAL],
+        "sources": [f["name"] for f in FEEDS],
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"\nTotal: {len(unique)} unique research items -> {OUTPUT}")
+    return result
+
+
+if __name__ == "__main__":
+    print(f"Fetching research RSS at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
+    fetch_research()
--- a/scripts/summarize_policy.py
+++ b/scripts/summarize_policy.py
@ -0,0 +1,121 @@
+"""Use DeepSeek to summarize policy news into categorized Chinese briefs."""
+import json, os, requests, re
+from datetime import datetime, timezone, timedelta
+
+API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
+API_URL = "https://api.deepseek.com/v1/chat/completions"
+MODEL = "deepseek-v4-flash"
+
+INPUT = "/var/www/nav/data/policy.json"
+OUTPUT = "/var/www/nav/data/policy_summary.json"
+tz = timezone(timedelta(hours=8))
+
+
+def load_raw():
+    if not os.path.exists(INPUT):
+        return []
+    with open(INPUT) as f:
+        data = json.load(f)
+    return data.get("items", [])
+
+
+def summarize(items):
+    news_text = ""
+    for i, item in enumerate(items[:100], 1):
+        title = item.get("title", "")
+        summary = item.get("summary", "")[:200].replace("<", "").replace(">", "")
+        source = item.get("source", "")
+        link = item.get("link", "")
+        news_text += f"{i}. [{source}] {title}\n   {summary}\n   url: {link}\n\n"
+
+    prompt = f"""你是一个政策分析助手。请从原始政策新闻中精选出**最重要的政策动态**，按优先级排序。
+
+## 选稿标准
+- 国务院/发改委/部委最新政策文件
+- 习近平总书记重要讲话/活动
+- 重大经济/产业政策（数字经济、AI、能源、金融）
+- 重大改革举措
+- 重要人事任免
+
+## 输出
+- 分类名：**政策速递**（只有一个分类）
+- 每条：**15-30字完整标题**（subtitle），**300-500字政策解读摘要**（summary）
+- 总计 **10-20 条**，按重要性排序
+
+## 输出格式
+每条输出到 items 数组，包含：rank, subtitle, title, source, link, summary
+summary 用自然段落，关键信息用 **加粗**。
+
+## 原始新闻
+{news_text}
+
+JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}"""
+
+    resp = requests.post(
+        API_URL,
+        headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
+        json={
+            "model": MODEL,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0.3,
+            "max_tokens": 16000,
+        },
+        timeout=180,
+    )
+
+    if resp.status_code != 200:
+        print(f"API error: {resp.status_code}")
+        return None
+
+    content = resp.json()["choices"][0]["message"]["content"].strip()
+    if content.startswith("```"):
+        lines = content.split("\n")
+        content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:])
+
+    try:
+        result = json.loads(content)
+    except:
+        import re as re_
+        content = re_.sub(r',\s*}', '}', content)
+        content = re_.sub(r',\s*]', ']', content)
+        try:
+            result = json.loads(content)
+        except:
+            print(f"JSON parse failed, len={len(content)}")
+            print(f"First 500: {content[:500]}")
+            return None
+
+    return result
+
+
+def run():
+    now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
+    items = load_raw()
+    if not items:
+        print("No policy data")
+        exit(0)
+
+    print(f"Loaded {len(items)} policy items. AI summarizing...")
+    data = summarize(items)
+
+    if not data:
+        print("Summarization failed")
+        exit(1)
+
+    result = {
+        "updated_at": now,
+        "categories": data.get("categories", []),
+    }
+
+    total = sum(len(c["items"]) for c in result["categories"])
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"Written {total} policy briefs -> {OUTPUT}")
+    for c in result["categories"]:
+        print(f"  {c['name']}: {len(c['items'])} items")
+
+
+if __name__ == "__main__":
+    run()
--- a/scripts/summarize_research.py
+++ b/scripts/summarize_research.py
@ -0,0 +1,117 @@
+"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs."""
+import json, os, requests, re
+from datetime import datetime, timezone, timedelta
+
+API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228"
+API_URL = "https://api.deepseek.com/v1/chat/completions"
+MODEL = "deepseek-v4-flash"
+
+INPUT = "/var/www/nav/data/research.json"
+OUTPUT = "/var/www/nav/data/research_summary.json"
+tz = timezone(timedelta(hours=8))
+
+
+def load_raw():
+    if not os.path.exists(INPUT):
+        return []
+    with open(INPUT) as f:
+        data = json.load(f)
+    return data.get("items", [])
+
+
+def summarize(items):
+    news_text = ""
+    for i, item in enumerate(items[:100], 1):
+        title = item.get("title", "")
+        summary = item.get("summary", "")[:300].replace("<", "").replace(">", "")
+        source = item.get("source", "")
+        link = item.get("link", "")
+        news_text += f"{i}. [{source}] {title}\n   {summary}\n   url: {link}\n\n"
+
+    prompt = f"""你是一个 AI/量化研究资讯编辑。请从原始 RSS 中精选出**最有价值的论文和博客文章**，按主题分类。
+
+## 选稿标准
+- ArXiv 论文：AI 重大突破（LLM、Agent、多模态）、量化交易新方法
+- 博客文章：有实操价值的策略思路、代码实现、回测分析
+- 过滤：纯新闻通告、产品广告、重复性内容
+
+## 分类（每个分类 8-15 条）
+- AI 前沿论文（ArXiv cs.AI / cs.LG 重要论文）
+- 量化交易研究（策略思路、回测方法、风险管理）
+- 行业动态（AI 公司、产品发布、监管动态）
+
+## 输出格式
+每条：rank, subtitle（15-30字）, title, source, link, summary（200-400字，说清楚这篇讲了什么、有什么用）
+
+## 原始内容
+{news_text}
+
+JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}"""
+
+    resp = requests.post(
+        API_URL,
+        headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"},
+        json={
+            "model": MODEL,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0.3,
+            "max_tokens": 16000,
+        },
+        timeout=180,
+    )
+
+    if resp.status_code != 200:
+        print(f"API error: {resp.status_code}")
+        return None
+
+    content = resp.json()["choices"][0]["message"]["content"].strip()
+    if content.startswith("```"):
+        lines = content.split("\n")
+        content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:])
+
+    try:
+        result = json.loads(content)
+    except:
+        import re as re_
+        content = re_.sub(r',\s*}', '}', content)
+        content = re_.sub(r',\s*]', ']', content)
+        try:
+            result = json.loads(content)
+        except:
+            print(f"JSON parse failed, len={len(content)}")
+            return None
+
+    return result
+
+
+def run():
+    now = datetime.now(tz).strftime("%Y-%m-%d %H:%M")
+    items = load_raw()
+    if not items:
+        print("No research data")
+        exit(0)
+
+    print(f"Loaded {len(items)} research items. AI summarizing...")
+    data = summarize(items)
+
+    if not data:
+        print("Summarization failed")
+        exit(1)
+
+    result = {
+        "updated_at": now,
+        "categories": data.get("categories", []),
+    }
+
+    total = sum(len(c["items"]) for c in result["categories"])
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+
+    print(f"Written {total} research briefs -> {OUTPUT}")
+    for c in result["categories"]:
+        print(f"  {c['name']}: {len(c['items'])} items")
+
+
+if __name__ == "__main__":
+    run()