diff --git a/scripts/deep_dive.py b/scripts/deep_dive.py new file mode 100644 index 0000000..9b0b662 --- /dev/null +++ b/scripts/deep_dive.py @@ -0,0 +1,105 @@ +"""Deep dive: pick top news and generate in-depth analysis via DeepSeek.""" +import json, os, requests +from datetime import datetime, timezone, timedelta + +API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" +API_URL = "https://api.deepseek.com/v1/chat/completions" +MODEL = "deepseek-v4-flash" + +NEWS_SUMMARY = "/var/www/nav/data/news_summary.json" +OUTPUT = "/var/www/nav/data/deep_dive.json" +tz = timezone(timedelta(hours=8)) + + +def load_top_news(): + if not os.path.exists(NEWS_SUMMARY): + return None + with open(NEWS_SUMMARY) as f: + data = json.load(f) + # Pick the #1 item from the first category (usually 国际要闻 or most important) + for cat in data.get("categories", []): + if cat.get("items"): + return cat["items"][0], cat["name"] + return None, None + + +def generate_deep_dive(top_item, category): + title = top_item.get("title", "") + subtitle = top_item.get("subtitle", "") + summary = top_item.get("summary", "") + source = top_item.get("source", "") + link = top_item.get("link", "") + + prompt = f"""你是一位资深的国际事务/财经分析员。用户看到了一条新闻的简要摘要,请你写一篇深度解读,帮助用户理解这条新闻的**完整背景、深层含义和未来走向**。 + +## 新闻信息 +- 分类:{category} +- 标题:{subtitle} +- 原文标题:{title} +- 来源:{source} +- 原文链接:{link} +- 简要摘要:{summary} + +## 写作要求 +1. **800-1200 字**,中文,客观专业 +2. **结构清晰**:事件背景 → 关键事实 → 各方立场 → 深层分析 → 可能走向 → 对中国/市场的影响 +3. **不重复摘要内容**,要在摘要基础上做增量分析 +4. **引用具体数据/事件**支撑论点,不要空泛议论 +5. **结尾一句话总结**这条新闻对用户(中国中层干部、量化交易者)的核心启示 +6. 关键信息用 **加粗** 标出 + +请直接输出正文,不要输出 JSON。""" + + resp = requests.post( + API_URL, + headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, + json={ + "model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.5, + "max_tokens": 4000, + }, + timeout=120, + ) + + if resp.status_code != 200: + print(f"API error: {resp.status_code}") + return None + + content = resp.json()["choices"][0]["message"]["content"] + return content.strip() + + +def run(): + top_item, category = load_top_news() + if not top_item: + print("No news data available for deep dive") + return + + print(f"Generating deep dive for: {top_item.get('subtitle', '')[:50]}...") + + analysis = generate_deep_dive(top_item, category) + if not analysis: + print("Deep dive generation failed") + return + + result = { + "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"), + "source_news": { + "subtitle": top_item.get("subtitle", ""), + "title": top_item.get("title", ""), + "link": top_item.get("link", ""), + "category": category, + }, + "analysis": analysis, + } + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"Deep dive written -> {OUTPUT}") + + +if __name__ == "__main__": + run() diff --git a/scripts/fetch_market.py b/scripts/fetch_market.py new file mode 100644 index 0000000..1fb3ccb --- /dev/null +++ b/scripts/fetch_market.py @@ -0,0 +1,87 @@ +"""Fetch market data: major indices, crypto, forex via public APIs (no API key needed).""" +import json, os, requests +from datetime import datetime, timezone, timedelta + +OUTPUT = "/var/www/nav/data/market.json" +tz = timezone(timedelta(hours=8)) + +# Public endpoints (no API key) +COINGECKO_URL = "https://api.coingecko.com/api/v3/simple/price?ids=bitcoin,ethereum,solana,binancecoin,ripple&vs_currencies=usd&include_24hr_change=true" +# A-share indices via eastmoney (free, no key) +EM_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get?fltt=2&secids=1.000001,0.399001,0.399006&fields=f2,f3,f4,f12,f14" + + +def fetch_crypto(): + try: + resp = requests.get(COINGECKO_URL, timeout=10) + data = resp.json() + items = [] + icons = {"bitcoin": "₿", "ethereum": "Ξ", "solana": "◎", "binancecoin": "BNB", "ripple": "XRP"} + names = {"bitcoin": "BTC", "ethereum": "ETH", "solana": "SOL", "binancecoin": "BNB", "ripple": "XRP"} + for coin_id, icon in icons.items(): + if coin_id not in data: + continue + price = data[coin_id]["usd"] + change = data[coin_id].get("usd_24h_change", 0) or 0 + direction = "▲" if change >= 0 else "▼" + items.append({ + "type": "crypto", + "symbol": names[coin_id], + "icon": icon, + "price": f"${price:,.2f}", + "change_pct": f"{direction} {abs(change):.2f}%", + "is_up": change >= 0, + }) + return items + except Exception as e: + print(f" Crypto error: {e}") + return [] + + +def fetch_cn_indices(): + """Fetch A-share indices via Eastmoney API (no key).""" + try: + resp = requests.get(EM_URL, timeout=10) + data = resp.json() + items = [] + for diff in data.get("data", {}).get("diff", []): + name = diff.get("f14", "") + price = diff.get("f2", 0) + change = diff.get("f4", 0) + pct = diff.get("f3", 0) + direction = "▲" if change >= 0 else "▼" + items.append({ + "type": "cn_index", + "symbol": name, + "price": f"{price:.2f}", + "change": f"{direction} {abs(change):.2f}", + "change_pct": f"{direction} {abs(pct):.2f}%", + "is_up": change >= 0, + }) + return items + except Exception as e: + print(f" CN indices error: {e}") + return [] + + +def fetch_market(): + crypto = fetch_crypto() + cn_indices = fetch_cn_indices() + + result = { + "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"), + "crypto": crypto, + "cn_indices": cn_indices, + } + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"Crypto: {len(crypto)} | CN indices: {len(cn_indices)} -> {OUTPUT}") + return result + + +if __name__ == "__main__": + print(f"Fetching market data at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}") + fetch_market() diff --git a/scripts/fetch_policy.py b/scripts/fetch_policy.py new file mode 100644 index 0000000..ca1af6a --- /dev/null +++ b/scripts/fetch_policy.py @@ -0,0 +1,80 @@ +"""Fetch policy news from Chinese official sources RSS.""" +import json, os, feedparser +from datetime import datetime, timezone, timedelta + +OUTPUT = "/var/www/nav/data/policy.json" +tz = timezone(timedelta(hours=8)) + +FEEDS = [ + # 新华社 + {"name": "新华网·时政", "url": "http://www.xinhuanet.com/politics/news_politics.xml", "lang": "zh"}, + {"name": "新华网·财经", "url": "http://www.xinhuanet.com/fortune/news_fortune.xml", "lang": "zh"}, + {"name": "新华网·国际", "url": "http://www.xinhuanet.com/world/news_world.xml", "lang": "zh"}, + # 人民网 + {"name": "人民网·时政", "url": "http://www.people.com.cn/rss/politics.xml", "lang": "zh"}, + # 中国政府网(国务院) + {"name": "中国政府网", "url": "http://www.gov.cn/xinwen/yaowen.htm", "lang": "zh"}, + # 发改委(尝试 RSS,若无则用新华网发改委专题) + {"name": "新华网·财经政策", "url": "https://www.xinhuanet.com/fortune/policy.xml", "lang": "zh"}, + # 财新(需确认 RSS 可用性) + {"name": "财新网", "url": "https://rss.caixin.com/conf/rss/valsart.xml", "lang": "zh"}, + # 观察者网(政策解读强) + {"name": "观察者网", "url": "https://www.guancha.cn/index.rss", "lang": "zh"}, +] + +MAX_PER_FEED = 30 +MAX_TOTAL = 200 + + +def fetch_feed(feed_info): + try: + feed = feedparser.parse(feed_info["url"]) + items = [] + for entry in feed.entries[:MAX_PER_FEED]: + items.append({ + "title": entry.get("title", ""), + "link": entry.get("link", ""), + "summary": entry.get("summary", "")[:300], + "published": entry.get("published", ""), + "source": feed_info["name"], + "lang": feed_info["lang"], + }) + print(f" {feed_info['name']}: {len(items)} items") + return items + except Exception as e: + print(f" {feed_info['name']}: ERROR - {e}") + return [] + + +def fetch_policy(): + items = [] + for feed_info in FEEDS: + items.extend(fetch_feed(feed_info)) + + # Deduplicate by title + seen = set() + unique = [] + for item in items: + key = item["title"][:30] + if key not in seen: + seen.add(key) + unique.append(item) + + result = { + "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"), + "total": len(unique), + "items": unique[:MAX_TOTAL], + "sources": [f["name"] for f in FEEDS], + } + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"\nTotal: {len(unique)} unique items -> {OUTPUT}") + return result + + +if __name__ == "__main__": + print(f"Fetching policy news at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}") + fetch_policy() diff --git a/scripts/fetch_research.py b/scripts/fetch_research.py new file mode 100644 index 0000000..01cfd01 --- /dev/null +++ b/scripts/fetch_research.py @@ -0,0 +1,77 @@ +"""Fetch AI/quant research RSS: ArXiv cs.AI + quant blogs.""" +import json, os, feedparser +from datetime import datetime, timezone, timedelta + +OUTPUT = "/var/www/nav/data/research.json" +tz = timezone(timedelta(hours=8)) + +FEEDS = [ + # ArXiv AI (via arxiv-rss.appspot.com or direct) + {"name": "ArXiv cs.AI", "url": "https://rss.arxiv.org/rss/cs.AI", "lang": "en"}, + {"name": "ArXiv q-fin", "url": "https://rss.arxiv.org/rss/q-fin", "lang": "en"}, + # Quant blogs + {"name": "Quantitative Finance Stack Exchange", "url": "https://quant.stackexchange.com/feeds", "lang": "en"}, + {"name": "Alpha Vantage Blog", "url": "https://blog.alphavantage.co/rss.xml", "lang": "en"}, + # 机器之心(中文 AI 资讯) + {"name": "机器之心", "url": "https://www.jiqizhixin.com/rss", "lang": "zh"}, + # Towards Data Science + {"name": "Towards Data Science", "url": "https://towardsdatascience.com/feed", "lang": "en"}, + # AI Alignment Forum + {"name": "AI Alignment Forum", "url": "https://www.alignmentforum.org/feed.xml", "lang": "en"}, +] + +MAX_PER_FEED = 20 +MAX_TOTAL = 150 + + +def fetch_feed(feed_info): + try: + feed = feedparser.parse(feed_info["url"]) + items = [] + for entry in feed.entries[:MAX_PER_FEED]: + items.append({ + "title": entry.get("title", ""), + "link": entry.get("link", ""), + "summary": entry.get("summary", "")[:400], + "published": entry.get("published", ""), + "source": feed_info["name"], + "lang": feed_info["lang"], + }) + print(f" {feed_info['name']}: {len(items)} items") + return items + except Exception as e: + print(f" {feed_info['name']}: ERROR - {e}") + return [] + + +def fetch_research(): + items = [] + for feed_info in FEEDS: + items.extend(fetch_feed(feed_info)) + + seen = set() + unique = [] + for item in items: + key = item["title"][:30] + if key not in seen: + seen.add(key) + unique.append(item) + + result = { + "updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"), + "total": len(unique), + "items": unique[:MAX_TOTAL], + "sources": [f["name"] for f in FEEDS], + } + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"\nTotal: {len(unique)} unique research items -> {OUTPUT}") + return result + + +if __name__ == "__main__": + print(f"Fetching research RSS at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}") + fetch_research() diff --git a/scripts/summarize_policy.py b/scripts/summarize_policy.py new file mode 100644 index 0000000..77da4d0 --- /dev/null +++ b/scripts/summarize_policy.py @@ -0,0 +1,121 @@ +"""Use DeepSeek to summarize policy news into categorized Chinese briefs.""" +import json, os, requests, re +from datetime import datetime, timezone, timedelta + +API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" +API_URL = "https://api.deepseek.com/v1/chat/completions" +MODEL = "deepseek-v4-flash" + +INPUT = "/var/www/nav/data/policy.json" +OUTPUT = "/var/www/nav/data/policy_summary.json" +tz = timezone(timedelta(hours=8)) + + +def load_raw(): + if not os.path.exists(INPUT): + return [] + with open(INPUT) as f: + data = json.load(f) + return data.get("items", []) + + +def summarize(items): + news_text = "" + for i, item in enumerate(items[:100], 1): + title = item.get("title", "") + summary = item.get("summary", "")[:200].replace("<", "").replace(">", "") + source = item.get("source", "") + link = item.get("link", "") + news_text += f"{i}. [{source}] {title}\n {summary}\n url: {link}\n\n" + + prompt = f"""你是一个政策分析助手。请从原始政策新闻中精选出**最重要的政策动态**,按优先级排序。 + +## 选稿标准 +- 国务院/发改委/部委最新政策文件 +- 习近平总书记重要讲话/活动 +- 重大经济/产业政策(数字经济、AI、能源、金融) +- 重大改革举措 +- 重要人事任免 + +## 输出 +- 分类名:**政策速递**(只有一个分类) +- 每条:**15-30字完整标题**(subtitle),**300-500字政策解读摘要**(summary) +- 总计 **10-20 条**,按重要性排序 + +## 输出格式 +每条输出到 items 数组,包含:rank, subtitle, title, source, link, summary +summary 用自然段落,关键信息用 **加粗**。 + +## 原始新闻 +{news_text} + +JSON格式: {{"categories": [{{"name":"政策速递","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}""" + + resp = requests.post( + API_URL, + headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, + json={ + "model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, + "max_tokens": 16000, + }, + timeout=180, + ) + + if resp.status_code != 200: + print(f"API error: {resp.status_code}") + return None + + content = resp.json()["choices"][0]["message"]["content"].strip() + if content.startswith("```"): + lines = content.split("\n") + content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:]) + + try: + result = json.loads(content) + except: + import re as re_ + content = re_.sub(r',\s*}', '}', content) + content = re_.sub(r',\s*]', ']', content) + try: + result = json.loads(content) + except: + print(f"JSON parse failed, len={len(content)}") + print(f"First 500: {content[:500]}") + return None + + return result + + +def run(): + now = datetime.now(tz).strftime("%Y-%m-%d %H:%M") + items = load_raw() + if not items: + print("No policy data") + exit(0) + + print(f"Loaded {len(items)} policy items. AI summarizing...") + data = summarize(items) + + if not data: + print("Summarization failed") + exit(1) + + result = { + "updated_at": now, + "categories": data.get("categories", []), + } + + total = sum(len(c["items"]) for c in result["categories"]) + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"Written {total} policy briefs -> {OUTPUT}") + for c in result["categories"]: + print(f" {c['name']}: {len(c['items'])} items") + + +if __name__ == "__main__": + run() diff --git a/scripts/summarize_research.py b/scripts/summarize_research.py new file mode 100644 index 0000000..44fadc0 --- /dev/null +++ b/scripts/summarize_research.py @@ -0,0 +1,117 @@ +"""Use DeepSeek to summarize AI/quant research RSS into categorized briefs.""" +import json, os, requests, re +from datetime import datetime, timezone, timedelta + +API_KEY = "sk-bbca4a0380d549389f0d27cdea0b5228" +API_URL = "https://api.deepseek.com/v1/chat/completions" +MODEL = "deepseek-v4-flash" + +INPUT = "/var/www/nav/data/research.json" +OUTPUT = "/var/www/nav/data/research_summary.json" +tz = timezone(timedelta(hours=8)) + + +def load_raw(): + if not os.path.exists(INPUT): + return [] + with open(INPUT) as f: + data = json.load(f) + return data.get("items", []) + + +def summarize(items): + news_text = "" + for i, item in enumerate(items[:100], 1): + title = item.get("title", "") + summary = item.get("summary", "")[:300].replace("<", "").replace(">", "") + source = item.get("source", "") + link = item.get("link", "") + news_text += f"{i}. [{source}] {title}\n {summary}\n url: {link}\n\n" + + prompt = f"""你是一个 AI/量化研究资讯编辑。请从原始 RSS 中精选出**最有价值的论文和博客文章**,按主题分类。 + +## 选稿标准 +- ArXiv 论文:AI 重大突破(LLM、Agent、多模态)、量化交易新方法 +- 博客文章:有实操价值的策略思路、代码实现、回测分析 +- 过滤:纯新闻通告、产品广告、重复性内容 + +## 分类(每个分类 8-15 条) +- AI 前沿论文(ArXiv cs.AI / cs.LG 重要论文) +- 量化交易研究(策略思路、回测方法、风险管理) +- 行业动态(AI 公司、产品发布、监管动态) + +## 输出格式 +每条:rank, subtitle(15-30字), title, source, link, summary(200-400字,说清楚这篇讲了什么、有什么用) + +## 原始内容 +{news_text} + +JSON格式: {{"categories": [{{"name":"分类名","items":[{{"rank":1,"subtitle":"...","title":"...","source":"...","link":"...","summary":"..."}}]}}]}}""" + + resp = requests.post( + API_URL, + headers={"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}, + json={ + "model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.3, + "max_tokens": 16000, + }, + timeout=180, + ) + + if resp.status_code != 200: + print(f"API error: {resp.status_code}") + return None + + content = resp.json()["choices"][0]["message"]["content"].strip() + if content.startswith("```"): + lines = content.split("\n") + content = "\n".join(lines[1:-1]) if lines[-1].strip() == "```" else "\n".join(lines[1:]) + + try: + result = json.loads(content) + except: + import re as re_ + content = re_.sub(r',\s*}', '}', content) + content = re_.sub(r',\s*]', ']', content) + try: + result = json.loads(content) + except: + print(f"JSON parse failed, len={len(content)}") + return None + + return result + + +def run(): + now = datetime.now(tz).strftime("%Y-%m-%d %H:%M") + items = load_raw() + if not items: + print("No research data") + exit(0) + + print(f"Loaded {len(items)} research items. AI summarizing...") + data = summarize(items) + + if not data: + print("Summarization failed") + exit(1) + + result = { + "updated_at": now, + "categories": data.get("categories", []), + } + + total = sum(len(c["items"]) for c in result["categories"]) + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + print(f"Written {total} research briefs -> {OUTPUT}") + for c in result["categories"]: + print(f" {c['name']}: {len(c['items'])} items") + + +if __name__ == "__main__": + run()