78 lines
2.6 KiB
Python
78 lines
2.6 KiB
Python
"""Fetch AI/quant research RSS: ArXiv cs.AI + quant blogs."""
|
|
import json, os, feedparser
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
OUTPUT = "/var/www/nav/data/research.json"
|
|
tz = timezone(timedelta(hours=8))
|
|
|
|
FEEDS = [
|
|
# ArXiv AI (via arxiv-rss.appspot.com or direct)
|
|
{"name": "ArXiv cs.AI", "url": "https://rss.arxiv.org/rss/cs.AI", "lang": "en"},
|
|
{"name": "ArXiv q-fin", "url": "https://rss.arxiv.org/rss/q-fin", "lang": "en"},
|
|
# Quant blogs
|
|
{"name": "Quantitative Finance Stack Exchange", "url": "https://quant.stackexchange.com/feeds", "lang": "en"},
|
|
{"name": "Alpha Vantage Blog", "url": "https://blog.alphavantage.co/rss.xml", "lang": "en"},
|
|
# 机器之心（中文 AI 资讯）
|
|
{"name": "机器之心", "url": "https://www.jiqizhixin.com/rss", "lang": "zh"},
|
|
# Towards Data Science
|
|
{"name": "Towards Data Science", "url": "https://towardsdatascience.com/feed", "lang": "en"},
|
|
# AI Alignment Forum
|
|
{"name": "AI Alignment Forum", "url": "https://www.alignmentforum.org/feed.xml", "lang": "en"},
|
|
]
|
|
|
|
MAX_PER_FEED = 20
|
|
MAX_TOTAL = 150
|
|
|
|
|
|
def fetch_feed(feed_info):
|
|
try:
|
|
feed = feedparser.parse(feed_info["url"])
|
|
items = []
|
|
for entry in feed.entries[:MAX_PER_FEED]:
|
|
items.append({
|
|
"title": entry.get("title", ""),
|
|
"link": entry.get("link", ""),
|
|
"summary": entry.get("summary", "")[:400],
|
|
"published": entry.get("published", ""),
|
|
"source": feed_info["name"],
|
|
"lang": feed_info["lang"],
|
|
})
|
|
print(f" {feed_info['name']}: {len(items)} items")
|
|
return items
|
|
except Exception as e:
|
|
print(f" {feed_info['name']}: ERROR - {e}")
|
|
return []
|
|
|
|
|
|
def fetch_research():
|
|
items = []
|
|
for feed_info in FEEDS:
|
|
items.extend(fetch_feed(feed_info))
|
|
|
|
seen = set()
|
|
unique = []
|
|
for item in items:
|
|
key = item["title"][:30]
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique.append(item)
|
|
|
|
result = {
|
|
"updated_at": datetime.now(tz).strftime("%Y-%m-%d %H:%M"),
|
|
"total": len(unique),
|
|
"items": unique[:MAX_TOTAL],
|
|
"sources": [f["name"] for f in FEEDS],
|
|
}
|
|
|
|
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
|
with open(OUTPUT, "w", encoding="utf-8") as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"\nTotal: {len(unique)} unique research items -> {OUTPUT}")
|
|
return result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print(f"Fetching research RSS at {datetime.now(tz).strftime('%Y-%m-%d %H:%M')}")
|
|
fetch_research()
|