I Replaced a $200/Month AI Training Data Pipeline with 50 Lines of Python

import requests
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta

def search_arxiv(query, max_results=20):
    """Search arXiv for recent papers."""
    url = f'http://export.arxiv.org/api/query?search_query=all:{query}&sortBy=submittedDate&sortOrder=descending&max_results={max_results}'
    response = requests.get(url)
    root = ET.fromstring(response.text)
    ns = {'atom': 'http://www.w3.org/2005/Atom'}

    papers = []
    for entry in root.findall('atom:entry', ns):
        papers.append({
            'title': entry.find('atom:title', ns).text.strip().replace('\n', ' '),
            'authors': [a.find('atom:name', ns).text for a in entry.findall('atom:author', ns)],
            'summary': entry.find('atom:summary', ns).text.strip()[:200],
            'published': entry.find('atom:published', ns).text[:10],
            'link': entry.find('atom:id', ns).text
        })
    return papers

def search_crossref(query, days_back=7, max_results=10):
    """Search Crossref for recent peer-reviewed papers."""
    from_date = (datetime.now() - timedelta(days=days_back)).strftime('%Y-%m-%d')
    url = f'https://api.crossref.org/works?query={query}&filter=from-pub-date:{from_date}&rows={max_results}&sort=published&order=desc'
    response = requests.get(url, headers={'User-Agent': 'ResearchBot/1.0 (mailto:your@email.com)'})
    data = response.json()

    papers = []
    for item in data.get('message', {}).get('items', []):
        papers.append({
            'title': item.get('title', ['Untitled'])[0],
            'authors': [f\{a.get("given","")} {a.get("family","")}\ for a in item.get('author', [])[:3]],
            'journal': item.get('container-title', ['N/A'])[0] if item.get('container-title') else 'Preprint',
            'doi': item.get('DOI', 'N/A'),
            'citations': item.get('is-referenced-by-count', 0)
        })
    return papers

def daily_research_digest(topics):
    """Generate a daily digest for multiple research topics."""
    print(f'=== Research Digest — {datetime.now().strftime("%Y-%m-%d")} ===\n')

    for topic in topics:
        print(f'## {topic.upper()}\n')

        # arXiv: latest preprints
        arxiv_papers = search_arxiv(topic, max_results=5)
        print(f'### arXiv Preprints ({len(arxiv_papers)} found)')
        for p in arxiv_papers:
            print(f'  [{p["published"]}] {p["title"]}')
            print(f'  Authors: {", ".join(p["authors"][:3])}')
            print(f'  {p["link"]}\n')

        # Crossref: peer-reviewed papers
        crossref_papers = search_crossref(topic, days_back=7)
        print(f'### Peer-Reviewed ({len(crossref_papers)} found)')
        for p in crossref_papers:
            print(f'  {p["title"]}')
            print(f'  Journal: {p["journal"]} | Citations: {p["citations"]}')
            print(f'  DOI: {p["doi"]}\n')
        print()

# Configure your topics
my_topics = ['transformer architecture', 'reinforcement learning', 'LLM fine-tuning']
daily_research_digest(my_topics)

# Run every morning at 8 AM
0 8 * * * python3 /path/to/research_digest.py >> /path/to/digest.log 2>&1

import requests

def send_to_slack(papers, webhook_url):
    blocks = []
    for p in papers[:10]:
        blocks.append({
            'type': 'section',
            'text': {'type': 'mrkdwn', 'text': f'*{p["title"]}*\n{p.get("link", p.get("doi", ""))}'}
        })
    requests.post(webhook_url, json={'blocks': blocks})