4. Source-specific normalizers

from typing import List, Tuple
from api_helpers import safe_get, extract_items_and_meta
from models import Article


def normalize_newsapi(response: dict) -> Tuple[List[Article], dict]:
    """
    Transform NewsAPI response into canonical Article format.

    Returns a tuple of (articles, meta). The meta dict carries per-response
    bookkeeping; the aggregator uses meta["total"] for the "X of Y matching"
    display, while the dataclass holds only per-article fields.

    NewsAPI structure:
      - articles in 'articles' array at root
      - source nested as object with 'name' field
      - timestamp in 'publishedAt' (ISO 8601)
      - optional: author, description, urlToImage, content
    """
    # Step 1: extract items + meta using Chapter 10 helper
    items, meta = extract_items_and_meta(response)

    articles = []

    # Step 2: loop defensively
    for item in items:
        try:
            # Step 3: extract and validate required fields; fail fast on missing data
            title = item.get("title", "").strip()
            url = item.get("url", "").strip()
            published_at = item.get("publishedAt", "").strip()
            if not title or not url or not published_at:
                continue

            # Step 4: map remaining fields (source name, optionals, format conversions)
            source_name = safe_get(item, "source.name", "Unknown Source")

            description = item.get("description")
            if description:
                description = description.strip()

            author = item.get("author")
            if author:
                author = author.strip()

            image_url = item.get("urlToImage")
            if image_url:
                image_url = image_url.strip()

            content = item.get("content")
            content_preview = None
            if content:
                content_preview = content[:200] + "..." if len(content) > 200 else content

            # Step 5: construct the canonical Article
            article = Article(
                title=title,
                url=url,
                published_at=published_at,
                source_name=source_name,
                description=description,
                author=author,
                image_url=image_url,
                content_preview=content_preview,
            )

            articles.append(article)

        except (ValueError, KeyError, TypeError) as e:
            # Log and skip malformed items; don't crash entire batch
            print(f"Warning: skipping malformed NewsAPI item: {e}")
            continue

    return articles, meta

import os
import requests
from dotenv import load_dotenv
from normalize_newsapi import normalize_newsapi

load_dotenv()

NEWSAPI_KEY = os.environ.get("NEWSAPI_KEY")

response = requests.get(
    "https://newsapi.org/v2/everything",
    params={"q": "python", "pageSize": 5, "apiKey": NEWSAPI_KEY},
    timeout=10,
).json()

articles, meta = normalize_newsapi(response)

total = meta.get("total")
suffix = f" of {total} matching" if total else ""
print(f"NewsAPI: normalized {len(articles)}{suffix} articles\n")
for i, article in enumerate(articles[:3], 1):
    print(f"{i}. {article.title}")
    print(f"   Source: {article.source_name}")
    print(f"   Published: {article.format_timestamp()}")
    if article.author:
        print(f"   By: {article.author}")
    print()

$ python test_newsapi_normalizer.py
NewsAPI: normalized 5 of 12847 matching articles

1. Python 3.13 Performance Improvements
   Source: TechCrunch
   Published: January 15, 2025 at 02:30 PM
   By: Sarah Chen

2. Machine Learning with Python: 2025 Guide
   Source: Medium
   Published: January 14, 2025 at 11:45 AM

3. Django 5.0 Released
   Source: Django Project
   Published: January 13, 2025 at 09:00 AM
   By: Django Team

from api_helpers import extract_items_and_meta, safe_get

# Guardian wraps everything in 'response'; bail to (articles, meta) shape on bad input.
if not isinstance(response, dict) or "response" not in response:
    return [], {}

response_obj = response["response"]
items, meta = extract_items_and_meta(response_obj, container_hints=["results"])

# Inside the loop: extract Guardian's field names
title = item.get("webTitle", "").strip()                    # not 'title'
url = item.get("webUrl", "").strip()                        # not 'url'
published_at = item.get("webPublicationDate", "").strip()   # not 'publishedAt'

# Source: use section name as identifier
source_name = item.get("sectionName", "The Guardian")

# Skip items missing required fields; identical pattern to NewsAPI
if not title or not url or not published_at:
    continue

# safe_get drills into the optional 'fields' sub-object in one call.
description = safe_get(item, "fields.trailText")
if description:
    description = description.strip()

author = safe_get(item, "fields.byline")
if author:
    author = author.strip()

image_url = safe_get(item, "fields.thumbnail")
if image_url:
    image_url = image_url.strip()

body = safe_get(item, "fields.bodyText")
content_preview = None
if body:
    content_preview = body[:200] + "..." if len(body) > 200 else body

article = Article(
    title=title,
    url=url,
    published_at=published_at,
    source_name=source_name,
    description=description,
    author=author,
    image_url=image_url,
    content_preview=content_preview,
)

articles.append(article)

from api_helpers import extract_items_and_meta, try_fields

# Standard pattern: HackerNews uses 'hits' as the container
items, meta = extract_items_and_meta(response, container_hints=["hits"])

# Inside the loop: HackerNews uses simple field names
title = item.get("title", "").strip()
url = item.get("url", "").strip()

# HackerNews items without URLs are Ask HN or Show HN posts
# Construct URL from objectID when missing
if not url:
    object_id = item.get("objectID", "")
    if object_id:
        url = f"https://news.ycombinator.com/item?id={object_id}"
    # If still no URL after construction, validation below will skip the item

# Source is constant for HackerNews
source_name = "HackerNews"

from datetime import datetime, timezone

# try_fields returns the first present value; the type tells us which branch.
timestamp = try_fields(item, ["created_at_i", "created_at"], default="")
if isinstance(timestamp, int):
    try:
        dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
        published_at = dt.isoformat().replace("+00:00", "Z")
    except (ValueError, OSError):
        # Out-of-range Unix value; fall through to validation, which skips
        # the item because published_at is empty.
        published_at = ""
else:
    # Already an ISO string (or "" if neither field was present).
    published_at = timestamp

# HackerNews has minimal metadata; most fields are sparse
description = item.get("story_text")
if description:
    description = description.strip()

author = item.get("author")  # Username string
if author:
    author = author.strip()

image_url = None  # No images in HackerNews
content_preview = None

import os
import requests
from dotenv import load_dotenv
from models import Article
from normalize_newsapi import normalize_newsapi
from normalize_guardian import normalize_guardian
from normalize_hackernews import normalize_hackernews

load_dotenv()

NEWSAPI_KEY = os.environ.get("NEWSAPI_KEY")
GUARDIAN_KEY = os.environ.get("GUARDIAN_KEY")


def _summary(name, articles, meta):
    total = meta.get("total")
    suffix = f" of {total} matching" if total else ""
    print(f"{name}: {len(articles)}{suffix} articles normalized")


def test_all_normalizers():
    """Test all three normalizers produce canonical output."""
    print("=== TESTING ALL NORMALIZERS ===\n")

    newsapi_response = requests.get(
        "https://newsapi.org/v2/everything",
        params={"q": "python", "pageSize": 2, "apiKey": NEWSAPI_KEY},
        timeout=10,
    ).json()
    newsapi_articles, newsapi_meta = normalize_newsapi(newsapi_response)
    _summary("NewsAPI", newsapi_articles, newsapi_meta)

    guardian_response = requests.get(
        "https://content.guardianapis.com/search",
        params={"q": "python", "page-size": 2, "show-fields": "all",
                "api-key": GUARDIAN_KEY},
        timeout=10,
    ).json()
    guardian_articles, guardian_meta = normalize_guardian(guardian_response)
    _summary("Guardian", guardian_articles, guardian_meta)

    hn_response = requests.get(
        "https://hn.algolia.com/api/v1/search?query=python&hitsPerPage=2",
        timeout=10,
    ).json()
    hn_articles, hn_meta = normalize_hackernews(hn_response)
    _summary("HackerNews", hn_articles, hn_meta)

    all_articles = newsapi_articles + guardian_articles + hn_articles
    print(f"\nTotal: {len(all_articles)} articles")
    print(f"All are Article instances: {all(isinstance(a, Article) for a in all_articles)}")
    print(
        "All have required fields: "
        f"{all(a.title and a.url and a.published_at and a.source_name for a in all_articles)}"
    )


if __name__ == "__main__":
    test_all_normalizers()

$ python test_all_normalizers.py
=== TESTING ALL NORMALIZERS ===

NewsAPI: 2 of 12847 matching articles normalized
Guardian: 2 of 8453 matching articles normalized
HackerNews: 2 of 583201 matching articles normalized

Total: 6 articles
All are Article instances: True
All have required fields: True