generate_artifact_availability

src.generators.generate_artifact_availability

Generate artifact availability (liveness) data.

Checks whether artifact URLs are still reachable and writes per-artifact availability results for analysis of decay rates by platform, age, and area.

Writes

assets/data/artifact_availability.json — per-artifact liveness results

Usage

python -m src.generators.generate_artifact_availability --conf_regex '.*20[12][0-9]' --output_dir ../reprodb.github.io

generate_availability(results: dict[str, list[dict]]) -> tuple[list[dict], dict, dict]

Check URL liveness for all artifacts and produce per-artifact records.

Returns a list of dicts, one per artifact, with fields: conference, year, area, title, url_key, url, platform, accessible

Source code in src/generators/generate_artifact_availability.py

def generate_availability(results: dict[str, list[dict]]) -> tuple[list[dict], dict, dict]:
    """Check URL liveness for all artifacts and produce per-artifact records.

    Returns a list of dicts, one per artifact, with fields:
      conference, year, area, title, url_key, url, platform, accessible
    """
    # Run parallel liveness checks
    results, counts, failed = check_artifact_exists(results, URL_KEYS)

    records = []
    for conf_year, artifacts in results.items():
        conf_name, year = _extract_conference_year(conf_year)
        if year is None:
            continue
        area = _conf_area(conf_name)
        for artifact in artifacts:
            title = artifact.get("title", "")
            for url_key in URL_KEYS:
                url = artifact.get(url_key, "")
                if not url or (isinstance(url, list) and not url):
                    continue
                if isinstance(url, list):
                    url = url[0]
                if not isinstance(url, str) or not url.strip():
                    continue
                exists_key = f"{url_key}_exists"
                accessible = artifact.get(exists_key, False)
                platform = _detect_platform(url)
                records.append(
                    {
                        "conference": conf_name,
                        "year": year,
                        "area": area,
                        "title": title,
                        "url_key": url_key,
                        "url": url,
                        "platform": platform,
                        "accessible": accessible,
                    }
                )

    return records, counts, failed

build_summary(records: list[dict]) -> dict[str, Any]

Aggregate availability records into summary statistics.

Source code in src/generators/generate_artifact_availability.py

def build_summary(records: list[dict]) -> dict[str, Any]:
    """Aggregate availability records into summary statistics."""
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")

    # Overall
    total = len(records)
    accessible = sum(1 for r in records if r["accessible"])

    # By platform
    by_platform = defaultdict(lambda: {"total": 0, "accessible": 0})
    for r in records:
        by_platform[r["platform"]]["total"] += 1
        if r["accessible"]:
            by_platform[r["platform"]]["accessible"] += 1

    # By area
    by_area = defaultdict(lambda: {"total": 0, "accessible": 0})
    for r in records:
        by_area[r["area"]]["total"] += 1
        if r["accessible"]:
            by_area[r["area"]]["accessible"] += 1

    # By year
    by_year = defaultdict(lambda: {"total": 0, "accessible": 0})
    for r in records:
        by_year[r["year"]]["total"] += 1
        if r["accessible"]:
            by_year[r["year"]]["accessible"] += 1

    # By year × area
    by_year_area = defaultdict(lambda: defaultdict(lambda: {"total": 0, "accessible": 0}))
    for r in records:
        by_year_area[r["year"]][r["area"]]["total"] += 1
        if r["accessible"]:
            by_year_area[r["year"]][r["area"]]["accessible"] += 1

    # By year × platform
    by_year_platform = defaultdict(lambda: defaultdict(lambda: {"total": 0, "accessible": 0}))
    for r in records:
        by_year_platform[r["year"]][r["platform"]]["total"] += 1
        if r["accessible"]:
            by_year_platform[r["year"]][r["platform"]]["accessible"] += 1

    # By conference
    by_conf = defaultdict(lambda: {"total": 0, "accessible": 0})
    for r in records:
        by_conf[r["conference"]]["total"] += 1
        if r["accessible"]:
            by_conf[r["conference"]]["accessible"] += 1

    def _pct(d):
        return round(100 * d["accessible"] / d["total"], 1) if d["total"] > 0 else 0

    summary = {
        "checked_at": now,
        "total_urls": total,
        "accessible_urls": accessible,
        "accessibility_pct": round(100 * accessible / total, 1) if total > 0 else 0,
        "by_platform": {k: {**v, "pct": _pct(v)} for k, v in sorted(by_platform.items())},
        "by_area": {k: {**v, "pct": _pct(v)} for k, v in sorted(by_area.items())},
        "by_year": {str(k): {**v, "pct": _pct(v)} for k, v in sorted(by_year.items())},
        "by_year_area": {
            str(y): {a: {**d, "pct": _pct(d)} for a, d in sorted(data.items())}
            for y, data in sorted(by_year_area.items())
        },
        "by_year_platform": {
            str(y): {p: {**d, "pct": _pct(d)} for p, d in sorted(data.items())}
            for y, data in sorted(by_year_platform.items())
        },
        "by_conference": {k: {**v, "pct": _pct(v)} for k, v in sorted(by_conf.items())},
    }
    return summary