"""Median duration of completed drug and biological trials on ClinicalTrials.gov.

Reproduces the summary table in /guides/how-long-do-clinical-trials-take.

Cohort: interventional studies with overall status COMPLETED, at least one
DRUG or BIOLOGICAL intervention, a registered phase (Early Phase 1 to Phase 4;
"NA" excluded) and a study completion date between 2021-01-01 and 2025-12-31.
The start date must not be marked ESTIMATED and the completion date must be
ACTUAL. Month-only dates are read as the first day of the month.

Durations are calendar days from study start to primary completion (last
participant's last primary-outcome measurement, ACTUAL dates only) and from
study start to study completion (last participant's last visit). Records with
a completion date before the start date are dropped.

Groups: all trials; lead sponsor class INDUSTRY versus every other class; and
cancer trials, defined as any condition MeSH term or ancestor under
"Neoplasms by Histologic Type" (D009370) or "Neoplasms by Site" (D009371), or a
condition coded directly as "Neoplasms" (D009369).

Usage: python analyze.py [output.csv]   (standard library only)
"""

import csv
import json
import statistics
import sys
import time
import urllib.parse
import urllib.request
from datetime import date

API = "https://clinicaltrials.gov/api/v2"
QUERY = (
    "AREA[StudyType]INTERVENTIONAL AND AREA[OverallStatus]COMPLETED"
    " AND AREA[InterventionType](DRUG OR BIOLOGICAL)"
    " AND AREA[CompletionDate]RANGE[2021-01-01,2025-12-31]"
)
FIELDS = ",".join(
    [
        "NCTId",
        "Phase",
        "LeadSponsorClass",
        "protocolSection.statusModule.startDateStruct",
        "protocolSection.statusModule.primaryCompletionDateStruct",
        "protocolSection.statusModule.completionDateStruct",
        "derivedSection.conditionBrowseModule",
    ]
)
PHASES = {
    ("EARLY_PHASE1",): "Early Phase 1",
    ("PHASE1",): "Phase 1",
    ("PHASE1", "PHASE2"): "Phase 1/2",
    ("PHASE2",): "Phase 2",
    ("PHASE2", "PHASE3"): "Phase 2/3",
    ("PHASE3",): "Phase 3",
    ("PHASE4",): "Phase 4",
}
CANCER_BRANCHES = {"D009370", "D009371"}
CANCER_DIRECT = {"D009369"}


def get(path, params):
    url = f"{API}/{path}?{urllib.parse.urlencode(params)}"
    for attempt in range(5):
        try:
            with urllib.request.urlopen(url, timeout=60) as response:
                return json.load(response)
        except Exception:
            if attempt == 4:
                raise
            time.sleep(2 * (attempt + 1))


def parse_date(struct):
    if not struct or "date" not in struct:
        return None
    parts = [int(p) for p in struct["date"].split("-")]
    while len(parts) < 3:
        parts.append(1)
    return date(*parts)


def is_cancer(browse):
    meshes = {m["id"] for m in browse.get("meshes", [])}
    ancestors = {a["id"] for a in browse.get("ancestors", [])}
    return bool((meshes | ancestors) & CANCER_BRANCHES or meshes & CANCER_DIRECT)


def fetch():
    params = {"query.term": QUERY, "fields": FIELDS, "pageSize": 1000}
    while True:
        page = get("studies", params)
        yield from page["studies"]
        token = page.get("nextPageToken")
        if not token:
            return
        params["pageToken"] = token


def summarize(values):
    if not values:
        return {"n": 0, "q1": "", "median": "", "q3": ""}
    if len(values) == 1:
        return {"n": 1, "q1": values[0], "median": values[0], "q3": values[0]}
    q1, median, q3 = statistics.quantiles(values, n=4, method="inclusive")
    return {"n": len(values), "q1": round(q1), "median": round(median), "q3": round(q3)}


def main():
    out_path = sys.argv[1] if len(sys.argv) > 1 else "summary.csv"
    version = get("version", {})
    counts = {"retrieved": 0, "no_phase": 0, "bad_dates": 0, "included": 0}
    groups = {}

    for study in fetch():
        counts["retrieved"] += 1
        protocol = study["protocolSection"]
        status = protocol["statusModule"]
        phase = PHASES.get(tuple(protocol.get("designModule", {}).get("phases", [])))
        if phase is None:
            counts["no_phase"] += 1
            continue
        start_struct = status.get("startDateStruct")
        end_struct = status.get("completionDateStruct")
        start = parse_date(start_struct)
        end = parse_date(end_struct)
        if (
            start is None
            or end is None
            or start_struct.get("type") == "ESTIMATED"
            or end_struct.get("type") != "ACTUAL"
            or end < start
        ):
            counts["bad_dates"] += 1
            continue
        counts["included"] += 1
        primary_struct = status.get("primaryCompletionDateStruct")
        primary = parse_date(primary_struct)
        primary_days = (
            (primary - start).days
            if primary and primary_struct.get("type") == "ACTUAL" and primary >= start
            else None
        )
        sponsor = protocol.get("sponsorCollaboratorsModule", {}).get("leadSponsor", {})
        labels = [
            "All",
            "Industry" if sponsor.get("class") == "INDUSTRY" else "Non-industry",
            "Cancer"
            if is_cancer(study.get("derivedSection", {}).get("conditionBrowseModule", {}))
            else "Non-cancer",
        ]
        for label in labels:
            bucket = groups.setdefault((phase, label), {"primary": [], "study": []})
            bucket["study"].append((end - start).days)
            if primary_days is not None:
                bucket["primary"].append(primary_days)

    order = list(PHASES.values())
    group_order = ["All", "Industry", "Non-industry", "Cancer", "Non-cancer"]
    with open(out_path, "w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(
            [
                "phase",
                "group",
                "n_study",
                "study_q1_days",
                "study_median_days",
                "study_q3_days",
                "n_primary",
                "primary_q1_days",
                "primary_median_days",
                "primary_q3_days",
            ]
        )
        for phase in order:
            for label in group_order:
                bucket = groups.get((phase, label))
                if not bucket:
                    continue
                s = summarize(sorted(bucket["study"]))
                p = summarize(sorted(bucket["primary"]))
                writer.writerow(
                    [phase, label, s["n"], s["q1"], s["median"], s["q3"], p["n"], p["q1"], p["median"], p["q3"]]
                )

    print(json.dumps({"api": version, "query": QUERY, **counts}, indent=2))


if __name__ == "__main__":
    main()