#!/usr/bin/env python3
"""Check the Tuatara Decide vs Jev comparison on BANKING77 yourself.

Downloads four public files, lines them up row by row, and prints each system's
accuracy and the exact McNemar test against Jev. Python 3.8+, standard library only.

    python3 score.py

Sources:
  official test labels    https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/master/banking_data/test.csv
  Jev's answers           https://raw.githubusercontent.com/simonmesmith/jev-banking77-experiment/main/results/predictions.csv
  blend's answers         https://cymetica.com/static/research/jev-banking77/blend_predictions.csv
  embedding head answers  https://cymetica.com/static/research/jev-banking77/head_predictions.csv
"""
import csv
import io
import math
import urllib.request

OFFICIAL = "https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/master/banking_data/test.csv"
JEV = "https://raw.githubusercontent.com/simonmesmith/jev-banking77-experiment/main/results/predictions.csv"
BLEND = "https://cymetica.com/static/research/jev-banking77/blend_predictions.csv"
HEAD = "https://cymetica.com/static/research/jev-banking77/head_predictions.csv"


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": "banking77-score/1.0"})
    with urllib.request.urlopen(req, timeout=60) as r:
        return list(csv.DictReader(io.StringIO(r.read().decode("utf-8"))))


def mcnemar_exact(b, c):
    """Two-sided exact binomial test on the discordant pairs."""
    n, k = b + c, min(b, c)
    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
    return min(1.0, 2 * tail)


official, jev = fetch(OFFICIAL), fetch(JEV)
truth = [r["category"] for r in official]
assert len(truth) == len(jev) == 3080, "expected 3,080 test rows in each file"
jev_ok = [j["prediction"] == t for j, t in zip(jev, truth)]
n = len(truth)
print(f"test messages                    {n:,}")
print(f"Jev correct                      {sum(jev_ok):,}  ({100 * sum(jev_ok) / n:.2f}%)")

for name, url in (("Tuatara blend", BLEND), ("embedding head", HEAD)):
    ours = fetch(url)
    assert len(ours) == n, f"{name}: expected 3,080 rows"
    for i, (t, j, b, o) in enumerate(zip(truth, jev, ours, official)):
        assert j["id"] == b["id"] == f"test-{i:05d}", f"{name} row {i}: ids out of order"
        assert j["truth"] == t and b["truth"] == t, f"{name} row {i}: label mismatch with the official test set"
        assert b["text"] == o["text"], f"{name} row {i}: text mismatch with the official test set"
    ok = [b["prediction"] == t for b, t in zip(ours, truth)]
    only_ours = sum(b and not j for b, j in zip(ok, jev_ok))
    only_jev = sum(j and not b for b, j in zip(ok, jev_ok))
    print()
    print(f"{name + ' correct':<32} {sum(ok):,}  ({100 * sum(ok) / n:.2f}%)")
    print(f"{name + ' right, Jev wrong':<32} {only_ours}")
    print(f"{'Jev right, ' + name + ' wrong':<32} {only_jev}")
    print(f"{'exact McNemar p':<32} {mcnemar_exact(only_ours, only_jev):.2g}")