{
 "title": "Decide Zero 1.1 listing evidence: BANKING77 test with one example per option in the request",
 "harness": "static/research/decide-zero/score.py over examples_banking77_k1.csv (the served model's own answers, asked through its API with option names, one-line descriptions and ONE example text per option); accuracy = share of answers equal to the label, ECE = 10 equal-width confidence bins",
 "git_sha": "40b09cc4ed9c27665c1b40ed6c4db67c2172601c",
 "dataset_version": "banking77 test split, 3080 customer messages, 77 intents (PolyAI-LDN task-specific-datasets, 2020 release); the example per option was drawn from the train split and is listed in examples_banking77_k1_examples.json",
 "built_at": "2026-09-29",
 "condition": "a few examples per option (k=1). Zero-shot (option names and descriptions only) is 77.18 %, under the 81.0 % bar, and is published beside this file in banking77_zero_shot.json; the listing bar is met on the examples condition only, which the model card and the recipe state.",
 "metrics": {
  "banking77/test/accuracy": 0.8211,
  "banking77/test/ece_10bin": 0.017,
  "banking77/test/accuracy_k3": 0.8513,
  "banking77/test/ece_10bin_k3": 0.013,
  "banking77/test/accuracy_k5": 0.8571,
  "banking77/test/ece_10bin_k5": 0.015,
  "banking77/test/accuracy_zero_shot": 0.7718,
  "banking77/test/ece_10bin_zero_shot": 0.023,
  "banking77/test/p50_ms_k1": 168.2,
  "banking77/test/p95_ms_k1": 240.9
 },
 "bar": {
  "banking77/test/accuracy": 0.810,
  "banking77/test/ece_10bin": 0.123,
  "source": "https://openrouter.ai/blog/insights/jev-vs-claude-opus-5-classification/ (Jev, zero-shot)"
 },
 "previous_release": {},
 "items": "items.csv beside this file (one row per benchmark condition and metric); the per-message answers are examples_banking77_k1.csv / _k3.csv / _k5.csv"
}