{
 "generated": "2026-11-03",
 "engine": "src/services/tuatarasum (Python port of the Tcl TuataraSum GS1 + GS2, no outside model)",
 "metric": "ROUGE F1 with Porter stemming (rouge-score); rougeLsum over newline-separated sentences; tokens = whitespace tokens of the extract",
 "protocol": "settings frozen on a seeded sample of the validation split, then one run on a seeded sample of the test split; seed 20261028; baselines run in the same harness (scripts/tuatarasum_bench.py)",
 "parity": {
  "tcl_configurations_ok": 70,
  "tcl_configurations_total": 70,
  "golden_fixtures": "tests/fixtures/tuatarasum (9 documents x 5 configurations), tests/test_tuatarasum_parity.py"
 },
 "datasets": [
  {
   "slug": "cnn_dailymail",
   "name": "CNN/DailyMail 3.0.0 test, 3 sentences",
   "n": 1000,
   "k": 3,
   "seed": 20261028,
   "file": "https://cymetica.com/static/research/tuatara-summarize/cnn_test_seeded.json",
   "rows": [
    {
     "system": "lead",
     "name": "Lead-k",
     "rouge1": 40.04,
     "rouge2": 17.67,
     "rougeL": 25.18,
     "rougeLsum": 36.47,
     "tokens": 77.5,
     "ms_per_doc": 47.7,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:tuned-best",
     "name": "TuataraSum tuned-best",
     "rouge1": 39.46,
     "rouge2": 17.34,
     "rougeL": 24.97,
     "rougeLsum": 35.88,
     "tokens": 81.7,
     "ms_per_doc": 798.4,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:tuned",
     "name": "TuataraSum tuned",
     "rouge1": 38.56,
     "rouge2": 16.63,
     "rougeL": 24.9,
     "rougeLsum": 35.03,
     "tokens": 78.4,
     "ms_per_doc": 254.6,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:faithful",
     "name": "TuataraSum faithful (2015 CGI)",
     "rouge1": 37.02,
     "rouge2": 15.09,
     "rougeL": 23.06,
     "rougeLsum": 31.13,
     "tokens": 78.2,
     "ms_per_doc": 468.7,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:gs1",
     "name": "TuataraSum GS1",
     "rouge1": 36.01,
     "rouge2": 15.04,
     "rougeL": 23.11,
     "rougeLsum": 32.64,
     "tokens": 96.4,
     "ms_per_doc": 201.8,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "lexrank",
     "name": "LexRank (sumy)",
     "rouge1": 34.1,
     "rouge2": 12.09,
     "rougeL": 20.99,
     "rougeLsum": 30.44,
     "tokens": 85.1,
     "ms_per_doc": 378.7,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:best",
     "name": "TuataraSum fusion",
     "rouge1": 33.74,
     "rouge2": 13.07,
     "rougeL": 21.15,
     "rougeLsum": 30.36,
     "tokens": 103.8,
     "ms_per_doc": 858.8,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "luhn",
     "name": "Luhn (sumy)",
     "rouge1": 32.68,
     "rouge2": 12.35,
     "rougeL": 20.6,
     "rougeLsum": 29.24,
     "tokens": 102.6,
     "ms_per_doc": 321.3,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:gs2",
     "name": "TuataraSum GS2",
     "rouge1": 30.66,
     "rouge2": 10.53,
     "rougeL": 18.88,
     "rougeLsum": 27.51,
     "tokens": 104.1,
     "ms_per_doc": 3072.5,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "lsa",
     "name": "LSA (sumy)",
     "rouge1": 30.49,
     "rouge2": 9.43,
     "rougeL": 18.64,
     "rougeLsum": 27.16,
     "tokens": 68.3,
     "ms_per_doc": 1195.9,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "textrank",
     "name": "TextRank (sumy)",
     "rouge1": 29.77,
     "rouge2": 10.01,
     "rougeL": 18.55,
     "rougeLsum": 26.5,
     "tokens": 106.7,
     "ms_per_doc": 461.0,
     "n": 1000,
     "errors": 0
    }
   ]
  },
  {
   "slug": "xsum",
   "name": "XSum test, 1 sentence",
   "n": 1000,
   "k": 1,
   "seed": 20261028,
   "file": "https://cymetica.com/static/research/tuatara-summarize/xsum_test_seeded.json",
   "rows": [
    {
     "system": "tuatara:gs1",
     "name": "TuataraSum GS1",
     "rouge1": 18.93,
     "rouge2": 2.65,
     "rougeL": 13.1,
     "rougeLsum": 13.36,
     "tokens": 31.5,
     "ms_per_doc": 113.7,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:best",
     "name": "TuataraSum fusion",
     "rouge1": 18.72,
     "rouge2": 2.65,
     "rougeL": 12.95,
     "rougeLsum": 13.34,
     "tokens": 34.2,
     "ms_per_doc": 369.8,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:faithful",
     "name": "TuataraSum faithful (2015 CGI)",
     "rouge1": 18.55,
     "rouge2": 2.39,
     "rougeL": 12.9,
     "rougeLsum": 12.9,
     "tokens": 25.3,
     "ms_per_doc": 239.3,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "lexrank",
     "name": "LexRank (sumy)",
     "rouge1": 18.48,
     "rouge2": 2.52,
     "rougeL": 13.13,
     "rougeLsum": 13.13,
     "tokens": 26.0,
     "ms_per_doc": 285.6,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:tuned",
     "name": "TuataraSum tuned",
     "rouge1": 18.39,
     "rouge2": 2.3,
     "rougeL": 13.21,
     "rougeLsum": 13.27,
     "tokens": 23.1,
     "ms_per_doc": 163.8,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "luhn",
     "name": "Luhn (sumy)",
     "rouge1": 18.35,
     "rouge2": 2.46,
     "rougeL": 13.0,
     "rougeLsum": 13.0,
     "tokens": 32.8,
     "ms_per_doc": 238.6,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "textrank",
     "name": "TextRank (sumy)",
     "rouge1": 18.29,
     "rouge2": 2.47,
     "rougeL": 13.05,
     "rougeLsum": 13.05,
     "tokens": 34.3,
     "ms_per_doc": 314.6,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:gs2",
     "name": "TuataraSum GS2",
     "rouge1": 17.91,
     "rouge2": 2.34,
     "rougeL": 12.49,
     "rougeLsum": 12.87,
     "tokens": 36.0,
     "ms_per_doc": 1391.8,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "tuatara:tuned-best",
     "name": "TuataraSum tuned-best",
     "rouge1": 17.42,
     "rouge2": 1.79,
     "rougeL": 12.55,
     "rougeLsum": 12.56,
     "tokens": 22.1,
     "ms_per_doc": 330.5,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "lead",
     "name": "Lead-k",
     "rouge1": 16.55,
     "rouge2": 1.46,
     "rougeL": 12.11,
     "rougeLsum": 12.13,
     "tokens": 21.5,
     "ms_per_doc": 20.3,
     "n": 1000,
     "errors": 0
    },
    {
     "system": "lsa",
     "name": "LSA (sumy)",
     "rouge1": 16.02,
     "rouge2": 1.67,
     "rougeL": 11.31,
     "rougeLsum": 11.31,
     "tokens": 21.9,
     "ms_per_doc": 612.8,
     "n": 1000,
     "errors": 0
    }
   ]
  }
 ],
 "fidelity_curve": [
  {
   "mode": "best",
   "ratio": 0.1,
   "tokens_retained": 0.152,
   "rouge1_recall": 47.58,
   "rouge2_recall": 18.6,
   "n": 300
  },
  {
   "mode": "best",
   "ratio": 0.2,
   "tokens_retained": 0.281,
   "rouge1_recall": 63.29,
   "rouge2_recall": 27.18,
   "n": 300
  },
  {
   "mode": "best",
   "ratio": 0.3,
   "tokens_retained": 0.401,
   "rouge1_recall": 71.57,
   "rouge2_recall": 33.27,
   "n": 300
  },
  {
   "mode": "best",
   "ratio": 0.4,
   "tokens_retained": 0.511,
   "rouge1_recall": 77.41,
   "rouge2_recall": 38.3,
   "n": 300
  },
  {
   "mode": "best",
   "ratio": 0.5,
   "tokens_retained": 0.613,
   "rouge1_recall": 81.51,
   "rouge2_recall": 42.24,
   "n": 300
  },
  {
   "mode": "gs1",
   "ratio": 0.1,
   "tokens_retained": 0.143,
   "rouge1_recall": 47.75,
   "rouge2_recall": 18.97,
   "n": 300
  },
  {
   "mode": "gs1",
   "ratio": 0.2,
   "tokens_retained": 0.268,
   "rouge1_recall": 63.84,
   "rouge2_recall": 28.5,
   "n": 300
  },
  {
   "mode": "gs1",
   "ratio": 0.3,
   "tokens_retained": 0.384,
   "rouge1_recall": 72.11,
   "rouge2_recall": 34.65,
   "n": 300
  },
  {
   "mode": "gs1",
   "ratio": 0.4,
   "tokens_retained": 0.494,
   "rouge1_recall": 77.89,
   "rouge2_recall": 39.64,
   "n": 300
  },
  {
   "mode": "gs1",
   "ratio": 0.5,
   "tokens_retained": 0.597,
   "rouge1_recall": 81.49,
   "rouge2_recall": 42.64,
   "n": 300
  },
  {
   "mode": "gs2",
   "ratio": 0.1,
   "tokens_retained": 0.153,
   "rouge1_recall": 42.8,
   "rouge2_recall": 14.2,
   "n": 300
  },
  {
   "mode": "gs2",
   "ratio": 0.2,
   "tokens_retained": 0.28,
   "rouge1_recall": 58.69,
   "rouge2_recall": 22.77,
   "n": 300
  },
  {
   "mode": "gs2",
   "ratio": 0.3,
   "tokens_retained": 0.399,
   "rouge1_recall": 68.75,
   "rouge2_recall": 30.35,
   "n": 300
  },
  {
   "mode": "gs2",
   "ratio": 0.4,
   "tokens_retained": 0.509,
   "rouge1_recall": 75.45,
   "rouge2_recall": 35.5,
   "n": 300
  },
  {
   "mode": "gs2",
   "ratio": 0.5,
   "tokens_retained": 0.612,
   "rouge1_recall": 80.28,
   "rouge2_recall": 40.01,
   "n": 300
  }
 ],
 "tuning": {
  "file": "https://cymetica.com/static/research/tuatara-summarize/tune_cnn_val_seeded.json",
  "rows": [
   {
    "system": "lead",
    "name": "Lead-k",
    "rouge1": 42.07,
    "rouge2": 19.37,
    "rougeL": 26.17,
    "rougeLsum": 38.45,
    "tokens": 77.3,
    "ms_per_doc": 24.7,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:best:lead_prior=8+length_norm=1",
    "name": "tuatara:best:lead_prior=8+length_norm=1",
    "rouge1": 41.95,
    "rouge2": 19.32,
    "rougeL": 26.11,
    "rougeLsum": 38.42,
    "tokens": 79.3,
    "ms_per_doc": 469.6,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:best:lead_prior=16+length_norm=1",
    "name": "tuatara:best:lead_prior=16+length_norm=1",
    "rouge1": 41.94,
    "rouge2": 19.31,
    "rougeL": 26.12,
    "rougeLsum": 38.41,
    "tokens": 79.4,
    "ms_per_doc": 438.6,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:best:lead_prior=16+length_norm=2",
    "name": "tuatara:best:lead_prior=16+length_norm=2",
    "rouge1": 41.63,
    "rouge2": 18.96,
    "rougeL": 26.22,
    "rougeLsum": 38.04,
    "tokens": 74.0,
    "ms_per_doc": 447.1,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:best:lead_prior=4+length_norm=2",
    "name": "tuatara:best:lead_prior=4+length_norm=2",
    "rouge1": 41.57,
    "rouge2": 18.82,
    "rougeL": 26.18,
    "rougeLsum": 37.95,
    "tokens": 73.1,
    "ms_per_doc": 438.5,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=1+lead_prior=16",
    "name": "tuatara:gs1:length_norm=1+lead_prior=16",
    "rouge1": 40.78,
    "rouge2": 18.08,
    "rougeL": 25.95,
    "rougeLsum": 37.07,
    "tokens": 76.7,
    "ms_per_doc": 131.1,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=1+lead_prior=4",
    "name": "tuatara:gs1:length_norm=1+lead_prior=4",
    "rouge1": 40.26,
    "rouge2": 17.58,
    "rougeL": 25.79,
    "rougeLsum": 36.6,
    "tokens": 76.7,
    "ms_per_doc": 126.5,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=1.5+lead_prior=16",
    "name": "tuatara:gs1:length_norm=1.5+lead_prior=16",
    "rouge1": 40.13,
    "rouge2": 17.47,
    "rougeL": 25.75,
    "rougeLsum": 36.34,
    "tokens": 69.5,
    "ms_per_doc": 130.3,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=1.5+lead_prior=8",
    "name": "tuatara:gs1:length_norm=1.5+lead_prior=8",
    "rouge1": 39.95,
    "rouge2": 17.25,
    "rougeL": 25.49,
    "rougeLsum": 36.19,
    "tokens": 69.6,
    "ms_per_doc": 124.5,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=1.5+lead_prior=4",
    "name": "tuatara:gs1:length_norm=1.5+lead_prior=4",
    "rouge1": 39.6,
    "rouge2": 16.96,
    "rougeL": 25.34,
    "rougeLsum": 35.83,
    "tokens": 69.1,
    "ms_per_doc": 130.4,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1:length_norm=2+lead_prior=8",
    "name": "tuatara:gs1:length_norm=2+lead_prior=8",
    "rouge1": 37.51,
    "rouge2": 15.75,
    "rougeL": 24.22,
    "rougeLsum": 34.1,
    "tokens": 59.8,
    "ms_per_doc": 126.6,
    "n": 300,
    "errors": 0
   },
   {
    "system": "tuatara:gs1",
    "name": "TuataraSum GS1",
    "rouge1": 37.18,
    "rouge2": 15.49,
    "rougeL": 23.48,
    "rougeLsum": 33.68,
    "tokens": 95.1,
    "ms_per_doc": 133.3,
    "n": 300,
    "errors": 0
   }
  ]
 },
 "sampling_note": "abisee/cnn_dailymail lists every CNN article before the DailyMail ones; a head slice is CNN-only (Lead-3 about 29 R1 there). The *_head_* files record the runs taken before this was found; the reported rows use the seeded sample.",
 "cnn_head_subset": {
  "file": "https://cymetica.com/static/research/tuatara-summarize/cnn_head_test.json",
  "rows": [
   {
    "system": "tuatara:tuned-best",
    "name": "TuataraSum tuned-best",
    "rouge1": 29.92,
    "rouge2": 11.66,
    "rougeL": 20.22,
    "rougeLsum": 26.57,
    "tokens": 75.6,
    "ms_per_doc": 660.1,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "tuatara:tuned",
    "name": "TuataraSum tuned",
    "rouge1": 29.6,
    "rouge2": 10.85,
    "rougeL": 20.11,
    "rougeLsum": 26.32,
    "tokens": 61.6,
    "ms_per_doc": 160.5,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "lead",
    "name": "Lead-k",
    "rouge1": 29.55,
    "rouge2": 11.27,
    "rougeL": 19.95,
    "rougeLsum": 26.26,
    "tokens": 65.4,
    "ms_per_doc": 23.8,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "tuatara:faithful",
    "name": "TuataraSum faithful (2015 CGI)",
    "rouge1": 27.23,
    "rouge2": 9.3,
    "rougeL": 17.79,
    "rougeLsum": 21.74,
    "tokens": 76.3,
    "ms_per_doc": 212.4,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "tuatara:gs1",
    "name": "TuataraSum GS1",
    "rouge1": 26.41,
    "rouge2": 9.71,
    "rougeL": 17.67,
    "rougeLsum": 23.43,
    "tokens": 99.3,
    "ms_per_doc": 169.2,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "lexrank",
    "name": "LexRank (sumy)",
    "rouge1": 25.65,
    "rouge2": 7.63,
    "rougeL": 16.82,
    "rougeLsum": 22.48,
    "tokens": 78.0,
    "ms_per_doc": 336.8,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "tuatara:best",
    "name": "TuataraSum fusion",
    "rouge1": 24.77,
    "rouge2": 8.58,
    "rougeL": 16.4,
    "rougeLsum": 21.96,
    "tokens": 105.1,
    "ms_per_doc": 672.0,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "lsa",
    "name": "LSA (sumy)",
    "rouge1": 23.4,
    "rouge2": 5.89,
    "rougeL": 14.91,
    "rougeLsum": 20.21,
    "tokens": 67.0,
    "ms_per_doc": 1193.7,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "luhn",
    "name": "Luhn (sumy)",
    "rouge1": 23.28,
    "rouge2": 7.16,
    "rougeL": 15.24,
    "rougeLsum": 20.41,
    "tokens": 100.7,
    "ms_per_doc": 309.2,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "tuatara:gs2",
    "name": "TuataraSum GS2",
    "rouge1": 22.86,
    "rouge2": 7.17,
    "rougeL": 14.97,
    "rougeLsum": 20.15,
    "tokens": 104.7,
    "ms_per_doc": 1304.7,
    "n": 1000,
    "errors": 0
   },
   {
    "system": "textrank",
    "name": "TextRank (sumy)",
    "rouge1": 22.09,
    "rouge2": 6.7,
    "rougeL": 14.76,
    "rougeLsum": 19.38,
    "tokens": 103.8,
    "ms_per_doc": 340.4,
    "n": 1000,
    "errors": 0
   }
  ]
 },
 "harness": "scripts/tuatarasum_bench.py \u2014 rouge-score 0.1.2 (Porter stemmer; rougeLsum over newline-joined sentences), NLTK punkt sentences for the baselines, sumy 0.11 baselines; seed 20261028; n=1000 per split",
 "git_sha": "7ec0b92d76c077da7a57f15f119198dfae08cb98",
 "dataset_version": "abisee/cnn_dailymail 3.0.0 test (seeded sample of 1000, k=3); EdinburghNLP/xsum test (seeded sample of 1000, k=1)",
 "built_at": "2026-10-28",
 "served_system": "tuatara:tuned-best",
 "metrics": {
  "cnn_dailymail/test/rouge1": 39.46,
  "cnn_dailymail/test/rouge2": 17.34,
  "cnn_dailymail/test/rougeLsum": 35.88,
  "xsum/test/rouge1": 17.42
 },
 "previous_release": {}
}