mohsenhariri · Oct 20, 2026
diff --git a/README.md b/README.md
@@ -97,6 +97,35 @@ Particularly, you can provide `--start_date` and `--end_date` flags (using the `
 python -m lcb_runner.evaluation.compute_scores --eval_all_file {saved_eval_all_file} --start_date 2023-09-01
 ```
 
+Add `--bayes_at_n` to this scoring command to report the posterior mean and
+posterior standard deviation of average success probability over the selected
+code-generation questions. `--bayes_output_file posterior.json` also saves the
+overall and nonempty difficulty-group summaries as JSON, with `mean`, `std`,
+`num_questions`, and `samples_per_question`. Existing date and platform filters
+apply before these summaries are computed; the evaluated input file is preserved.
+
+```sh
+python -m lcb_runner.evaluation.compute_scores --eval_all_file results_eval_all.json \
+    --bayes_at_n --bayes_output_file posterior.json
+```
+
+This is the binary, uniform-prior case of
+[Bayes@N](https://arxiv.org/abs/2510.04265). Each question has an independent
+Beta(1, 1) prior. If `c` of its `N` generations pass, its posterior is
+Beta(`c + 1`, `N - c + 1`). The corpus mean averages question posterior means;
+the corpus SD is the square root of the sum of question posterior variances,
+divided by the number of questions. Both use the [0, 1] score scale. SD describes
+uncertainty about success probability on this fixed question set.
+
+Use a complete `codegeneration` evaluation file with independent generations
+under a fixed sampling configuration. The option rejects empty selections,
+missing or duplicate question IDs, missing/nonbinary grades, and unequal positive
+sample counts. It recomputes from `graded_list` after filtering, so a combined
+file from a resumed run is scored as one corpus. The file cannot reveal questions
+omitted entirely. Self-repair, historical evidence, and categorical rubrics are
+outside this option's scope. Runner aggregate files and default score reporting
+are unchanged.
+
 **NOTE: We have pruned a large number of test cases from the original benchmark and created `code_generation_lite` which is set as the default benchmark offering similar performance estimation much faster. If you wish to use the original benchmark, please use the `--not_fast` flag. We are in the process of updating the leaderboard scores with this updated setting.** 
 
 **NOTE: V2 Update: to run the update LiveCodeBench please use `--release_version release_v2`. In addition, if you have existing results from `release_v1` you can add `--continue_existing` or better `--continue_existing_with_eval` flags to reuse the old completions or evaluations respectively.**

diff --git a/lcb_runner/evaluation/bayes.py b/lcb_runner/evaluation/bayes.py
@@ -0,0 +1,44 @@
+"""Binary Bayes@N summaries (https://arxiv.org/abs/2510.04265)."""
+
+import math
+from numbers import Real
+
+
+def bayes_at_n(results: list[dict]) -> dict:
+    """Summarize average success probability with independent Beta(1, 1) priors.
+
+    Each record is a question with an ID and repeated binary `graded_list`.
+    All questions must have the same positive number of independent generations.
+    Returns the posterior mean and SD on the [0, 1] score scale, plus cohort size.
+    """
+    if not results:
+        raise ValueError("Bayes@N needs at least one question")
+    identifiers = set()
+    counts = []
+    trial_count = None
+    for result in results:
+        identifier = result.get("question_id")
+        if not isinstance(identifier, str) or not identifier or identifier in identifiers:
+            raise ValueError("Bayes@N needs unique, nonempty question_id strings")
+        identifiers.add(identifier)
+        grades = result.get("graded_list")
+        if not isinstance(grades, list) or not grades:
+            raise ValueError("Bayes@N needs a nonempty graded_list for every question")
+        if any(not isinstance(g, Real) or g not in (0, 1) for g in grades):
+            raise ValueError("Bayes@N needs binary grades (0/1 or false/true)")
+        if trial_count is None:
+            trial_count = len(grades)
+        elif len(grades) != trial_count:
+            raise ValueError("Bayes@N needs the same sample count for every question")
+        counts.append(sum(grades))
+
+    total = trial_count + 2
+    means = [(c + 1) / total for c in counts]
+    variances = [((c + 1) / total) * ((trial_count - c + 1) / total)
+                 / (total + 1) for c in counts]
+    return {
+        "mean": math.fsum(means) / len(results),
+        "std": math.sqrt(math.fsum(variances)) / len(results),
+        "num_questions": len(results),
+        "samples_per_question": trial_count,
+    }
diff --git a/lcb_runner/evaluation/compute_scores.py b/lcb_runner/evaluation/compute_scores.py
@@ -2,6 +2,9 @@
 import argparse
 import numpy as np
 from datetime import datetime
+from pathlib import Path
+
+from lcb_runner.evaluation.bayes import bayes_at_n
 
 from lcb_runner.lm_styles import LanguageModelStore
 from lcb_runner.evaluation.pass_k_utils import (
@@ -14,6 +17,14 @@
 
 def get_parser():
     parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--bayes_at_n", action="store_true",
+        help="Report binary Bayes@N posterior mean and standard deviation",
+    )
+    parser.add_argument(
+        "--bayes_output_file", type=str, default=None,
+        help="Save Bayes@N summaries as JSON (requires --bayes_at_n)",
+    )
     parser.add_argument(
         "--model",
         type=str,
@@ -70,6 +81,14 @@ def get_parser():
 
 
 def compute_scores(args):
+    use_bayes = getattr(args, "bayes_at_n", False)
+    posterior_path = getattr(args, "bayes_output_file", None)
+    if posterior_path and not use_bayes:
+        raise ValueError("--bayes_output_file requires --bayes_at_n")
+    if use_bayes and args.scenario != Scenario.codegeneration:
+        raise ValueError("Bayes@N currently supports only codegeneration scores")
+    if posterior_path and Path(posterior_path).resolve() == Path(args.eval_all_file).resolve():
+        raise ValueError("Bayes@N output must differ from the evaluated input file")
     with open(args.eval_all_file, "r") as f:
         results = json.load(f)
 
@@ -91,6 +110,18 @@ def compute_scores(args):
     if args.platform is not None:
         results = [result for result in results if result["platform"] == args.platform]
 
+    if use_bayes:
+        posterior = {"overall": bayes_at_n(results)}
+        for difficulty in ("easy", "medium", "hard"):
+            subset = [r for r in results if r.get("difficulty") == difficulty]
+            if subset:
+                posterior[difficulty] = bayes_at_n(subset)
+        for name, moments in posterior.items():
+            print(f"Bayes@N ({name}): mean={moments['mean']:.6f}, std={moments['std']:.6f}")
+        if posterior_path:
+            with open(posterior_path, "w") as f:
+                json.dump(posterior, f, indent=4)
+
     print(len(results))
     totals = [len(x["graded_list"]) for x in results]
     corrects = [sum(x["graded_list"]) for x in results]

diff --git a/tests/test_bayes.py b/tests/test_bayes.py
@@ -0,0 +1,135 @@
+import json
+import math
+import subprocess
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from lcb_runner.evaluation.bayes import bayes_at_n
+from lcb_runner.evaluation.compute_scores import compute_scores
+from lcb_runner.utils.scenarios import Scenario
+
+
+def records(rows):
+    return [{"question_id": str(i), "graded_list": row} for i, row in enumerate(rows)]
+
+
+@pytest.mark.parametrize("rows,mean,variance", [
+    ([[0, 0], [1, 1]], 0.5, 3 / 160),
+    ([[0, 1], [0, 1]], 0.5, 1 / 40),
+    ([[0]], 1 / 3, 1 / 18),
+    ([[1]], 2 / 3, 1 / 18),
+])
+def test_exact_moments(rows, mean, variance):
+    result = bayes_at_n(records(rows))
+    assert result["mean"] == pytest.approx(mean)
+    assert result["std"] == pytest.approx(math.sqrt(variance))
+    assert result["num_questions"] == len(rows)
+    assert result["samples_per_question"] == len(rows[0])
+
+
+def test_combined_resume_evidence_and_permutations():
+    old = records([[0, 0]])
+    new = [{"question_id": "new", "graded_list": [0, 1]}]
+    result = bayes_at_n(old + new)
+    assert result["mean"] == pytest.approx(3 / 8)
+    assert result["std"] == pytest.approx(math.sqrt(7 / 320))
+    assert result == bayes_at_n(new + old)
+    assert result == bayes_at_n([dict(r, graded_list=r["graded_list"][::-1]) for r in old + new])
+    assert result["std"] != pytest.approx((bayes_at_n(old)["std"] + bayes_at_n(new)["std"]) / 2)
+
+
+@pytest.mark.parametrize("bad", [None, [], [0, 1, 0], [None, 1], ["1", 0],
+                                 [0.5, 1], [float("nan"), 0], [-1, 1], [2, 0]])
+def test_invalid_rows(bad):
+    data = records([[0, 0], [1, 1]])
+    data[1]["graded_list"] = bad
+    with pytest.raises(ValueError, match="Bayes@N"):
+        bayes_at_n(data)
+
+
+@pytest.mark.parametrize("bad", [None, "", "0", 1])
+def test_missing_or_duplicate_ids(bad):
+    data = records([[0, 0], [1, 1]])
+    data[1]["question_id"] = bad
+    with pytest.raises(ValueError, match="question_id"):
+        bayes_at_n(data)
+
+
+def test_empty_corpus():
+    with pytest.raises(ValueError, match="at least one"):
+        bayes_at_n([])
+
+
+def input_file(tmp_path):
+    data = records([[0, 0], [1, 1], [0, 1], [1, 1]])
+    for i, row in enumerate(data):
+        row.update(contest_date="2024-01-02T00:00:00", platform="leetcode",
+                   difficulty=["easy", "medium", "hard", "hard"][i],
+                   **{"pass@1": sum(row["graded_list"]) / 2})
+    data[3]["contest_date"] = "2023-01-01T00:00:00"
+    data[2]["platform"] = "atcoder"
+    path = tmp_path / "eval_all.json"
+    path.write_text(json.dumps(data))
+    return path
+
+
+def cli(path, *extra):
+    return subprocess.run([sys.executable, "-m", "lcb_runner.evaluation.compute_scores",
+                           "--eval_all_file", str(path), *extra],
+                          capture_output=True, text=True)
+
+
+def test_documented_cli_filters_and_json(tmp_path):
+    path = input_file(tmp_path)
+    before = path.read_bytes()
+    output = tmp_path / "posterior.json"
+    run = cli(path, "--bayes_at_n", "--bayes_output_file", str(output),
+              "--start_date", "2024-01-01", "--end_date", "2024-12-31",
+              "--platform", "leetcode")
+    assert run.returncode == 0, run.stderr
+    assert "Bayes@N (overall)" in run.stdout
+    assert "Pass@1" in run.stdout
+    result = json.loads(output.read_text())
+    assert set(result) == {"overall", "easy", "medium"}
+    assert result["overall"] == pytest.approx({"mean": 0.5, "std": math.sqrt(3 / 160),
+                                             "num_questions": 2, "samples_per_question": 2})
+    assert result["easy"]["std"] == pytest.approx(math.sqrt(3 / 80))
+    assert result["medium"]["mean"] == 0.75
+    assert path.read_bytes() == before
+
+
+def test_default_cli_has_no_posterior(tmp_path):
+    run = cli(input_file(tmp_path))
+    assert run.returncode == 0, run.stderr
+    assert "Bayes@N" not in run.stdout
+    assert "Pass@1" in run.stdout
+
+
+@pytest.mark.parametrize("extra", [
+    ["--bayes_output_file", "unused.json"],
+    ["--bayes_at_n", "--scenario", "selfrepair"],
+    ["--bayes_at_n", "--start_date", "2030-01-01"],
+])
+def test_cli_invalid_scope(tmp_path, extra):
+    run = cli(input_file(tmp_path), *extra)
+    assert run.returncode != 0
+    assert "Bayes@N" in run.stderr or "requires --bayes_at_n" in run.stderr
+
+
+def test_refuse_input_overwrite(tmp_path):
+    path = input_file(tmp_path)
+    before = path.read_bytes()
+    run = cli(path, "--bayes_at_n", "--bayes_output_file", str(path))
+    assert run.returncode != 0
+    assert path.read_bytes() == before
+
+
+def test_python_public_interface_uses_filtered_records(tmp_path, capsys):
+    path = input_file(tmp_path)
+    compute_scores(SimpleNamespace(
+        eval_all_file=str(path), start_date="2024-01-01", end_date=None,
+        platform="leetcode", scenario=Scenario.codegeneration, bayes_at_n=True,
+    ))
+    assert "Bayes@N (overall): mean=0.500000, std=0.136931" in capsys.readouterr().out