"""Tests for the Chapter 14 lab. Run with `python -m pytest -q` from this folder. Set RL_SOLUTIONS=1 to run the same tests against the reference solution. """ import importlib import math import os from fractions import Fraction import pytest from helpers import Answer, SimulatedJudge, last_number_reward, naive_substring_reward, sigmoid R = importlib.import_module("solutions.rewards" if os.environ.get("RL_SOLUTIONS") else "rewards") # --------------------------------------------------------------------------- # Task 1: extract_boxed # --------------------------------------------------------------------------- @pytest.mark.parametrize("text, expected", [ (r"The answer is \boxed{42}.", ["42"]), (r"so \boxed{\frac{1}{2}}.", [r"\frac{1}{2}"]), # nested braces (r"\boxed{\frac{\sqrt{3}}{2}}", [r"\frac{\sqrt{3}}{2}"]), # two levels deep (r"\boxed{3} or \boxed{ 4 }", ["3", "4"]), # several, stripped (r"\boxed {7}", ["7"]), # space before { (r"\boxed{\{1, 2\}}", [r"\{1, 2\}"]), # escaped braces ("no box here", []), (r"\boxed{5", []), # never closed (r"\boxed{1} then \boxed{2", ["1"]), # second never closed (r"\boxed{}", [""]), # empty box (r"\boxed 5 and \boxed{6}", ["6"]), # \boxed without { ]) def test_extract_boxed(text, expected): assert R.extract_boxed(text) == expected, f"extract_boxed({text!r}) should be {expected!r}" # --------------------------------------------------------------------------- # Task 2: parse_number # --------------------------------------------------------------------------- HALF = Fraction(1, 2) @pytest.mark.parametrize("s, expected", [ ("42", 42), (" 42 ", 42), ("-7", -7), ("−7", -7), ("+3", 3), ("3.0", 3), ("0.5", HALF), (".5", HALF), ("0.50", HALF), ("0.125", Fraction(1, 8)), ("1/2", HALF), ("2/4", HALF), ("(3)", 3), (r"\frac{1}{2}", HALF), (r"\dfrac{3}{4}", Fraction(3, 4)), (r"\tfrac12", HALF), (r"\frac12", HALF), (r"-\frac{1}{2}", -HALF), (r"\frac{-1}{2}", -HALF), (r"\frac{10}{4}", Fraction(5, 2)), (r"\left(\frac{1}{2}\right)", HALF), ("1,000", 1000), ("1{,}000", 1000), ("12,345,678", 12345678), ("1,234.5", Fraction(2469, 2)), ("$5$", 5), ("x = 5", 5), (r"y=\frac{1}{2}", HALF), ("5.", 5), (r"90^\circ", 90), (r"90^{\circ}", 90), (r"50\%", 50), ("50%", 50), (r"5\text{ cm}", 5), (r"12 \mbox{ apples}", 12), (r"3\,000", 3000), ]) def test_parse_number_valid(s, expected): got = R.parse_number(s) exp = None if expected is None else Fraction(expected) assert got == exp, f"parse_number({s!r}) should be {exp!r}, got {got!r}" @pytest.mark.parametrize("s", [ "", "abc", r"\sqrt{2}", "x+1", "1,2", "1/0", "1.2.3", r"\text{five}", "1e3", "2^3", "1, 2, 3", ]) def test_parse_number_rejects_non_numbers(s): assert R.parse_number(s) is None, f"parse_number({s!r}) should be None (not a single number)" def test_parse_number_returns_exact_fraction(): got = R.parse_number("0.1") assert isinstance(got, Fraction), "parse_number should return a fractions.Fraction, not a float" assert got == Fraction(1, 10), "0.1 should parse to exactly 1/10 (use Fraction('0.1'), not float)" # --------------------------------------------------------------------------- # Task 3: math_reward # --------------------------------------------------------------------------- @pytest.mark.parametrize("response, gold, expected", [ (r"Adding, 37+48 = \boxed{85}.", "85", 1.0), (r"Adding, 37+48 = \boxed{75}.", "85", 0.0), (r"Half of it: \boxed{\frac{1}{2}}", "1/2", 1.0), # equivalent forms (r"Half of it: \boxed{0.5}", r"\frac{1}{2}", 1.0), (r"\boxed{1,000}", "1000", 1.0), (r"\boxed{x = 12}", "12", 1.0), ("So the answer is 0.5.", "1/2", 1.0), # fallbacks ("Answer: 42", "42", 1.0), ("Final answer is: 42", "42", 1.0), ("The answer is 85. I'm 100% sure.", "85", 1.0), # stop at the sentence end ("I think it's 42", "42", 0.0), # no answer marked ("", "42", 0.0), (r"\boxed{157}", "57", 0.0), # no substring matching (r"It's \boxed{0.5} or maybe \boxed{0.25}", "1/2", 0.0), # hedging (r"First \boxed{0.25}... wait, \boxed{0.5}", "1/2", 0.0), # changing your mind is also 2 answers (r"\boxed{0.5}, i.e. \boxed{\frac{1}{2}}", "1/2", 1.0), # same answer twice is fine (" ".join(rf"\boxed{{{n}}}" for n in range(20, 199)), "85", 0.0), # spam (r"\boxed{x^2 + 1}", "x^2+1", 1.0), # non-numeric, normalized (r"\boxed{x^2 + 2}", "x^2+1", 0.0), (r"The answer is \boxed{41}. Answer: 42", "42", 0.0), # boxes take precedence ]) def test_math_reward(response, gold, expected): got = R.math_reward(response, gold) assert got == expected, f"math_reward({response[:60]!r}..., gold={gold!r}) should be {expected}, got {got}" def test_math_reward_max_answers(): resp = r"It's \boxed{0.25} or \boxed{0.5}" assert R.math_reward(resp, "1/2", max_answers=1) == 0.0, "two distinct answers with max_answers=1 -> 0" assert R.math_reward(resp, "1/2", max_answers=2) == 1.0, "with max_answers=2, grade the last answer (0.5 = 1/2)" assert R.math_reward(resp, "1/4", max_answers=2) == 0.0, "with max_answers=2, the LAST answer is the one graded" def test_naive_verifiers_are_fooled_but_yours_is_not(): spam = " ".join(rf"\boxed{{{n}}}" for n in range(20, 199)) assert naive_substring_reward(spam, "85") == 1.0 # sanity check of the helper assert R.math_reward(spam, "85") == 0.0, "spamming every number must earn 0" assert last_number_reward(r"\boxed{\frac{1}{2}}", "0.5") == 0.0 # helper's blind spot assert R.math_reward(r"\boxed{\frac{1}{2}}", "0.5") == 1.0, "\\frac{1}{2} must match 0.5" # --------------------------------------------------------------------------- # Task 4: rubric_reward # --------------------------------------------------------------------------- RUBRIC = [ {"criterion": "Says the sky scatters blue light more", "category": "essential"}, # w = 1.0 {"criterion": "Mentions sunlight contains all colours", "category": "important"}, # w = 0.7 {"criterion": "Uses an everyday analogy", "category": "optional"}, # w = 0.3 {"criterion": "Avoids claiming the sky reflects the ocean", "category": "pitfall"}, # w = 0.9 ] def test_rubric_all_pass_and_all_fail(): assert R.rubric_reward(RUBRIC, [1, 1, 1, 1]) == pytest.approx(1.0) assert R.rubric_reward(RUBRIC, [0, 0, 0, 0]) == pytest.approx(0.0) def test_rubric_hand_example(): # (1.0*1 + 0.7*0 + 0.3*1 + 0.9*1) / (1.0 + 0.7 + 0.3 + 0.9) = 2.2 / 2.9 got = R.rubric_reward(RUBRIC, [1, 0, 1, 1]) assert got == pytest.approx(2.2 / 2.9), f"expected 2.2/2.9 = {2.2/2.9:.4f}, got {got}" def test_rubric_partial_credit_and_explicit_weight(): rubric = [{"criterion": "a", "category": "essential"}, {"criterion": "b", "category": "optional", "weight": 2.0}] # weight overrides 0.3 got = R.rubric_reward(rubric, [0.5, 1.0]) assert got == pytest.approx((1.0 * 0.5 + 2.0 * 1.0) / 3.0), "explicit 'weight' should override the category default" def test_rubric_essential_gate(): # Missing the essential criterion: 1.9/2.9 without the gate, 0 with it. assert R.rubric_reward(RUBRIC, [0, 1, 1, 1]) == pytest.approx(1.9 / 2.9) assert R.rubric_reward(RUBRIC, [0, 1, 1, 1], gate_essential=True) == 0.0, "gate: failed essential -> 0" assert R.rubric_reward(RUBRIC, [0.9, 1, 1, 1], gate_essential=True) == 0.0, "gate: essential check < 1 -> 0" assert R.rubric_reward(RUBRIC, [1, 0, 1, 1], gate_essential=True) == pytest.approx(2.2 / 2.9), \ "gate only applies to essential criteria" def test_rubric_errors(): with pytest.raises(ValueError): R.rubric_reward(RUBRIC, [1, 1, 1]) with pytest.raises(ValueError): R.rubric_reward([{"criterion": "a", "category": "essential", "weight": 0.0}], [1]) # --------------------------------------------------------------------------- # Task 5: order swapping # --------------------------------------------------------------------------- GOOD_SHORT = Answer("concise and correct", quality=7.0, n_words=150) OKAY_LONG = Answer("padded", quality=6.0, n_words=600) OKAY_SHORT = Answer("okay", quality=6.0, n_words=150) def test_debiased_preference_exact_formula(): judge = SimulatedJudge(sharpness=1.0, position_bias=2.0) # soft: returns probabilities got = R.debiased_preference(judge, "q", GOOD_SHORT, OKAY_SHORT) expected = 0.5 * (sigmoid(1 + 2) + sigmoid(1 - 2)) # = 0.5[σ(Δ+b) + σ(Δ-b)] assert got == pytest.approx(expected), f"expected 0.5*(σ(3)+σ(-1)) = {expected:.4f}, got {got:.4f}" assert judge.calls == 2, f"with n_samples=1 the judge should be called exactly twice, not {judge.calls}" def test_swapping_fixes_position_bias(): judge = SimulatedJudge(sharpness=1.0, position_bias=3.0) # Shown second, the better answer loses: sigmoid(-1 + 3) > 0.5 for the worse answer. assert judge("q", OKAY_SHORT, GOOD_SHORT) > 0.5 p = R.debiased_preference(judge, "q", GOOD_SHORT, OKAY_SHORT) assert p > 0.5, f"after swapping, the better answer should be preferred (got P = {p:.3f})" # ...but the size of the preference shrinks toward 1/2 (the chapter's derivation) assert p < sigmoid(1.0), "a position-biased judge, even debiased, is less decisive than an unbiased one" def test_swapping_does_not_fix_verbosity_bias(): judge = SimulatedJudge(sharpness=1.0, position_bias=1.0, verbosity_bias=1.0) # The padded answer is 4x longer (2 doublings = +2 logits), worth more than 1 quality point. p = R.debiased_preference(judge, "q", GOOD_SHORT, OKAY_LONG) assert p < 0.5, f"verbosity bias is the same in both orders, so it survives swapping (got P = {p:.3f})" def test_debiased_preference_with_sampled_verdicts(): judge = SimulatedJudge(sharpness=1.0, position_bias=2.0, soft=False, seed=0) p = R.debiased_preference(judge, "q", GOOD_SHORT, OKAY_SHORT, n_samples=4000) expected = 0.5 * (sigmoid(3) + sigmoid(-1)) assert abs(p - expected) < 0.03, f"with 4000 sampled verdicts per order, expected about {expected:.3f}, got {p:.3f}" assert judge.calls == 8000, "n_samples judgments in each of the two orders" def test_consistent_verdict(): fair = SimulatedJudge(sharpness=1.0) assert R.consistent_verdict(fair, "q", GOOD_SHORT, OKAY_SHORT) == "a" assert R.consistent_verdict(fair, "q", OKAY_SHORT, GOOD_SHORT) == "b" biased = SimulatedJudge(sharpness=1.0, position_bias=3.0) # always prefers the first slot assert R.consistent_verdict(biased, "q", GOOD_SHORT, OKAY_SHORT) == "tie", \ "the verdict flips with the order, so MT-Bench's rule calls it a tie" assert R.consistent_verdict(fair, "q", OKAY_SHORT, OKAY_SHORT) == "tie", "identical answers: tie" mild = SimulatedJudge(sharpness=1.0, position_bias=0.5) assert R.consistent_verdict(mild, "q", GOOD_SHORT, OKAY_SHORT) == "a", "a small bias doesn't flip a clear win" assert math.isclose(sigmoid(0.0), 0.5)