"""Helper code for the Chapter 14 lab. Complete: you don't need to change it. Read it once. It contains: * normalize_text a light string normalizer for non-numeric answers * naive_substring_reward, last_number_reward two *naive* math verifiers, so you can see how they break * Answer, SimulatedJudge a toy LLM-as-a-judge with position and verbosity bias (the same model as the chapter's judge-bias demo) * make_response text produced by four answering "strategies" for the addition task; run.py trains a tiny policy over them """ from __future__ import annotations import math import re from dataclasses import dataclass import numpy as np # --------------------------------------------------------------------------- # Text normalization for answers that are not plain numbers # --------------------------------------------------------------------------- def normalize_text(s: str) -> str: """Normalize a non-numeric answer string for exact comparison. Removes `$` signs, `\\left` / `\\right`, LaTeX spacing commands (`\\,`, `\\!`, `\\;`, `\\ `), all whitespace, and one trailing period; lowercases the rest. >>> normalize_text(" $x^2 + 1$. ") 'x^2+1' """ s = s.strip().replace("$", "") s = s.replace("\\left", "").replace("\\right", "") for cmd in ("\\,", "\\!", "\\;", "\\ ", "\\displaystyle"): s = s.replace(cmd, "") s = re.sub(r"\s+", "", s) s = s.rstrip(".") return s.lower() # --------------------------------------------------------------------------- # Two naive verifiers (do NOT use these for training!) # --------------------------------------------------------------------------- def naive_substring_reward(response: str, gold: str) -> float: """1.0 if the gold answer appears *anywhere* in the response. The first thing many people write. Rewards listing many answers, and accepts "157" when the answer is "57". """ return 1.0 if str(gold).strip() in response else 0.0 _NUM = re.compile(r"-?\d+(?:\.\d+)?") def last_number_reward(response: str, gold: str) -> float: """1.0 if the last number in the response equals the gold number. Better than substring matching, but it rejects "\\frac{1}{2}" for gold 0.5, "1,000" for 1000, and accepts a correct number that merely happens to come last (e.g. in "... I am 100% sure"). """ nums = _NUM.findall(response) if not nums: return 0.0 try: return 1.0 if float(nums[-1]) == float(gold) else 0.0 except ValueError: return 0.0 # --------------------------------------------------------------------------- # A toy LLM judge with position and verbosity bias # --------------------------------------------------------------------------- @dataclass(frozen=True) class Answer: """A candidate answer with a hidden true quality (0-10) and a length in words.""" text: str quality: float n_words: int def sigmoid(z: float) -> float: return 1.0 / (1.0 + math.exp(-z)) class SimulatedJudge: """A pairwise judge: "which of these two answers is better?" The judge's probability of preferring the answer shown FIRST is P(first wins) = sigmoid( sharpness * (q_first - q_second) + position_bias + verbosity_bias * log2(words_first / words_second) ) * sharpness: logits per quality point. 1.0 means a 1-point better answer wins 73% of the time against an unbiased judge. * position_bias: extra logit for whatever is shown first (negative = favours second). * verbosity_bias: extra logits per doubling of relative length. With soft=True, calling the judge returns that probability (like reading the probabilities of the tokens "1" and "2", as RLAIF does). With soft=False it returns a sampled verdict, 1.0 or 0.0, using its own seeded random generator. The judge also counts how many times it has been called, in `self.calls`. """ def __init__(self, sharpness: float = 1.0, position_bias: float = 0.0, verbosity_bias: float = 0.0, soft: bool = True, seed: int = 0): self.sharpness = sharpness self.position_bias = position_bias self.verbosity_bias = verbosity_bias self.soft = soft self.rng = np.random.default_rng(seed) self.calls = 0 def prob_first_wins(self, first: Answer, second: Answer) -> float: z = (self.sharpness * (first.quality - second.quality) + self.position_bias + self.verbosity_bias * math.log2(first.n_words / second.n_words)) return sigmoid(z) def __call__(self, prompt: str, first: Answer, second: Answer) -> float: self.calls += 1 p = self.prob_first_wins(first, second) if self.soft: return p return 1.0 if self.rng.random() < p else 0.0 # --------------------------------------------------------------------------- # Answering "strategies" for the addition task (used by run.py) # --------------------------------------------------------------------------- STRATEGIES = ["honest (boxed)", "honest (unboxed)", "hedge 3 answers", "spam all answers"] def honest_guess(a: int, b: int, rng: np.random.Generator, p_correct: float = 0.6) -> int: """The sum, or (with prob 1 - p_correct) the sum off by 10: a dropped or extra carry.""" s = a + b if rng.random() < p_correct: return s return s + (10 if rng.random() < 0.5 else -10) def make_response(strategy: int, a: int, b: int, rng: np.random.Generator) -> str: """The text a 'policy' following `strategy` writes for the prompt "a+b=". 0 honest (boxed): works it out, boxes one answer (correct 60% of the time) 1 honest (unboxed): same, but ends with "The answer is N." and no box 2 hedge 3 answers: boxes guess-10, guess and guess+10 ("it's one of these") 3 spam all answers: boxes every number from 20 to 198 """ g = honest_guess(a, b, rng) if strategy == 0: return f"{a} plus {b}: add the tens, then the ones. So {a}+{b} = \\boxed{{{g}}}." if strategy == 1: return f"{a} plus {b}: add the tens, then the ones. The answer is {g}." if strategy == 2: return (f"I'm not sure about the carry, so it is one of " f"\\boxed{{{g - 10}}}, \\boxed{{{g}}} or \\boxed{{{g + 10}}}.") if strategy == 3: return " ".join(f"\\boxed{{{n}}}" for n in range(20, 199)) raise ValueError(f"unknown strategy {strategy}")