{
  "type": "excalidraw", "version": 2, "source": "https://excalidraw.com",
  "appState": {"viewBackgroundColor": "#FAF9F5"}, "files": {},
  "elements": [
    {"id":"sft","type":"text","x":30,"y":20,"width":500,"height":30,"text":"SFT · Learn from examples","fontSize":24,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"sft-note","type":"text","x":30,"y":60,"width":500,"height":25,"text":"Often a starting step. Not RL.","fontSize":20,"fontFamily":3,"strokeColor":"#62635B","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"sft-down","type":"arrow","x":190,"y":105,"width":0,"height":45,"points":[[0,0],[0,45]],"strokeColor":"#62635B","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"try","type":"text","x":30,"y":170,"width":470,"height":30,"text":"Model tries new answers","fontSize":24,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"try-down","type":"arrow","x":190,"y":220,"width":0,"height":35,"points":[[0,0],[0,35]],"strokeColor":"#0C5F44","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"feedback-label","type":"text","x":30,"y":277,"width":600,"height":30,"text":"Choose where feedback comes from","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"rlhf","type":"text","x":30,"y":340,"width":190,"height":30,"text":"RLHF","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25,"link":"https://rlresearch.ai/methodologies/rlhf/"},
    {"id":"rlhf-note","type":"text","x":30,"y":380,"width":195,"height":50,"text":"People choose\nbetter answers","fontSize":20,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"rlaif","type":"text","x":250,"y":340,"width":190,"height":30,"text":"RLAIF","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25,"link":"https://rlresearch.ai/methodologies/rlaif/"},
    {"id":"rlaif-note","type":"text","x":250,"y":380,"width":195,"height":50,"text":"AI judges\nthe answers","fontSize":20,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"rlcd","type":"text","x":470,"y":340,"width":190,"height":30,"text":"RLCD","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25,"link":"https://rlresearch.ai/methodologies/rlcd/"},
    {"id":"rlcd-note","type":"text","x":470,"y":380,"width":210,"height":50,"text":"Opposite prompts\ncreate preferences","fontSize":20,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"rlvr","type":"text","x":725,"y":340,"width":190,"height":30,"text":"RLVR","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25,"link":"https://rlresearch.ai/methodologies/rlvr/"},
    {"id":"rlvr-note","type":"text","x":725,"y":380,"width":195,"height":50,"text":"Tests check\nif answers work","fontSize":20,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"human-reward","type":"arrow","x":120,"y":450,"width":210,"height":40,"points":[[0,0],[0,20],[210,20],[210,40]],"strokeColor":"#62635B","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"ai-reward","type":"arrow","x":345,"y":450,"width":0,"height":40,"points":[[0,0],[0,40]],"strokeColor":"#62635B","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"contrast-reward","type":"arrow","x":580,"y":450,"width":220,"height":40,"points":[[0,0],[0,20],[-220,20],[-220,40]],"strokeColor":"#62635B","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"reward","type":"text","x":180,"y":510,"width":370,"height":27.5,"text":"Learned reward model","fontSize":22,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"reward-note","type":"text","x":180,"y":547,"width":430,"height":25,"text":"It scores the model's new answers","fontSize":20,"fontFamily":3,"strokeColor":"#62635B","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"reward-update","type":"arrow","x":345,"y":590,"width":0,"height":60,"points":[[0,0],[0,60]],"strokeColor":"#0C5F44","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"verifier-update","type":"arrow","x":810,"y":450,"width":445,"height":200,"points":[[0,0],[0,170],[-445,170],[-445,200]],"strokeColor":"#0C5F44","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"direct-note","type":"text","x":620,"y":520,"width":178,"height":50,"text":"Reward straight\nfrom the check","fontSize":18,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"optimizer","type":"text","x":180,"y":675,"width":450,"height":30,"text":"PPO or GRPO · Update the model","fontSize":24,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"optimizer-note","type":"text","x":180,"y":715,"width":630,"height":25,"text":"These use rewards. They do not choose the judge.","fontSize":20,"fontFamily":3,"strokeColor":"#62635B","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"loop","type":"arrow","x":710,"y":687,"width":290,"height":502,"points":[[0,0],[290,0],[290,-502],[-240,-502]],"strokeColor":"#0C5F44","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"repeat","type":"text","x":880,"y":220,"width":95,"height":25,"text":"Repeat","fontSize":20,"fontFamily":3,"strokeColor":"#0C5F44","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"dpo-branch","type":"arrow","x":430,"y":40,"width":120,"height":0,"points":[[0,0],[120,0]],"strokeColor":"#62635B","strokeWidth":2,"opacity":100,"roughness":0,"endArrowhead":"arrow"},
    {"id":"alternative","type":"text","x":470,"y":8,"width":50,"height":25,"text":"or","fontSize":20,"fontFamily":3,"strokeColor":"#62635B","opacity":100,"roughness":0,"lineHeight":1.25},
    {"id":"dpo","type":"text","x":580,"y":20,"width":420,"height":30,"text":"DPO · A separate route","fontSize":24,"fontFamily":3,"strokeColor":"#141413","opacity":100,"roughness":0,"lineHeight":1.25,"link":"https://rlresearch.ai/methodologies/dpo/"},
    {"id":"dpo-note","type":"text","x":580,"y":60,"width":420,"height":75,"text":"Preferred + rejected examples\nDirect update, no reward model\nNo online RL loop required","fontSize":20,"fontFamily":3,"strokeColor":"#62635B","opacity":100,"roughness":0,"lineHeight":1.25}
  ]
}