{ "name": "LLMInterview synthetic candidate-risk benchmark", "version": "1.0", "url": "https://llminterview.com/benchmark", "tests": [ "detection of a planted risk in a candidate work history", "interpretation of the risk into interviewer probes", "propagation of the risk into preparation output", "confidence calibration: confirmed versus inferred versus unknown" ], "synthetic": true, "disclaimer": "All fixtures are synthetic. No real candidate, recruiter, resume or company interview process is represented, and no formal academic validation is claimed.", "usage": "You may use these synthetic fixtures to test interview-prep or candidate-analysis systems. Attribution to LLMInterview.com is appreciated but not required.", "fixtures": [ { "id": "spellbrush-late-creative-alignment", "risk_class": "Late creative alignment", "role": "LLM Engineer, generative game content (synthetic evaluation against a public job description)", "recruiter_notes": "Fictional recruiter notes: the loop includes a technical screen and a project deep dive. Nothing was said about a workshop or a collaboration round.", "buried_resume_line": "Rewrote the character-dialogue prompt architecture end to end and shipped it, then walked the art and narrative leads through the new behaviour once it was stable.", "planted_risk": "Major prompt architecture changes are completed independently and creative partners are involved only after the behaviour is already set.", "expected_detection_terms": [ "creative", "late", "alignment" ], "expected_probes": [ "When in the change did you bring the narrative and art leads in?", "Describe a time a creative partner disagreed with model behaviour you had already shipped.", "How do you co-define intended character behaviour before you implement it?" ], "expected_exposure": { "headline": "Creative partners are brought in late, after behaviour is already set", "detail": "Your strongest prompt-architecture work reads as independently completed and reviewed afterwards. For a role where model behaviour is a creative artefact, that timing is the thing most likely to get pressed: interviewers ask when alignment happened, not whether the system worked." }, "expected_propagation": { "scenario": "A narrative lead says the new dialogue model is technically better but off-voice. Walk through how you would have avoided that before implementation, and what you would change now.", "drill": "Rewrite one shipped prompt-architecture change as a collaboration timeline: who defined intended behaviour, when, against which examples, and where sign-off happened before code.", "move": "Rebuild your deep-dive story so creative alignment happens before implementation, not after." }, "confidence_expectations": { "confirmed_rounds": [ "Technical screen", "Project deep dive" ], "inferred_rounds": [ { "name": "Creative collaboration probe", "confidence": "likely" }, { "name": "Behavioural round", "confidence": "likely" }, { "name": "Design workshop with creative leads", "confidence": "unknown" } ] } }, { "id": "metric-without-measurement", "risk_class": "Impressive metric, weak measurement substantiation", "role": "Applied AI Engineer, retrieval systems (synthetic)", "recruiter_notes": "Fictional recruiter notes: an AI system design round and a hiring-manager conversation are scheduled. No evaluation round was named.", "buried_resume_line": "Improved answer accuracy by 40% after rebuilding the retrieval pipeline.", "planted_risk": "A headline number with no stated baseline, evaluation set, or measurement method behind it.", "expected_detection_terms": [ "baseline", "measurement", "evaluation" ], "expected_probes": [ "What was the baseline, and how was it measured?", "Who labelled the evaluation set, and how large was it?", "How would someone else reproduce that comparison?" ], "expected_exposure": { "headline": "A headline number with no measurement story behind it", "detail": "The strongest claim in your background is a percentage improvement. No baseline, evaluation set, or measurement method appears next to it, so the first follow-up is how it was measured — and an unsupported number costs more credibility than no number at all." }, "expected_propagation": { "scenario": "Defend your accuracy improvement to a skeptical interviewer: state the baseline, the evaluation set, the metric, and what the number does not cover.", "drill": "Reconstruct the measurement: write the eval set size, labelling method, held-out split, and the one confound you cannot rule out.", "move": "Attach a baseline, an evaluation set, and a stated limitation to your headline metric before the first call." }, "confidence_expectations": { "confirmed_rounds": [ "AI system design", "Hiring manager conversation" ], "inferred_rounds": [ { "name": "Evaluation and reliability probe", "confidence": "likely" }, { "name": "Standalone evals round", "confidence": "unknown" } ] } }, { "id": "depth-without-production", "risk_class": "Strong technical depth, unclear production deployment", "role": "LLM Engineer, platform (synthetic)", "recruiter_notes": "Fictional recruiter notes: a coding round and a system design round are confirmed. Nothing was said about on-call or operations.", "buried_resume_line": "Built and benchmarked three fine-tuned summarisation models; the internal pilot ran for two months.", "planted_risk": "Deep modelling work with no evidence that anything reached and survived production traffic.", "expected_detection_terms": [ "production", "pilot", "rollback" ], "expected_probes": [ "Did this serve real traffic, and for how long?", "What broke in production, and how did you find out?", "Who owned it after you shipped it?" ], "expected_exposure": { "headline": "Depth is clear; production ownership is not", "detail": "Your background shows model work and benchmarks but stops at a pilot. A platform loop presses on what happened after deployment: traffic, incidents, rollback, ownership. Saying a pilot was a pilot is safer than implying it was production." }, "expected_propagation": { "scenario": "Take one pilot to production out loud: rollout, monitoring, rollback criteria, and the first incident you would expect.", "drill": "Write the release gate you would have needed: metrics watched, thresholds, kill switch, and who is paged.", "move": "Label the pilot honestly and prepare the production path you would run today." }, "confidence_expectations": { "confirmed_rounds": [ "Coding round", "System design round" ], "inferred_rounds": [ { "name": "Production operations probe", "confidence": "likely" }, { "name": "On-call / incident round", "confidence": "unknown" } ] } }, { "id": "strong-resume-unknown-coding-round", "risk_class": "Strong resume, unknown coding round", "role": "Senior Machine Learning Engineer (synthetic)", "recruiter_notes": "Fictional recruiter notes: a hiring-manager screen is scheduled. The recruiter did not say whether there is a coding exercise or what tools are allowed.", "buried_resume_line": "Eight years shipping ML systems; most recent coding was review and design, not implementation.", "planted_risk": "Seniority has moved the candidate away from hands-on implementation while the coding format stays unknown.", "expected_detection_terms": [ "coding", "unknown", "hands-on" ], "expected_probes": [ "Implement a small data or tool workflow live.", "Read unfamiliar code and change its behaviour.", "Explain your tests before you run them." ], "expected_exposure": { "headline": "Hands-on coding is the untested part of a strong record", "detail": "The record is senior and credible, but recent work reads as review and design. Whether there is a coding exercise is unknown — the recruiter did not say — so the honest move is to ask, and to warm up implementation either way." }, "expected_propagation": { "scenario": "Implement a small retrieval utility while narrating trade-offs, then extend it under a changed requirement.", "drill": "Time-boxed implementation with tests written first, in the language you would pick on the call.", "move": "Ask the recruiter for the coding format and AI-tool policy, then warm up implementation this week." }, "confidence_expectations": { "confirmed_rounds": [ "Hiring manager screen" ], "inferred_rounds": [ { "name": "Practical coding round", "confidence": "unknown" }, { "name": "Code reading exercise", "confidence": "likely" } ] } } ] }