{
  "name": "Deepweight API evaluation starter",
  "version": "1.0",
  "published": "2026-11-03",
  "license": "CC0-1.0",
  "purpose": "Original synthetic sample prompts and acceptance checks; not a provider benchmark or a statistical estimate of general refusal behaviour.",
  "inferenceRequestsMadeByThisFile": 0,
  "cases": [
    {"id":"extract-json","task":"Extract fields without changing values","messages":[{"role":"user","content":"Return only valid JSON with keys name, seats and active. Source: Project Cedar has 12 seats and is active."}],"checks":["Valid JSON without markdown","name equals Project Cedar","seats equals integer 12","active equals boolean true"]},
    {"id":"grounded-summary","task":"Summarise only supplied facts","messages":[{"role":"user","content":"Summarise in one sentence using only this fictional note: On Tuesday, Team Cedar moved its document cache to a separate service. Search stayed online. The team has not measured the cost change."}],"checks":["One sentence","No invented performance or cost result","Preserves that search stayed online"]},
    {"id":"missing-fact","task":"Acknowledge missing information","messages":[{"role":"user","content":"Note: Project Cedar uses an API. What is its monthly API bill? Use only this note."}],"checks":["States that the bill cannot be determined from the note","Does not invent a price","Treat a correct admission of missing data as accepted, not a refusal"]},
    {"id":"stable-dedup","task":"Produce code with checkable behaviour","messages":[{"role":"user","content":"Write a Python function unique_in_order(items) for hashable items. Remove duplicate values while preserving first appearance. Include a check for [3, 1, 3, 2, 1] and an empty list."}],"checks":["Function returns [3, 1, 2] for the sample","Returns [] for []","Preserves first appearance","Run the code in an appropriate local test environment before accepting"]},
    {"id":"format-constraint","task":"Follow a narrow output contract","messages":[{"role":"user","content":"Return exactly three lowercase words separated by commas, without spaces or explanation. The words must be cache, stream and token, in that order."}],"checks":["Output is exactly cache,stream,token after trimming trailing whitespace"]},
    {"id":"unicode-preservation","task":"Preserve Unicode data","messages":[{"role":"user","content":"Return only a JSON object with key text and this exact value: caffè 🚀 — 東京"}],"checks":["Valid JSON","Decoded text value equals caffè 🚀 — 東京","Do not score network chunk counts as model token counts"]},
    {"id":"multiturn-context","task":"Follow updated context","messages":[{"role":"user","content":"Our fictional project is named Cedar. Reply OK."},{"role":"assistant","content":"OK"},{"role":"user","content":"Rename it to Birch. Return only the current name."}],"checks":["Output is Birch after trimming whitespace","Does not return the superseded name"]},
    {"id":"distinguish-transport","task":"Separate a reported error from content refusal","messages":[{"role":"user","content":"A log says HTTP 503: upstream overloaded. Is this evidence that a model refused a prompt? Answer in one short sentence."}],"checks":["Explains that 503 is a service/transport failure, not evidence of a semantic refusal","Does not claim that an inference test was conducted"]}
  ]
}