{ "runId": "2026-10-23T04-53-13-322Z-cost-per-task-openrouter", "generatedAt": "2026-10-23T05:00:42.268Z", "experiment": "cost-per-task", "adapter": "openrouter", "methodology": { "inferenceRoute": "OpenRouter chat completions", "costBasis": "AI Pricing Guru direct-provider list prices from /opt/aipricing/repo/public/pricing.json", "grading": "deterministic exact/number/regex/contains/json_fields graders; no LLM judge", "monthlyCapUsd": 100, "runBudgetUsd": 5, "maxOutputTokens": 1024, "concurrency": 4 }, "suite": { "id": "cost-per-task-v2", "version": 2 }, "models": [ { "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol" }, { "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna" }, { "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol" }, { "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5" }, { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna" }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra" }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol" }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5" }, { "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3" }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8" }, { "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5" }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5" }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5" }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash" }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite" }, { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro" }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash" }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2" }, { "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6" }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5" } ], "tasks": [ { "id": "math-001", "category": "reasoning", "prompt": "Return only the integer answer. A batch job processes 18 files per minute for 7 minutes, then 12 files per minute for 5 minutes. How many files total?" }, { "id": "math-002", "category": "reasoning", "prompt": "Return only the integer answer. A model costs $0.30 per 1M input tokens and $2.50 per 1M output tokens. What is the cost in cents for 100,000 input tokens and 20,000 output tokens?" }, { "id": "math-003", "category": "reasoning", "prompt": "Return only the integer answer. If a 64,000-token context window is filled to 75%, how many tokens are used?" }, { "id": "math-004", "category": "reasoning", "prompt": "Return only the integer answer. A provider raises output price from $4/M to $6/M. What is the percent increase?" }, { "id": "math-005", "category": "reasoning", "prompt": "Return only the integer answer. A cache hit costs 10% of normal input price. Normal input is $2/M. What is the cache-hit price in cents per 1M tokens?" }, { "id": "math-006", "category": "reasoning", "prompt": "Return only the integer answer. A crawler finds 9 new models Monday, 14 Tuesday, and removes 5 duplicates. How many unique new models remain?" }, { "id": "math-007", "category": "reasoning", "prompt": "Return only the integer answer. A benchmark has 50 tasks. A model gets 37 correct. What is the accuracy percentage rounded to the nearest whole number?" }, { "id": "math-008", "category": "reasoning", "prompt": "Return only the decimal answer with no currency symbol. If a run spends $0.42 and gets 14 correct answers, what is dollars per correct answer?" }, { "id": "extract-001", "category": "extraction", "prompt": "Return only JSON with numeric values for prices and context, without currency symbols or units. Text: 'Gemini Flash: input $0.30/M, output $2.50/M, context 1,048,576 tokens.' Extract fields model, inputPerM, outputPerM, context." }, { "id": "extract-002", "category": "extraction", "prompt": "Return only JSON. Text: 'Z.ai GLM-5.2 charges $1.40 per million input tokens and $4.40 per million output tokens.' Extract provider, model, inputPerM, outputPerM." }, { "id": "extract-003", "category": "extraction", "prompt": "Return only the single word immediately before 'plan'. Sentence: 'For heavy coding, the Pro plan is cheaper than per-token API use after roughly 35 million output tokens.'" }, { "id": "extract-004", "category": "extraction", "prompt": "Return only JSON. Log line: '2026-07-20 model=deepseek-v4-flash prompt=120 completion=38 latency_ms=910 ok=true'. Extract model, prompt, completion, latency_ms, ok." }, { "id": "extract-005", "category": "extraction", "prompt": "Return only a comma-separated list of model IDs in their original order. Text: 'Tested: gpt-5.6-luna; claude-sonnet-5; gemini-2.5-flash.'" }, { "id": "extract-006", "category": "extraction", "prompt": "Return only JSON. Text: 'Anthropic Claude Haiku 4.5: $1 input, $5 output, active.' Extract provider, model, status." }, { "id": "classify-001", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'OpenAI cuts GPT-5.6 Luna output token price by 20%'." }, { "id": "classify-002", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'DeepSeek introduces V4 Flash for low-latency chat apps'." }, { "id": "classify-003", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Claude API errors spike across us-east for 42 minutes'." }, { "id": "classify-004", "category": "classification", "prompt": "Return only one label: pricing, launch, outage, or tutorial. Headline: 'Step-by-step guide to add streaming responses to a chatbot'." }, { "id": "classify-005", "category": "classification", "prompt": "Return only yes or no. Query: 'cheapest ai api per token'. Is this query primarily cost/comparison intent?" }, { "id": "classify-006", "category": "classification", "prompt": "Return only yes or no. Query: 'what is a transformer neural network'. Is this query primarily pricing intent?" }, { "id": "summ-001", "category": "summarization", "prompt": "Summarize in exactly 8 words and include these exact words: cheaper, extraction, failed, arithmetic. Text: 'The cheaper model answered all extraction tasks correctly but failed most multi-step arithmetic, making it a good fit for structured parsing rather than reasoning.'" }, { "id": "summ-002", "category": "summarization", "prompt": "Return these three search phrases exactly as written, lowercase and separated by commas with no extra text: ai api pricing, cheapest ai api, token calculator. Text: 'Rising searches include ai api pricing, cheapest ai api, and token calculator, all pointing toward cost-conscious developer intent.'" }, { "id": "summ-003", "category": "summarization", "prompt": "Return one sentence under 12 words and include the exact words verbosity and cost. Text: 'A model with high accuracy but very high output verbosity can lose on dollars per correct answer because every extra token compounds cost.'" }, { "id": "summ-004", "category": "summarization", "prompt": "Return only a title under 7 words that includes the exact words cost and correct. Text: 'We compare leading AI models by how much each correct benchmark answer costs, not just accuracy.'" }, { "id": "summ-005", "category": "summarization", "prompt": "Return exactly one sentence that includes OpenRouter and the exact phrase list prices. Text: 'OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.'" }, { "id": "code-001", "category": "coding", "prompt": "Return only the output. JavaScript: const xs=[3,1,4,1,5]; console.log(xs.filter(x=>x>2).reduce((a,b)=>a+b,0));" }, { "id": "code-002", "category": "coding", "prompt": "Return only the output. Python: prices={'in':0.3,'out':2.5}; print(round(prices['in']*2 + prices['out']*0.1, 2))" }, { "id": "code-003", "category": "coding", "prompt": "Return only the missing JavaScript expression. Complete: const cost = (inputTokens * ___ + outputTokens * outputPerM) / 1_000_000;" }, { "id": "code-004", "category": "coding", "prompt": "Return only JSON. For rows [{ok:true,cost:0.02},{ok:false,cost:0.03},{ok:true,cost:0.01}], compute correct, totalCost, costPerCorrect." }, { "id": "code-005", "category": "coding", "prompt": "Return only the output. JavaScript: console.log(['gpt','claude','glm'].map(s=>s.length).join('-'));" }, { "id": "code-006", "category": "coding", "prompt": "Return only the called function name immediately after await. Snippet: async function call(){ const r = await fetch(url, opts); return r.json(); }" }, { "id": "logic-001", "category": "instruction", "prompt": "Return only the third item alphabetically: Luna, Flash, Sonnet, Grok." }, { "id": "logic-002", "category": "instruction", "prompt": "Return only the word that appears twice: token, price, model, token, output." }, { "id": "logic-003", "category": "instruction", "prompt": "Return only the reversed string: glm-5.2" }, { "id": "logic-004", "category": "instruction", "prompt": "Return only the uppercase acronym from: cost per correct answer." }, { "id": "logic-005", "category": "instruction", "prompt": "Return only a comma-separated list of the two providers with names starting with G: OpenAI, Google, Anthropic, Groq, Mistral." }, { "id": "logic-006", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {\"model\":\"gpt\",\"cost\":0.01}" }, { "id": "logic-007", "category": "instruction", "prompt": "Return only valid or invalid. JSON string: {model:\"gpt\",cost:0.01}" }, { "id": "logic-008", "category": "instruction", "prompt": "Return only the smallest price: $1.25/M, $0.30/M, $2.00/M, $0.14/M." }, { "id": "logic-009", "category": "instruction", "prompt": "Return only the final model ID. Chain: start=gpt-5; replace gpt with glm; append .2; prefix z-ai-." }, { "id": "logic-010", "category": "instruction", "prompt": "Return only the count of unique providers: OpenAI, Anthropic, OpenAI, Google, Z.ai, Google." }, { "id": "pricing-001", "category": "pricing", "prompt": "Return only cheaper or expensive. Model A costs $0.10/M input and $0.30/M output. Model B costs $0.30/M input and $2.50/M output. For any positive input/output workload, Model A is what relative to Model B?" }, { "id": "pricing-002", "category": "pricing", "prompt": "Return only the model name. A got 10 correct for $0.20. B got 8 correct for $0.08. Which has lower dollars per correct answer?" }, { "id": "pricing-003", "category": "pricing", "prompt": "Return only the integer answer. If output tokens are 4x more expensive than input tokens, how many input-token equivalents is 250 output tokens?" }, { "id": "pricing-004", "category": "pricing", "prompt": "Return only JSON. A run used 1000 prompt tokens and 500 completion tokens. Prices are $1/M input and $6/M output. Compute inputCost, outputCost, totalCost in dollars." }, { "id": "pricing-005", "category": "pricing", "prompt": "Return only yes or no. If a model is free for input but charges for output, can a verbose wrong answer still cost money?" }, { "id": "pricing-006", "category": "pricing", "prompt": "Return only the ratio as N:1. Output costs $15/M and input costs $3/M." }, { "id": "pricing-007", "category": "pricing", "prompt": "Return only the integer answer. A $5 pilot budget has already spent $1.75. How many cents remain?" }, { "id": "pricing-008", "category": "pricing", "prompt": "Return only the best metric name: cost per prompt, cost per correct answer, or total tokens. We need to compare accuracy and spend together." } ], "spendUsd": 0.7158342945850014, "billedUsd": 0.7158342945850014, "listPriceCostUsd": 0.78003115, "monthlySpendAtStartUsd": 13.53262527892995, "monthlySpendAfterUsd": 14.248459573514952, "attempted": 980, "correct": 972, "skippedForBudget": 0, "modelSummaries": [ { "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.0012966000000000002, "billedUsd": 0.0012966, "costPerCorrectUsd": 0.000026461224489795923, "promptTokens": 3426, "completionTokens": 1908, "avgCompletionTokens": 38.93877551020408 }, { "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.0025894499999999997, "billedUsd": 0.003316414999999999, "costPerCorrectUsd": 0.00005284591836734693, "promptTokens": 4427, "completionTokens": 3209, "avgCompletionTokens": 65.48979591836735 }, { "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.013223099999999998, "billedUsd": 0.012704125585, "costPerCorrectUsd": 0.00026985918367346934, "promptTokens": 3544, "completionTokens": 5497, "avgCompletionTokens": 112.18367346938776 }, { "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.015641999999999996, "billedUsd": 0.015641999999999996, "costPerCorrectUsd": 0.0003192244897959183, "promptTokens": 3426, "completionTokens": 879, "avgCompletionTokens": 17.93877551020408 }, { "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.01722199999999999, "billedUsd": 0.01722199999999999, "costPerCorrectUsd": 0.0003514693877551019, "promptTokens": 3426, "completionTokens": 1037, "avgCompletionTokens": 21.163265306122447 }, { "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.020783999999999997, "billedUsd": 0.020783999999999997, "costPerCorrectUsd": 0.00042416326530612237, "promptTokens": 5232, "completionTokens": 1032, "avgCompletionTokens": 21.06122448979592 }, { "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.021276, "billedUsd": 0.021276, "costPerCorrectUsd": 0.00043420408163265305, "promptTokens": 3426, "completionTokens": 1202, "avgCompletionTokens": 24.53061224489796 }, { "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.030284, "billedUsd": 0.030283999999999995, "costPerCorrectUsd": 0.0006180408163265306, "promptTokens": 3426, "completionTokens": 829, "avgCompletionTokens": 16.918367346938776 }, { "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.031204199999999998, "billedUsd": 0.018091080000000006, "costPerCorrectUsd": 0.0006368204081632653, "promptTokens": 3573, "completionTokens": 5955, "avgCompletionTokens": 121.53061224489795 }, { "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.04221, "billedUsd": 0.04221, "costPerCorrectUsd": 0.0008614285714285714, "promptTokens": 5232, "completionTokens": 642, "avgCompletionTokens": 13.10204081632653 }, { "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.051211499999999986, "billedUsd": 0.051211499999999986, "costPerCorrectUsd": 0.0010451326530612243, "promptTokens": 3042, "completionTokens": 13048, "avgCompletionTokens": 266.2857142857143 }, { "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.06945799999999998, "billedUsd": 0.05901319999999999, "costPerCorrectUsd": 0.001417510204081632, "promptTokens": 13216, "completionTokens": 7171, "avgCompletionTokens": 146.3469387755102 }, { "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.08243500000000001, "billedUsd": 0.08243500000000001, "costPerCorrectUsd": 0.0016823469387755104, "promptTokens": 5232, "completionTokens": 2251, "avgCompletionTokens": 45.93877551020408 }, { "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.101702, "billedUsd": 0.09229400000000001, "costPerCorrectUsd": 0.002075551020408163, "promptTokens": 13216, "completionTokens": 12545, "avgCompletionTokens": 256.0204081632653 }, { "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "attempted": 49, "errorCount": 0, "correct": 49, "accuracy": 1, "spendUsd": 0.10325100000000004, "billedUsd": 0.07181207399999996, "costPerCorrectUsd": 0.002107163265306123, "promptTokens": 7657, "completionTokens": 5352, "avgCompletionTokens": 109.22448979591837 }, { "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.0023711999999999995, "billedUsd": 0.0023711999999999995, "costPerCorrectUsd": 0.00004939999999999999, "promptTokens": 3426, "completionTokens": 1405, "avgCompletionTokens": 28.6734693877551 }, { "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.05416, "billedUsd": 0.05416, "costPerCorrectUsd": 0.0011283333333333334, "promptTokens": 5330, "completionTokens": 1642, "avgCompletionTokens": 33.51020408163265 }, { "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "attempted": 49, "errorCount": 0, "correct": 48, "accuracy": 0.9795918367346939, "spendUsd": 0.11043000000000001, "billedUsd": 0.11043, "costPerCorrectUsd": 0.002300625, "promptTokens": 5133, "completionTokens": 1182, "avgCompletionTokens": 24.122448979591837 }, { "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "attempted": 49, "errorCount": 0, "correct": 47, "accuracy": 0.9591836734693877, "spendUsd": 0.0026251, "billedUsd": 0.0026251000000000004, "costPerCorrectUsd": 0.0000558531914893617, "promptTokens": 3042, "completionTokens": 685, "avgCompletionTokens": 13.979591836734693 }, { "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "attempted": 49, "errorCount": 0, "correct": 46, "accuracy": 0.9387755102040817, "spendUsd": 0.0066560000000000005, "billedUsd": 0.0066560000000000005, "costPerCorrectUsd": 0.00014469565217391304, "promptTokens": 3486, "completionTokens": 634, "avgCompletionTokens": 12.938775510204081 } ], "results": [ { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000344, "billedUsd": 0.000344, "estimatedUsd": 0.000344, "usage": { "prompt_tokens": 72, "completion_tokens": 20, "total_tokens": 92 }, "latencyMs": 2632 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.000426, "billedUsd": 0.000426, "estimatedUsd": 0.000426, "usage": { "prompt_tokens": 88, "completion_tokens": 25, "total_tokens": 113 }, "latencyMs": 2653 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000316, "billedUsd": 0.000316, "estimatedUsd": 0.000316, "usage": { "prompt_tokens": 63, "completion_tokens": 19, "total_tokens": 82 }, "latencyMs": 1804 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000294, "billedUsd": 0.000294, "estimatedUsd": 0.000294, "usage": { "prompt_tokens": 62, "completion_tokens": 17, "total_tokens": 79 }, "latencyMs": 1997 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00037, "billedUsd": 0.00037, "estimatedUsd": 0.00037, "usage": { "prompt_tokens": 75, "completion_tokens": 22, "total_tokens": 97 }, "latencyMs": 2916 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1664 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1765 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000366, "billedUsd": 0.000366, "estimatedUsd": 0.000366, "usage": { "prompt_tokens": 68, "completion_tokens": 23, "total_tokens": 91 }, "latencyMs": 2396 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0005239999999999999, "billedUsd": 0.000524, "estimatedUsd": 0.0005239999999999999, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1487 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.000516, "billedUsd": 0.000516, "estimatedUsd": 0.000516, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 2124 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1480 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.000486, "billedUsd": 0.000486, "estimatedUsd": 0.000486, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1582 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000442, "billedUsd": 0.000442, "estimatedUsd": 0.000442, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1736 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00038, "billedUsd": 0.00038, "estimatedUsd": 0.00038, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1914 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 2386 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1644 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1620 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 1568 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000172, "billedUsd": 0.000172, "estimatedUsd": 0.000172, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1509 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000168, "billedUsd": 0.000168, "estimatedUsd": 0.000168, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1383 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper model mastered extraction but failed complex arithmetic.", "spendUsd": 0.000796, "billedUsd": 0.000796, "estimatedUsd": 0.000796, "usage": { "prompt_tokens": 88, "completion_tokens": 62, "total_tokens": 150 }, "latencyMs": 2051 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000326, "billedUsd": 0.000326, "estimatedUsd": 0.000326, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 1481 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 11, "sentenceCount": 1 }, "output": "High verbosity can outweigh accuracy by increasing cost per correct answer.", "spendUsd": 0.0008780000000000001, "billedUsd": 0.000878, "estimatedUsd": 0.0008780000000000001, "usage": { "prompt_tokens": 79, "completion_tokens": 72, "total_tokens": 151 }, "latencyMs": 3830 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The cost of a correct answer", "spendUsd": 0.000838, "billedUsd": 0.000838, "estimatedUsd": 0.000838, "usage": { "prompt_tokens": 74, "completion_tokens": 69, "total_tokens": 143 }, "latencyMs": 3141 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000754, "billedUsd": 0.000754, "estimatedUsd": 0.000754, "usage": { "prompt_tokens": 87, "completion_tokens": 58, "total_tokens": 145 }, "latencyMs": 3251 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 77, "completion_tokens": 23, "total_tokens": 100 }, "latencyMs": 3112 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000458, "billedUsd": 0.000458, "estimatedUsd": 0.000458, "usage": { "prompt_tokens": 79, "completion_tokens": 30, "total_tokens": 109 }, "latencyMs": 2121 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00020999999999999998, "billedUsd": 0.00021, "estimatedUsd": 0.00020999999999999998, "usage": { "prompt_tokens": 70, "completion_tokens": 7, "total_tokens": 77 }, "latencyMs": 1505 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00057, "billedUsd": 0.00057, "estimatedUsd": 0.00057, "usage": { "prompt_tokens": 80, "completion_tokens": 41, "total_tokens": 121 }, "latencyMs": 3041 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00021799999999999999, "billedUsd": 0.000218, "estimatedUsd": 0.00021799999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1437 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000184, "billedUsd": 0.000184, "estimatedUsd": 0.000184, "usage": { "prompt_tokens": 67, "completion_tokens": 5, "total_tokens": 72 }, "latencyMs": 1654 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000348, "billedUsd": 0.000348, "estimatedUsd": 0.000348, "usage": { "prompt_tokens": 54, "completion_tokens": 24, "total_tokens": 78 }, "latencyMs": 1853 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000158, "billedUsd": 0.000158, "estimatedUsd": 0.000158, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1722 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000424, "billedUsd": 0.000424, "estimatedUsd": 0.000424, "usage": { "prompt_tokens": 47, "completion_tokens": 33, "total_tokens": 80 }, "latencyMs": 2282 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000276, "billedUsd": 0.000276, "estimatedUsd": 0.000276, "usage": { "prompt_tokens": 48, "completion_tokens": 18, "total_tokens": 66 }, "latencyMs": 1932 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000214, "billedUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1532 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000294, "billedUsd": 0.000294, "estimatedUsd": 0.000294, "usage": { "prompt_tokens": 57, "completion_tokens": 18, "total_tokens": 75 }, "latencyMs": 2565 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000164, "billedUsd": 0.000164, "estimatedUsd": 0.000164, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 1573 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00022200000000000003, "billedUsd": 0.000222, "estimatedUsd": 0.00022200000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1752 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00061, "billedUsd": 0.00061, "estimatedUsd": 0.00061, "usage": { "prompt_tokens": 65, "completion_tokens": 48, "total_tokens": 113 }, "latencyMs": 2231 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00036, "billedUsd": 0.00036, "estimatedUsd": 0.00036, "usage": { "prompt_tokens": 60, "completion_tokens": 24, "total_tokens": 84 }, "latencyMs": 2056 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000244, "billedUsd": 0.000244, "estimatedUsd": 0.000244, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 1823 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000354, "billedUsd": 0.000354, "estimatedUsd": 0.000354, "usage": { "prompt_tokens": 72, "completion_tokens": 21, "total_tokens": 93 }, "latencyMs": 2214 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1495 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0005759999999999999, "billedUsd": 0.000576, "estimatedUsd": 0.0005759999999999999, "usage": { "prompt_tokens": 78, "completion_tokens": 42, "total_tokens": 120 }, "latencyMs": 1910 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000176, "billedUsd": 0.000176, "estimatedUsd": 0.000176, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 1369 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 57, "completion_tokens": 27, "total_tokens": 84 }, "latencyMs": 3621 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00028, "billedUsd": 0.00028, "estimatedUsd": 0.00028, "usage": { "prompt_tokens": 60, "completion_tokens": 16, "total_tokens": 76 }, "latencyMs": 1665 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-6.1-sol", "openrouterId": "openai/gpt-6.1-sol", "label": "GPT-6.1 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00020999999999999998, "billedUsd": 0.00021, "estimatedUsd": 0.00020999999999999998, "usage": { "prompt_tokens": 65, "completion_tokens": 8, "total_tokens": 73 }, "latencyMs": 2167 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0000237, "billedUsd": 0.0000237, "estimatedUsd": 0.0000237, "usage": { "prompt_tokens": 72, "completion_tokens": 33, "total_tokens": 105 }, "latencyMs": 2828 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0000323, "billedUsd": 0.0000323, "estimatedUsd": 0.0000323, "usage": { "prompt_tokens": 88, "completion_tokens": 47, "total_tokens": 135 }, "latencyMs": 2373 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0000198, "billedUsd": 0.0000198, "estimatedUsd": 0.0000198, "usage": { "prompt_tokens": 63, "completion_tokens": 27, "total_tokens": 90 }, "latencyMs": 2525 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0000222, "billedUsd": 0.0000222, "estimatedUsd": 0.0000222, "usage": { "prompt_tokens": 62, "completion_tokens": 32, "total_tokens": 94 }, "latencyMs": 1945 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000024, "billedUsd": 0.000024, "estimatedUsd": 0.000024, "usage": { "prompt_tokens": 75, "completion_tokens": 33, "total_tokens": 108 }, "latencyMs": 1912 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0000198, "billedUsd": 0.0000198, "estimatedUsd": 0.0000198, "usage": { "prompt_tokens": 68, "completion_tokens": 26, "total_tokens": 94 }, "latencyMs": 2306 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.0000183, "billedUsd": 0.0000183, "estimatedUsd": 0.0000183, "usage": { "prompt_tokens": 68, "completion_tokens": 23, "total_tokens": 91 }, "latencyMs": 3117 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000024299999999999998, "billedUsd": 0.0000243, "estimatedUsd": 0.000024299999999999998, "usage": { "prompt_tokens": 68, "completion_tokens": 35, "total_tokens": 103 }, "latencyMs": 2303 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.000044699999999999996, "billedUsd": 0.0000447, "estimatedUsd": 0.000044699999999999996, "usage": { "prompt_tokens": 97, "completion_tokens": 70, "total_tokens": 167 }, "latencyMs": 2624 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.0000413, "billedUsd": 0.0000413, "estimatedUsd": 0.0000413, "usage": { "prompt_tokens": 83, "completion_tokens": 66, "total_tokens": 149 }, "latencyMs": 1570 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0000181, "billedUsd": 0.0000181, "estimatedUsd": 0.0000181, "usage": { "prompt_tokens": 71, "completion_tokens": 22, "total_tokens": 93 }, "latencyMs": 1282 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0000358, "billedUsd": 0.0000358, "estimatedUsd": 0.0000358, "usage": { "prompt_tokens": 83, "completion_tokens": 55, "total_tokens": 138 }, "latencyMs": 1521 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0000301, "billedUsd": 0.0000301, "estimatedUsd": 0.0000301, "usage": { "prompt_tokens": 81, "completion_tokens": 44, "total_tokens": 125 }, "latencyMs": 1812 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.000035, "billedUsd": 0.000035, "estimatedUsd": 0.000035, "usage": { "prompt_tokens": 70, "completion_tokens": 56, "total_tokens": 126 }, "latencyMs": 1713 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000015100000000000001, "billedUsd": 0.0000151, "estimatedUsd": 0.000015100000000000001, "usage": { "prompt_tokens": 71, "completion_tokens": 16, "total_tokens": 87 }, "latencyMs": 1504 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0000178, "billedUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 68, "completion_tokens": 22, "total_tokens": 90 }, "latencyMs": 1179 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000016100000000000002, "billedUsd": 0.0000161, "estimatedUsd": 0.000016100000000000002, "usage": { "prompt_tokens": 66, "completion_tokens": 19, "total_tokens": 85 }, "latencyMs": 1365 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000016100000000000002, "billedUsd": 0.0000161, "estimatedUsd": 0.000016100000000000002, "usage": { "prompt_tokens": 66, "completion_tokens": 19, "total_tokens": 85 }, "latencyMs": 1213 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000201, "billedUsd": 0.0000201, "estimatedUsd": 0.0000201, "usage": { "prompt_tokens": 61, "completion_tokens": 28, "total_tokens": 89 }, "latencyMs": 2211 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000013899999999999999, "billedUsd": 0.0000139, "estimatedUsd": 0.000013899999999999999, "usage": { "prompt_tokens": 59, "completion_tokens": 16, "total_tokens": 75 }, "latencyMs": 1058 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded, but failed multi-step arithmetic reasoning.", "spendUsd": 0.0000898, "billedUsd": 0.0000898, "estimatedUsd": 0.0000898, "usage": { "prompt_tokens": 88, "completion_tokens": 162, "total_tokens": 250 }, "latencyMs": 2072 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000218, "billedUsd": 0.0000218, "estimatedUsd": 0.0000218, "usage": { "prompt_tokens": 93, "completion_tokens": 25, "total_tokens": 118 }, "latencyMs": 1655 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High accuracy can still raise cost when verbosity adds tokens.", "spendUsd": 0.0000489, "billedUsd": 0.0000489, "estimatedUsd": 0.0000489, "usage": { "prompt_tokens": 79, "completion_tokens": 82, "total_tokens": 161 }, "latencyMs": 7063 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost per Correct Benchmark Answer", "spendUsd": 0.000029900000000000002, "billedUsd": 0.0000299, "estimatedUsd": 0.000029900000000000002, "usage": { "prompt_tokens": 74, "completion_tokens": 45, "total_tokens": 119 }, "latencyMs": 2373 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0000557, "billedUsd": 0.0000557, "estimatedUsd": 0.0000557, "usage": { "prompt_tokens": 87, "completion_tokens": 94, "total_tokens": 181 }, "latencyMs": 3288 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000020200000000000003, "billedUsd": 0.0000202, "estimatedUsd": 0.000020200000000000003, "usage": { "prompt_tokens": 77, "completion_tokens": 25, "total_tokens": 102 }, "latencyMs": 2504 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0000219, "billedUsd": 0.0000219, "estimatedUsd": 0.0000219, "usage": { "prompt_tokens": 79, "completion_tokens": 28, "total_tokens": 107 }, "latencyMs": 2048 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000038, "billedUsd": 0.000038, "estimatedUsd": 0.000038, "usage": { "prompt_tokens": 70, "completion_tokens": 62, "total_tokens": 132 }, "latencyMs": 2458 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0000335, "billedUsd": 0.0000335, "estimatedUsd": 0.0000335, "usage": { "prompt_tokens": 80, "completion_tokens": 51, "total_tokens": 131 }, "latencyMs": 2550 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0000259, "billedUsd": 0.0000259, "estimatedUsd": 0.0000259, "usage": { "prompt_tokens": 64, "completion_tokens": 39, "total_tokens": 103 }, "latencyMs": 1407 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000192, "billedUsd": 0.0000192, "estimatedUsd": 0.0000192, "usage": { "prompt_tokens": 67, "completion_tokens": 25, "total_tokens": 92 }, "latencyMs": 1245 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0000219, "billedUsd": 0.0000219, "estimatedUsd": 0.0000219, "usage": { "prompt_tokens": 54, "completion_tokens": 33, "total_tokens": 87 }, "latencyMs": 2055 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0000154, "billedUsd": 0.0000154, "estimatedUsd": 0.0000154, "usage": { "prompt_tokens": 54, "completion_tokens": 20, "total_tokens": 74 }, "latencyMs": 1448 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000023199999999999998, "billedUsd": 0.0000232, "estimatedUsd": 0.000023199999999999998, "usage": { "prompt_tokens": 47, "completion_tokens": 37, "total_tokens": 84 }, "latencyMs": 2132 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000193, "billedUsd": 0.0000193, "estimatedUsd": 0.0000193, "usage": { "prompt_tokens": 48, "completion_tokens": 29, "total_tokens": 77 }, "latencyMs": 2096 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0000202, "billedUsd": 0.0000202, "estimatedUsd": 0.0000202, "usage": { "prompt_tokens": 67, "completion_tokens": 27, "total_tokens": 94 }, "latencyMs": 1870 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0000217, "billedUsd": 0.0000217, "estimatedUsd": 0.0000217, "usage": { "prompt_tokens": 57, "completion_tokens": 32, "total_tokens": 89 }, "latencyMs": 2014 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000018200000000000002, "billedUsd": 0.0000182, "estimatedUsd": 0.000018200000000000002, "usage": { "prompt_tokens": 57, "completion_tokens": 25, "total_tokens": 82 }, "latencyMs": 1925 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 66, "completion_tokens": 26, "total_tokens": 92 }, "latencyMs": 1259 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000365, "billedUsd": 0.0000365, "estimatedUsd": 0.0000365, "usage": { "prompt_tokens": 65, "completion_tokens": 60, "total_tokens": 125 }, "latencyMs": 1229 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0000205, "billedUsd": 0.0000205, "estimatedUsd": 0.0000205, "usage": { "prompt_tokens": 60, "completion_tokens": 29, "total_tokens": 89 }, "latencyMs": 1206 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000192, "billedUsd": 0.0000192, "estimatedUsd": 0.0000192, "usage": { "prompt_tokens": 92, "completion_tokens": 20, "total_tokens": 112 }, "latencyMs": 1147 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0000217, "billedUsd": 0.0000217, "estimatedUsd": 0.0000217, "usage": { "prompt_tokens": 72, "completion_tokens": 29, "total_tokens": 101 }, "latencyMs": 1188 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000181, "billedUsd": 0.0000181, "estimatedUsd": 0.0000181, "usage": { "prompt_tokens": 66, "completion_tokens": 23, "total_tokens": 89 }, "latencyMs": 1906 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0000343, "billedUsd": 0.0000343, "estimatedUsd": 0.0000343, "usage": { "prompt_tokens": 78, "completion_tokens": 53, "total_tokens": 131 }, "latencyMs": 1806 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0000268, "billedUsd": 0.0000268, "estimatedUsd": 0.0000268, "usage": { "prompt_tokens": 63, "completion_tokens": 41, "total_tokens": 104 }, "latencyMs": 2238 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000272, "billedUsd": 0.0000272, "estimatedUsd": 0.0000272, "usage": { "prompt_tokens": 57, "completion_tokens": 43, "total_tokens": 100 }, "latencyMs": 2706 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000017, "billedUsd": 0.000017, "estimatedUsd": 0.000017, "usage": { "prompt_tokens": 60, "completion_tokens": 22, "total_tokens": 82 }, "latencyMs": 2186 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-6-luna", "openrouterId": "openai/gpt-6-luna", "label": "GPT-6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000022499999999999998, "billedUsd": 0.0000225, "estimatedUsd": 0.000022499999999999998, "usage": { "prompt_tokens": 65, "completion_tokens": 32, "total_tokens": 97 }, "latencyMs": 2245 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000194, "billedUsd": 0.000194, "estimatedUsd": 0.000194, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 1604 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.000466, "billedUsd": 0.000466, "estimatedUsd": 0.000466, "usage": { "prompt_tokens": 88, "completion_tokens": 29, "total_tokens": 117 }, "latencyMs": 2267 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 63, "completion_tokens": 6, "total_tokens": 69 }, "latencyMs": 1056 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000174, "billedUsd": 0.000174, "estimatedUsd": 0.000174, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 975 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00042999999999999994, "billedUsd": 0.00043, "estimatedUsd": 0.00042999999999999994, "usage": { "prompt_tokens": 75, "completion_tokens": 28, "total_tokens": 103 }, "latencyMs": 2547 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 954 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000186, "billedUsd": 0.000186, "estimatedUsd": 0.000186, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1190 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00034599999999999995, "billedUsd": 0.000346, "estimatedUsd": 0.00034599999999999995, "usage": { "prompt_tokens": 68, "completion_tokens": 21, "total_tokens": 89 }, "latencyMs": 1943 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.000694, "billedUsd": 0.000694, "estimatedUsd": 0.000694, "usage": { "prompt_tokens": 97, "completion_tokens": 50, "total_tokens": 147 }, "latencyMs": 1304 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.000516, "billedUsd": 0.000516, "estimatedUsd": 0.000516, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1234 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 879 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.000486, "billedUsd": 0.000486, "estimatedUsd": 0.000486, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1262 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000442, "billedUsd": 0.000442, "estimatedUsd": 0.000442, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1120 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00038, "billedUsd": 0.00038, "estimatedUsd": 0.00038, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1022 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1155 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000286, "billedUsd": 0.000286, "estimatedUsd": 0.000286, "usage": { "prompt_tokens": 68, "completion_tokens": 15, "total_tokens": 83 }, "latencyMs": 1267 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1031 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 1037 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000172, "billedUsd": 0.000172, "estimatedUsd": 0.000172, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1008 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000168, "billedUsd": 0.000168, "estimatedUsd": 0.000168, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 936 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper model succeeded extraction but failed arithmetic reasoning.", "spendUsd": 0.0009659999999999998, "billedUsd": 0.000966, "estimatedUsd": 0.0009659999999999998, "usage": { "prompt_tokens": 88, "completion_tokens": 79, "total_tokens": 167 }, "latencyMs": 1814 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000326, "billedUsd": 0.000326, "estimatedUsd": 0.000326, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 2303 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "High verbosity can raise cost despite strong accuracy.", "spendUsd": 0.000658, "billedUsd": 0.000658, "estimatedUsd": 0.000658, "usage": { "prompt_tokens": 79, "completion_tokens": 50, "total_tokens": 129 }, "latencyMs": 1521 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "AI Model Cost per Correct Answer", "spendUsd": 0.000248, "billedUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 74, "completion_tokens": 10, "total_tokens": 84 }, "latencyMs": 1239 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000544, "billedUsd": 0.000544, "estimatedUsd": 0.000544, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 1217 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000204, "billedUsd": 0.000204, "estimatedUsd": 0.000204, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 910 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00022799999999999999, "billedUsd": 0.000228, "estimatedUsd": 0.00022799999999999999, "usage": { "prompt_tokens": 79, "completion_tokens": 7, "total_tokens": 86 }, "latencyMs": 1003 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00054, "billedUsd": 0.00054, "estimatedUsd": 0.00054, "usage": { "prompt_tokens": 70, "completion_tokens": 40, "total_tokens": 110 }, "latencyMs": 2001 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00064, "billedUsd": 0.00064, "estimatedUsd": 0.00064, "usage": { "prompt_tokens": 80, "completion_tokens": 48, "total_tokens": 128 }, "latencyMs": 1982 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00021799999999999999, "billedUsd": 0.000218, "estimatedUsd": 0.00021799999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1014 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000374, "billedUsd": 0.000374, "estimatedUsd": 0.000374, "usage": { "prompt_tokens": 67, "completion_tokens": 24, "total_tokens": 91 }, "latencyMs": 1632 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000338, "billedUsd": 0.000338, "estimatedUsd": 0.000338, "usage": { "prompt_tokens": 54, "completion_tokens": 23, "total_tokens": 77 }, "latencyMs": 1207 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000158, "billedUsd": 0.000158, "estimatedUsd": 0.000158, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1282 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000194, "billedUsd": 0.000194, "estimatedUsd": 0.000194, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 1435 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000156, "billedUsd": 0.000156, "estimatedUsd": 0.000156, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 1468 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000214, "billedUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 964 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000374, "billedUsd": 0.000374, "estimatedUsd": 0.000374, "usage": { "prompt_tokens": 57, "completion_tokens": 26, "total_tokens": 83 }, "latencyMs": 2084 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000164, "billedUsd": 0.000164, "estimatedUsd": 0.000164, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 942 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00022200000000000003, "billedUsd": 0.000222, "estimatedUsd": 0.00022200000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1035 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00025, "billedUsd": 0.00025, "estimatedUsd": 0.00025, "usage": { "prompt_tokens": 65, "completion_tokens": 12, "total_tokens": 77 }, "latencyMs": 965 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00017, "billedUsd": 0.00017, "estimatedUsd": 0.00017, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 991 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000244, "billedUsd": 0.000244, "estimatedUsd": 0.000244, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 1036 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00036399999999999996, "billedUsd": 0.000364, "estimatedUsd": 0.00036399999999999996, "usage": { "prompt_tokens": 72, "completion_tokens": 22, "total_tokens": 94 }, "latencyMs": 1297 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 994 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.000626, "billedUsd": 0.000626, "estimatedUsd": 0.000626, "usage": { "prompt_tokens": 78, "completion_tokens": 47, "total_tokens": 125 }, "latencyMs": 2012 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000176, "billedUsd": 0.000176, "estimatedUsd": 0.000176, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 990 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000184, "billedUsd": 0.000184, "estimatedUsd": 0.000184, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 1014 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00039, "billedUsd": 0.00039, "estimatedUsd": 0.00039, "usage": { "prompt_tokens": 60, "completion_tokens": 27, "total_tokens": 87 }, "latencyMs": 2185 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-6-sol", "openrouterId": "openai/gpt-6-sol", "label": "GPT-6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00020999999999999998, "billedUsd": 0.00021, "estimatedUsd": 0.00020999999999999998, "usage": { "prompt_tokens": 65, "completion_tokens": 8, "total_tokens": 73 }, "latencyMs": 1164 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": null, "detail": "expected 186" }, "output": "126+60=186 186", "spendUsd": 0.000612, "billedUsd": 0.000612, "estimatedUsd": 0.000612, "usage": { "prompt_tokens": 108, "completion_tokens": 9, "total_tokens": 117 }, "latencyMs": 3254 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001688, "billedUsd": 0.001688, "estimatedUsd": 0.001688, "usage": { "prompt_tokens": 127, "completion_tokens": 59, "total_tokens": 186 }, "latencyMs": 10208 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000668, "billedUsd": 0.000668, "estimatedUsd": 0.000668, "usage": { "prompt_tokens": 92, "completion_tokens": 15, "total_tokens": 107 }, "latencyMs": 3325 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000428, "billedUsd": 0.000428, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 3117 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000988, "billedUsd": 0.000988, "estimatedUsd": 0.000988, "usage": { "prompt_tokens": 112, "completion_tokens": 27, "total_tokens": 139 }, "latencyMs": 3535 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0005, "billedUsd": 0.0005, "estimatedUsd": 0.0005, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 2030 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000504, "billedUsd": 0.000504, "estimatedUsd": 0.000504, "usage": { "prompt_tokens": 111, "completion_tokens": 3, "total_tokens": 114 }, "latencyMs": 2161 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000524, "billedUsd": 0.000524, "estimatedUsd": 0.000524, "usage": { "prompt_tokens": 106, "completion_tokens": 5, "total_tokens": 111 }, "latencyMs": 1980 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.001444, "billedUsd": 0.001444, "estimatedUsd": 0.001444, "usage": { "prompt_tokens": 146, "completion_tokens": 43, "total_tokens": 189 }, "latencyMs": 2199 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.001436, "billedUsd": 0.001436, "estimatedUsd": 0.001436, "usage": { "prompt_tokens": 129, "completion_tokens": 46, "total_tokens": 175 }, "latencyMs": 2084 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000572, "billedUsd": 0.000572, "estimatedUsd": 0.000572, "usage": { "prompt_tokens": 118, "completion_tokens": 5, "total_tokens": 123 }, "latencyMs": 2358 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0014360000000000002, "billedUsd": 0.001436, "estimatedUsd": 0.0014360000000000002, "usage": { "prompt_tokens": 134, "completion_tokens": 45, "total_tokens": 179 }, "latencyMs": 2028 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.001136, "billedUsd": 0.001136, "estimatedUsd": 0.001136, "usage": { "prompt_tokens": 119, "completion_tokens": 33, "total_tokens": 152 }, "latencyMs": 3301 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.001176, "billedUsd": 0.001176, "estimatedUsd": 0.001176, "usage": { "prompt_tokens": 109, "completion_tokens": 37, "total_tokens": 146 }, "latencyMs": 1868 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000552, "billedUsd": 0.000552, "estimatedUsd": 0.000552, "usage": { "prompt_tokens": 113, "completion_tokens": 5, "total_tokens": 118 }, "latencyMs": 1733 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0005239999999999999, "billedUsd": 0.000524, "estimatedUsd": 0.0005239999999999999, "usage": { "prompt_tokens": 111, "completion_tokens": 4, "total_tokens": 115 }, "latencyMs": 1645 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000504, "billedUsd": 0.000504, "estimatedUsd": 0.000504, "usage": { "prompt_tokens": 106, "completion_tokens": 4, "total_tokens": 110 }, "latencyMs": 1568 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000552, "billedUsd": 0.000552, "estimatedUsd": 0.000552, "usage": { "prompt_tokens": 108, "completion_tokens": 6, "total_tokens": 114 }, "latencyMs": 1535 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000468, "billedUsd": 0.000468, "estimatedUsd": 0.000468, "usage": { "prompt_tokens": 97, "completion_tokens": 4, "total_tokens": 101 }, "latencyMs": 5651 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00043599999999999997, "billedUsd": 0.000436, "estimatedUsd": 0.00043599999999999997, "usage": { "prompt_tokens": 94, "completion_tokens": 3, "total_tokens": 97 }, "latencyMs": 2098 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model aced extraction, failed most arithmetic.", "spendUsd": 0.0067280000000000005, "billedUsd": 0.006728, "estimatedUsd": 0.0067280000000000005, "usage": { "prompt_tokens": 152, "completion_tokens": 306, "total_tokens": 458 }, "latencyMs": 7115 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0010479999999999999, "billedUsd": 0.001048, "estimatedUsd": 0.0010479999999999999, "usage": { "prompt_tokens": 157, "completion_tokens": 21, "total_tokens": 178 }, "latencyMs": 2136 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "High verbosity raises cost per correct answer despite accuracy.", "spendUsd": 0.002988, "billedUsd": 0.002988, "estimatedUsd": 0.002988, "usage": { "prompt_tokens": 127, "completion_tokens": 124, "total_tokens": 251 }, "latencyMs": 3463 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The True Cost per Correct Answer", "spendUsd": 0.001804, "billedUsd": 0.001804, "estimatedUsd": 0.001804, "usage": { "prompt_tokens": 116, "completion_tokens": 67, "total_tokens": 183 }, "latencyMs": 3699 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.001824, "billedUsd": 0.001824, "estimatedUsd": 0.001824, "usage": { "prompt_tokens": 151, "completion_tokens": 61, "total_tokens": 212 }, "latencyMs": 1880 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000932, "billedUsd": 0.000932, "estimatedUsd": 0.000932, "usage": { "prompt_tokens": 118, "completion_tokens": 23, "total_tokens": 141 }, "latencyMs": 4938 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00468, "billedUsd": 0.00468, "estimatedUsd": 0.00468, "usage": { "prompt_tokens": 110, "completion_tokens": 212, "total_tokens": 322 }, "latencyMs": 7611 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00058, "billedUsd": 0.00058, "estimatedUsd": 0.00058, "usage": { "prompt_tokens": 110, "completion_tokens": 7, "total_tokens": 117 }, "latencyMs": 2139 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0017040000000000002, "billedUsd": 0.001704, "estimatedUsd": 0.0017040000000000002, "usage": { "prompt_tokens": 126, "completion_tokens": 60, "total_tokens": 186 }, "latencyMs": 4280 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000536, "billedUsd": 0.000536, "estimatedUsd": 0.000536, "usage": { "prompt_tokens": 99, "completion_tokens": 7, "total_tokens": 106 }, "latencyMs": 2673 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000504, "billedUsd": 0.000504, "estimatedUsd": 0.000504, "usage": { "prompt_tokens": 101, "completion_tokens": 5, "total_tokens": 106 }, "latencyMs": 2135 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.001128, "billedUsd": 0.001128, "estimatedUsd": 0.001128, "usage": { "prompt_tokens": 87, "completion_tokens": 39, "total_tokens": 126 }, "latencyMs": 4259 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000372, "billedUsd": 0.000372, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 78, "completion_tokens": 3, "total_tokens": 81 }, "latencyMs": 2119 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001208, "billedUsd": 0.001208, "estimatedUsd": 0.001208, "usage": { "prompt_tokens": 72, "completion_tokens": 46, "total_tokens": 118 }, "latencyMs": 3920 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000432, "billedUsd": 0.000432, "estimatedUsd": 0.000432, "usage": { "prompt_tokens": 78, "completion_tokens": 6, "total_tokens": 84 }, "latencyMs": 1908 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000572, "billedUsd": 0.000572, "estimatedUsd": 0.000572, "usage": { "prompt_tokens": 103, "completion_tokens": 8, "total_tokens": 111 }, "latencyMs": 1711 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000432, "billedUsd": 0.000432, "estimatedUsd": 0.000432, "usage": { "prompt_tokens": 88, "completion_tokens": 4, "total_tokens": 92 }, "latencyMs": 2898 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000448, "billedUsd": 0.000448, "estimatedUsd": 0.000448, "usage": { "prompt_tokens": 87, "completion_tokens": 5, "total_tokens": 92 }, "latencyMs": 1842 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000536, "billedUsd": 0.000536, "estimatedUsd": 0.000536, "usage": { "prompt_tokens": 94, "completion_tokens": 8, "total_tokens": 102 }, "latencyMs": 1911 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00142, "billedUsd": 0.00142, "estimatedUsd": 0.00142, "usage": { "prompt_tokens": 100, "completion_tokens": 51, "total_tokens": 151 }, "latencyMs": 4139 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001052, "billedUsd": 0.001052, "estimatedUsd": 0.001052, "usage": { "prompt_tokens": 98, "completion_tokens": 33, "total_tokens": 131 }, "latencyMs": 2878 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000652, "billedUsd": 0.000652, "estimatedUsd": 0.000652, "usage": { "prompt_tokens": 133, "completion_tokens": 6, "total_tokens": 139 }, "latencyMs": 1953 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0009159999999999999, "billedUsd": 0.000916, "estimatedUsd": 0.0009159999999999999, "usage": { "prompt_tokens": 104, "completion_tokens": 25, "total_tokens": 129 }, "latencyMs": 3003 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000492, "billedUsd": 0.000492, "estimatedUsd": 0.000492, "usage": { "prompt_tokens": 103, "completion_tokens": 4, "total_tokens": 107 }, "latencyMs": 1651 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00116, "billedUsd": 0.00116, "estimatedUsd": 0.00116, "usage": { "prompt_tokens": 125, "completion_tokens": 33, "total_tokens": 158 }, "latencyMs": 1838 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0008719999999999999, "billedUsd": 0.000872, "estimatedUsd": 0.0008719999999999999, "usage": { "prompt_tokens": 93, "completion_tokens": 25, "total_tokens": 118 }, "latencyMs": 3233 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0010439999999999998, "billedUsd": 0.001044, "estimatedUsd": 0.0010439999999999998, "usage": { "prompt_tokens": 86, "completion_tokens": 35, "total_tokens": 121 }, "latencyMs": 3532 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0009040000000000001, "billedUsd": 0.000904, "estimatedUsd": 0.0009040000000000001, "usage": { "prompt_tokens": 91, "completion_tokens": 27, "total_tokens": 118 }, "latencyMs": 2859 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-5-5", "openrouterId": "anthropic/claude-opus-5.5", "label": "Claude Opus 5.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.001044, "billedUsd": 0.001044, "estimatedUsd": 0.001044, "usage": { "prompt_tokens": 101, "completion_tokens": 32, "total_tokens": 133 }, "latencyMs": 3652 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000054, "billedUsd": 0.000054, "estimatedUsd": 0.000054, "usage": { "prompt_tokens": 72, "completion_tokens": 33, "total_tokens": 105 }, "latencyMs": 2346 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0000692, "billedUsd": 0.0000692, "estimatedUsd": 0.0000692, "usage": { "prompt_tokens": 88, "completion_tokens": 43, "total_tokens": 131 }, "latencyMs": 2305 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000044999999999999996, "billedUsd": 0.000045, "estimatedUsd": 0.000044999999999999996, "usage": { "prompt_tokens": 63, "completion_tokens": 27, "total_tokens": 90 }, "latencyMs": 1907 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0000424, "billedUsd": 0.0000424, "estimatedUsd": 0.0000424, "usage": { "prompt_tokens": 62, "completion_tokens": 25, "total_tokens": 87 }, "latencyMs": 1079 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0000522, "billedUsd": 0.0000522, "estimatedUsd": 0.0000522, "usage": { "prompt_tokens": 75, "completion_tokens": 31, "total_tokens": 106 }, "latencyMs": 1980 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1269 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1065 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00004599999999999999, "billedUsd": 0.000046, "estimatedUsd": 0.00004599999999999999, "usage": { "prompt_tokens": 68, "completion_tokens": 27, "total_tokens": 95 }, "latencyMs": 1904 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.3,\"outputPerM\":2.5,\"context\":1048576}", "spendUsd": 0.0000998, "billedUsd": 0.0000998, "estimatedUsd": 0.0000998, "usage": { "prompt_tokens": 97, "completion_tokens": 67, "total_tokens": 164 }, "latencyMs": 1600 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.00009219999999999999, "billedUsd": 0.0000922, "estimatedUsd": 0.00009219999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 63, "total_tokens": 146 }, "latencyMs": 1406 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000020200000000000003, "billedUsd": 0.0000202, "estimatedUsd": 0.000020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 928 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00008139999999999999, "billedUsd": 0.0000814, "estimatedUsd": 0.00008139999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 54, "total_tokens": 137 }, "latencyMs": 1355 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.0000498, "billedUsd": 0.0000498, "estimatedUsd": 0.0000498, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1056 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.0000764, "billedUsd": 0.0000764, "estimatedUsd": 0.0000764, "usage": { "prompt_tokens": 70, "completion_tokens": 52, "total_tokens": 122 }, "latencyMs": 1478 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000020200000000000003, "billedUsd": 0.0000202, "estimatedUsd": 0.000020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1067 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000019600000000000002, "billedUsd": 0.0000196, "estimatedUsd": 0.000019600000000000002, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1002 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000204, "billedUsd": 0.0000204, "estimatedUsd": 0.0000204, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1043 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000019200000000000003, "billedUsd": 0.0000192, "estimatedUsd": 0.000019200000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 900 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000018200000000000002, "billedUsd": 0.0000182, "estimatedUsd": 0.000018200000000000002, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1602 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0000178, "billedUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1278 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded; failed arithmetic favors structured parsing.", "spendUsd": 0.00018319999999999998, "billedUsd": 0.0001832, "estimatedUsd": 0.00018319999999999998, "usage": { "prompt_tokens": 88, "completion_tokens": 138, "total_tokens": 226 }, "latencyMs": 2016 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000354, "billedUsd": 0.0000354, "estimatedUsd": 0.0000354, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 1800 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 11, "sentenceCount": 1 }, "output": "High accuracy with excessive verbosity can increase cost per correct answer.", "spendUsd": 0.0001166, "billedUsd": 0.0001166, "estimatedUsd": 0.0001166, "usage": { "prompt_tokens": 79, "completion_tokens": 84, "total_tokens": 163 }, "latencyMs": 2294 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Correct Answers, Cost Comparison", "spendUsd": 0.0000748, "billedUsd": 0.0000748, "estimatedUsd": 0.0000748, "usage": { "prompt_tokens": 74, "completion_tokens": 50, "total_tokens": 124 }, "latencyMs": 1911 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0001158, "billedUsd": 0.0001158, "estimatedUsd": 0.0001158, "usage": { "prompt_tokens": 87, "completion_tokens": 82, "total_tokens": 169 }, "latencyMs": 1622 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000021400000000000002, "billedUsd": 0.0000214, "estimatedUsd": 0.000021400000000000002, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 944 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected 0.85" }, "output": "1.85", "spendUsd": 0.0000242, "billedUsd": 0.0000242, "estimatedUsd": 0.0000242, "usage": { "prompt_tokens": 79, "completion_tokens": 7, "total_tokens": 86 }, "latencyMs": 900 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0000224, "billedUsd": 0.0000224, "estimatedUsd": 0.0000224, "usage": { "prompt_tokens": 70, "completion_tokens": 7, "total_tokens": 77 }, "latencyMs": 924 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00007840000000000001, "billedUsd": 0.0000784, "estimatedUsd": 0.00007840000000000001, "usage": { "prompt_tokens": 80, "completion_tokens": 52, "total_tokens": 132 }, "latencyMs": 2294 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000023599999999999998, "billedUsd": 0.0000236, "estimatedUsd": 0.000023599999999999998, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 955 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000398, "billedUsd": 0.0000398, "estimatedUsd": 0.0000398, "usage": { "prompt_tokens": 67, "completion_tokens": 22, "total_tokens": 89 }, "latencyMs": 1304 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0000456, "billedUsd": 0.0000456, "estimatedUsd": 0.0000456, "usage": { "prompt_tokens": 54, "completion_tokens": 29, "total_tokens": 83 }, "latencyMs": 2654 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000016800000000000002, "billedUsd": 0.0000168, "estimatedUsd": 0.000016800000000000002, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 1690 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0000214, "billedUsd": 0.0000214, "estimatedUsd": 0.0000214, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 1073 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000016800000000000002, "billedUsd": 0.0000168, "estimatedUsd": 0.000016800000000000002, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 952 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000045799999999999995, "billedUsd": 0.0000458, "estimatedUsd": 0.000045799999999999995, "usage": { "prompt_tokens": 67, "completion_tokens": 27, "total_tokens": 94 }, "latencyMs": 1485 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000043799999999999994, "billedUsd": 0.0000438, "estimatedUsd": 0.000043799999999999994, "usage": { "prompt_tokens": 57, "completion_tokens": 27, "total_tokens": 84 }, "latencyMs": 2264 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000052199999999999995, "billedUsd": 0.0000522, "estimatedUsd": 0.000052199999999999995, "usage": { "prompt_tokens": 57, "completion_tokens": 34, "total_tokens": 91 }, "latencyMs": 2207 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000024, "billedUsd": 0.000024, "estimatedUsd": 0.000024, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 963 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00008740000000000001, "billedUsd": 0.0000874, "estimatedUsd": 0.00008740000000000001, "usage": { "prompt_tokens": 65, "completion_tokens": 62, "total_tokens": 127 }, "latencyMs": 1415 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000018, "billedUsd": 0.000018, "estimatedUsd": 0.000018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 979 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000256, "billedUsd": 0.0000256, "estimatedUsd": 0.0000256, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 903 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0000624, "billedUsd": 0.0000624, "estimatedUsd": 0.0000624, "usage": { "prompt_tokens": 72, "completion_tokens": 40, "total_tokens": 112 }, "latencyMs": 2645 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000204, "billedUsd": 0.0000204, "estimatedUsd": 0.0000204, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1043 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00009719999999999999, "billedUsd": 0.0000972, "estimatedUsd": 0.00009719999999999999, "usage": { "prompt_tokens": 78, "completion_tokens": 68, "total_tokens": 146 }, "latencyMs": 2172 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000186, "billedUsd": 0.0000186, "estimatedUsd": 0.0000186, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 915 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000594, "billedUsd": 0.0000594, "estimatedUsd": 0.0000594, "usage": { "prompt_tokens": 57, "completion_tokens": 40, "total_tokens": 97 }, "latencyMs": 2532 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000044399999999999995, "billedUsd": 0.0000444, "estimatedUsd": 0.000044399999999999995, "usage": { "prompt_tokens": 60, "completion_tokens": 27, "total_tokens": 87 }, "latencyMs": 2407 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-luna", "openrouterId": "openai/gpt-5.6-luna", "label": "GPT-5.6 Luna", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000526, "billedUsd": 0.0000526, "estimatedUsd": 0.0000526, "usage": { "prompt_tokens": 65, "completion_tokens": 33, "total_tokens": 98 }, "latencyMs": 2239 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 904 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00056, "billedUsd": 0.00056, "estimatedUsd": 0.00056, "usage": { "prompt_tokens": 88, "completion_tokens": 32, "total_tokens": 120 }, "latencyMs": 1853 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000198, "billedUsd": 0.000198, "estimatedUsd": 0.000198, "usage": { "prompt_tokens": 63, "completion_tokens": 6, "total_tokens": 69 }, "latencyMs": 949 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00018400000000000003, "billedUsd": 0.000184, "estimatedUsd": 0.00018400000000000003, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 880 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000486, "billedUsd": 0.000486, "estimatedUsd": 0.000486, "usage": { "prompt_tokens": 75, "completion_tokens": 28, "total_tokens": 103 }, "latencyMs": 1865 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1090 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 960 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00021999999999999998, "billedUsd": 0.00022, "estimatedUsd": 0.00021999999999999998, "usage": { "prompt_tokens": 68, "completion_tokens": 7, "total_tokens": 75 }, "latencyMs": 953 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.00059, "billedUsd": 0.00059, "estimatedUsd": 0.00059, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1149 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.0005859999999999999, "billedUsd": 0.000586, "estimatedUsd": 0.0005859999999999999, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1232 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00020200000000000003, "billedUsd": 0.000202, "estimatedUsd": 0.00020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 999 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1159 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.0010019999999999999, "billedUsd": 0.001002, "estimatedUsd": 0.0010019999999999999, "usage": { "prompt_tokens": 81, "completion_tokens": 70, "total_tokens": 151 }, "latencyMs": 1711 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.000428, "billedUsd": 0.000428, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1205 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00020200000000000003, "billedUsd": 0.000202, "estimatedUsd": 0.00020200000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1147 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000196, "billedUsd": 0.000196, "estimatedUsd": 0.000196, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 904 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1009 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000192, "billedUsd": 0.000192, "estimatedUsd": 0.000192, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 807 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 995 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000178, "billedUsd": 0.000178, "estimatedUsd": 0.000178, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1044 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper model aced extraction yet failed multi-step arithmetic", "spendUsd": 0.003188, "billedUsd": 0.003188, "estimatedUsd": 0.003188, "usage": { "prompt_tokens": 88, "completion_tokens": 251, "total_tokens": 339 }, "latencyMs": 2934 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000354, "billedUsd": 0.000354, "estimatedUsd": 0.000354, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 985 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "High verbosity can increase cost despite high accuracy.", "spendUsd": 0.000314, "billedUsd": 0.000314, "estimatedUsd": 0.000314, "usage": { "prompt_tokens": 79, "completion_tokens": 13, "total_tokens": 92 }, "latencyMs": 940 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "The Cost of Correct Answers", "spendUsd": 0.000256, "billedUsd": 0.000256, "estimatedUsd": 0.000256, "usage": { "prompt_tokens": 74, "completion_tokens": 9, "total_tokens": 83 }, "latencyMs": 1118 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000618, "billedUsd": 0.000618, "estimatedUsd": 0.000618, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 992 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000214, "billedUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 897 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00059, "billedUsd": 0.00059, "estimatedUsd": 0.00059, "usage": { "prompt_tokens": 79, "completion_tokens": 36, "total_tokens": 115 }, "latencyMs": 1888 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000452, "billedUsd": 0.000452, "estimatedUsd": 0.000452, "usage": { "prompt_tokens": 70, "completion_tokens": 26, "total_tokens": 96 }, "latencyMs": 1222 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.000844, "billedUsd": 0.000844, "estimatedUsd": 0.000844, "usage": { "prompt_tokens": 80, "completion_tokens": 57, "total_tokens": 137 }, "latencyMs": 2211 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000428, "billedUsd": 0.000428, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 64, "completion_tokens": 25, "total_tokens": 89 }, "latencyMs": 1058 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00041, "billedUsd": 0.00041, "estimatedUsd": 0.00041, "usage": { "prompt_tokens": 67, "completion_tokens": 23, "total_tokens": 90 }, "latencyMs": 1227 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000468, "billedUsd": 0.000468, "estimatedUsd": 0.000468, "usage": { "prompt_tokens": 54, "completion_tokens": 30, "total_tokens": 84 }, "latencyMs": 1865 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00016800000000000002, "billedUsd": 0.000168, "estimatedUsd": 0.00016800000000000002, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 829 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000862, "billedUsd": 0.000862, "estimatedUsd": 0.000862, "usage": { "prompt_tokens": 47, "completion_tokens": 64, "total_tokens": 111 }, "latencyMs": 2134 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00016800000000000002, "billedUsd": 0.000168, "estimatedUsd": 0.00016800000000000002, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 905 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 915 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000174, "billedUsd": 0.000174, "estimatedUsd": 0.000174, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 961 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00039, "billedUsd": 0.00039, "estimatedUsd": 0.00039, "usage": { "prompt_tokens": 57, "completion_tokens": 23, "total_tokens": 80 }, "latencyMs": 1877 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00024, "billedUsd": 0.00024, "estimatedUsd": 0.00024, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 956 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00079, "billedUsd": 0.00079, "estimatedUsd": 0.00079, "usage": { "prompt_tokens": 65, "completion_tokens": 55, "total_tokens": 120 }, "latencyMs": 1861 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00018, "billedUsd": 0.00018, "estimatedUsd": 0.00018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 905 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000256, "billedUsd": 0.000256, "estimatedUsd": 0.000256, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 907 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000468, "billedUsd": 0.000468, "estimatedUsd": 0.000468, "usage": { "prompt_tokens": 72, "completion_tokens": 27, "total_tokens": 99 }, "latencyMs": 1086 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00020400000000000003, "billedUsd": 0.000204, "estimatedUsd": 0.00020400000000000003, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 975 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.0009480000000000001, "billedUsd": 0.000948, "estimatedUsd": 0.0009480000000000001, "usage": { "prompt_tokens": 78, "completion_tokens": 66, "total_tokens": 144 }, "latencyMs": 2303 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00018600000000000002, "billedUsd": 0.000186, "estimatedUsd": 0.00018600000000000002, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 886 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000462, "billedUsd": 0.000462, "estimatedUsd": 0.000462, "usage": { "prompt_tokens": 57, "completion_tokens": 29, "total_tokens": 86 }, "latencyMs": 3056 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00018, "billedUsd": 0.00018, "estimatedUsd": 0.00018, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 938 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-terra", "openrouterId": "openai/gpt-5.6-terra", "label": "GPT-5.6 Terra", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000478, "billedUsd": 0.000478, "estimatedUsd": 0.000478, "usage": { "prompt_tokens": 65, "completion_tokens": 29, "total_tokens": 94 }, "latencyMs": 2169 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000388, "billedUsd": 0.000388, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 1243 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001212, "billedUsd": 0.001212, "estimatedUsd": 0.001212, "usage": { "prompt_tokens": 88, "completion_tokens": 43, "total_tokens": 131 }, "latencyMs": 2151 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000372, "billedUsd": 0.000372, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 63, "completion_tokens": 6, "total_tokens": 69 }, "latencyMs": 937 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000348, "billedUsd": 0.000348, "estimatedUsd": 0.000348, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 952 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00039999999999999996, "billedUsd": 0.0004, "estimatedUsd": 0.00039999999999999996, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 1157 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000372, "billedUsd": 0.000372, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 946 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000372, "billedUsd": 0.000372, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 879 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000412, "billedUsd": 0.000412, "estimatedUsd": 0.000412, "usage": { "prompt_tokens": 68, "completion_tokens": 7, "total_tokens": 75 }, "latencyMs": 840 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0010479999999999999, "billedUsd": 0.001048, "estimatedUsd": 0.0010479999999999999, "usage": { "prompt_tokens": 97, "completion_tokens": 33, "total_tokens": 130 }, "latencyMs": 1413 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.4,\"outputPerM\":4.4}", "spendUsd": 0.001032, "billedUsd": 0.001032, "estimatedUsd": 0.001032, "usage": { "prompt_tokens": 83, "completion_tokens": 35, "total_tokens": 118 }, "latencyMs": 1821 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 1611 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.000972, "billedUsd": 0.000972, "estimatedUsd": 0.000972, "usage": { "prompt_tokens": 83, "completion_tokens": 32, "total_tokens": 115 }, "latencyMs": 1218 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna,claude-sonnet-5,gemini-2.5-flash", "spendUsd": 0.000884, "billedUsd": 0.000884, "estimatedUsd": 0.000884, "usage": { "prompt_tokens": 81, "completion_tokens": 28, "total_tokens": 109 }, "latencyMs": 1319 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00076, "billedUsd": 0.00076, "estimatedUsd": 0.00076, "usage": { "prompt_tokens": 70, "completion_tokens": 24, "total_tokens": 94 }, "latencyMs": 1242 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 71, "completion_tokens": 5, "total_tokens": 76 }, "latencyMs": 884 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000372, "billedUsd": 0.000372, "estimatedUsd": 0.000372, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1008 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1001 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000364, "billedUsd": 0.000364, "estimatedUsd": 0.000364, "usage": { "prompt_tokens": 66, "completion_tokens": 5, "total_tokens": 71 }, "latencyMs": 941 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000344, "billedUsd": 0.000344, "estimatedUsd": 0.000344, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1163 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000336, "billedUsd": 0.000336, "estimatedUsd": 0.000336, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 1231 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model aced extraction, failed multi-step arithmetic.", "spendUsd": 0.0035319999999999995, "billedUsd": 0.003532, "estimatedUsd": 0.0035319999999999995, "usage": { "prompt_tokens": 88, "completion_tokens": 159, "total_tokens": 247 }, "latencyMs": 2636 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000652, "billedUsd": 0.000652, "estimatedUsd": 0.000652, "usage": { "prompt_tokens": 93, "completion_tokens": 14, "total_tokens": 107 }, "latencyMs": 987 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 11, "sentenceCount": 1 }, "output": "High verbosity can offset accuracy by increasing cost per correct answer.", "spendUsd": 0.000636, "billedUsd": 0.000636, "estimatedUsd": 0.000636, "usage": { "prompt_tokens": 79, "completion_tokens": 16, "total_tokens": 95 }, "latencyMs": 1506 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "cost per correct AI Answer", "spendUsd": 0.0018960000000000001, "billedUsd": 0.001896, "estimatedUsd": 0.0018960000000000001, "usage": { "prompt_tokens": 74, "completion_tokens": 80, "total_tokens": 154 }, "latencyMs": 2588 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.001088, "billedUsd": 0.001088, "estimatedUsd": 0.001088, "usage": { "prompt_tokens": 87, "completion_tokens": 37, "total_tokens": 124 }, "latencyMs": 1223 }, { "taskId": "code-001", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000408, "billedUsd": 0.000408, "estimatedUsd": 0.000408, "usage": { "prompt_tokens": 77, "completion_tokens": 5, "total_tokens": 82 }, "latencyMs": 1471 }, { "taskId": "code-002", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00045599999999999997, "billedUsd": 0.000456, "estimatedUsd": 0.00045599999999999997, "usage": { "prompt_tokens": 79, "completion_tokens": 7, "total_tokens": 86 }, "latencyMs": 944 }, { "taskId": "code-003", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00082, "billedUsd": 0.00082, "estimatedUsd": 0.00082, "usage": { "prompt_tokens": 70, "completion_tokens": 27, "total_tokens": 97 }, "latencyMs": 2627 }, { "taskId": "code-004", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0008, "billedUsd": 0.0008, "estimatedUsd": 0.0008, "usage": { "prompt_tokens": 80, "completion_tokens": 24, "total_tokens": 104 }, "latencyMs": 1059 }, { "taskId": "code-005", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00043599999999999997, "billedUsd": 0.000436, "estimatedUsd": 0.00043599999999999997, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 870 }, { "taskId": "code-006", "category": "coding", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000368, "billedUsd": 0.000368, "estimatedUsd": 0.000368, "usage": { "prompt_tokens": 67, "completion_tokens": 5, "total_tokens": 72 }, "latencyMs": 1347 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.000336, "billedUsd": 0.000336, "estimatedUsd": 0.000336, "usage": { "prompt_tokens": 54, "completion_tokens": 6, "total_tokens": 60 }, "latencyMs": 1059 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000316, "billedUsd": 0.000316, "estimatedUsd": 0.000316, "usage": { "prompt_tokens": 54, "completion_tokens": 5, "total_tokens": 59 }, "latencyMs": 889 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000388, "billedUsd": 0.000388, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 47, "completion_tokens": 10, "total_tokens": 57 }, "latencyMs": 1674 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000312, "billedUsd": 0.000312, "estimatedUsd": 0.000312, "usage": { "prompt_tokens": 48, "completion_tokens": 6, "total_tokens": 54 }, "latencyMs": 935 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000428, "billedUsd": 0.000428, "estimatedUsd": 0.000428, "usage": { "prompt_tokens": 67, "completion_tokens": 8, "total_tokens": 75 }, "latencyMs": 1124 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0006479999999999999, "billedUsd": 0.000648, "estimatedUsd": 0.0006479999999999999, "usage": { "prompt_tokens": 57, "completion_tokens": 21, "total_tokens": 78 }, "latencyMs": 2440 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000328, "billedUsd": 0.000328, "estimatedUsd": 0.000328, "usage": { "prompt_tokens": 57, "completion_tokens": 5, "total_tokens": 62 }, "latencyMs": 844 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00044400000000000006, "billedUsd": 0.000444, "estimatedUsd": 0.00044400000000000006, "usage": { "prompt_tokens": 66, "completion_tokens": 9, "total_tokens": 75 }, "latencyMs": 1817 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0005, "billedUsd": 0.0005, "estimatedUsd": 0.0005, "usage": { "prompt_tokens": 65, "completion_tokens": 12, "total_tokens": 77 }, "latencyMs": 1048 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00034, "billedUsd": 0.00034, "estimatedUsd": 0.00034, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 1088 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000488, "billedUsd": 0.000488, "estimatedUsd": 0.000488, "usage": { "prompt_tokens": 92, "completion_tokens": 6, "total_tokens": 98 }, "latencyMs": 894 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000388, "billedUsd": 0.000388, "estimatedUsd": 0.000388, "usage": { "prompt_tokens": 72, "completion_tokens": 5, "total_tokens": 77 }, "latencyMs": 790 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000384, "billedUsd": 0.000384, "estimatedUsd": 0.000384, "usage": { "prompt_tokens": 66, "completion_tokens": 6, "total_tokens": 72 }, "latencyMs": 1508 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.000832, "billedUsd": 0.000832, "estimatedUsd": 0.000832, "usage": { "prompt_tokens": 78, "completion_tokens": 26, "total_tokens": 104 }, "latencyMs": 1016 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000352, "billedUsd": 0.000352, "estimatedUsd": 0.000352, "usage": { "prompt_tokens": 63, "completion_tokens": 5, "total_tokens": 68 }, "latencyMs": 931 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000808, "billedUsd": 0.000808, "estimatedUsd": 0.000808, "usage": { "prompt_tokens": 57, "completion_tokens": 29, "total_tokens": 86 }, "latencyMs": 2174 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00034, "billedUsd": 0.00034, "estimatedUsd": 0.00034, "usage": { "prompt_tokens": 60, "completion_tokens": 5, "total_tokens": 65 }, "latencyMs": 983 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gpt-5.6-sol", "openrouterId": "openai/gpt-5.6-sol", "label": "GPT-5.6 Sol", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00041999999999999996, "billedUsd": 0.00042, "estimatedUsd": 0.00041999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 8, "total_tokens": 73 }, "latencyMs": 1354 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00216, "billedUsd": 0.00216, "estimatedUsd": 0.00216, "usage": { "prompt_tokens": 106, "completion_tokens": 22, "total_tokens": 128 }, "latencyMs": 3793 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00455, "billedUsd": 0.00455, "estimatedUsd": 0.00455, "usage": { "prompt_tokens": 125, "completion_tokens": 66, "total_tokens": 191 }, "latencyMs": 3850 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0011, "billedUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 2551 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0010500000000000002, "billedUsd": 0.00105, "estimatedUsd": 0.0010500000000000002, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 2801 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0023, "billedUsd": 0.0023, "estimatedUsd": 0.0023, "usage": { "prompt_tokens": 110, "completion_tokens": 24, "total_tokens": 134 }, "latencyMs": 3262 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00123, "billedUsd": 0.00123, "estimatedUsd": 0.00123, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 2736 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00124, "billedUsd": 0.00124, "estimatedUsd": 0.00124, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 2785 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00129, "billedUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 2759 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.00379, "billedUsd": 0.00379, "estimatedUsd": 0.00379, "usage": { "prompt_tokens": 144, "completion_tokens": 47, "total_tokens": 191 }, "latencyMs": 3039 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00377, "billedUsd": 0.00377, "estimatedUsd": 0.00377, "usage": { "prompt_tokens": 127, "completion_tokens": 50, "total_tokens": 177 }, "latencyMs": 2944 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00141, "billedUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 2785 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.00422, "billedUsd": 0.00422, "estimatedUsd": 0.00422, "usage": { "prompt_tokens": 132, "completion_tokens": 58, "total_tokens": 190 }, "latencyMs": 2989 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0028200000000000005, "billedUsd": 0.00282, "estimatedUsd": 0.0028200000000000005, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 2990 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00477, "billedUsd": 0.00477, "estimatedUsd": 0.00477, "usage": { "prompt_tokens": 107, "completion_tokens": 74, "total_tokens": 181 }, "latencyMs": 3303 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 2655 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00129, "billedUsd": 0.00129, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 2572 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0012399999999999998, "billedUsd": 0.00124, "estimatedUsd": 0.0012399999999999998, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 2913 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 2836 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00115, "billedUsd": 0.00115, "estimatedUsd": 0.00115, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 2699 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00107, "billedUsd": 0.00107, "estimatedUsd": 0.00107, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 2771 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded at extraction, failed arithmetic reasoning.", "spendUsd": 0.0089, "billedUsd": 0.0089, "estimatedUsd": 0.0089, "usage": { "prompt_tokens": 150, "completion_tokens": 148, "total_tokens": 298 }, "latencyMs": 5181 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0026, "billedUsd": 0.0026, "estimatedUsd": 0.0026, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 2866 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.0021, "billedUsd": 0.0021, "estimatedUsd": 0.0021, "usage": { "prompt_tokens": 125, "completion_tokens": 17, "total_tokens": 142 }, "latencyMs": 2730 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost per Correct Answer in AI", "spendUsd": 0.00189, "billedUsd": 0.00189, "estimatedUsd": 0.00189, "usage": { "prompt_tokens": 114, "completion_tokens": 15, "total_tokens": 129 }, "latencyMs": 2893 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00654, "billedUsd": 0.00654, "estimatedUsd": 0.00654, "usage": { "prompt_tokens": 149, "completion_tokens": 101, "total_tokens": 250 }, "latencyMs": 4180 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00176, "billedUsd": 0.00176, "estimatedUsd": 0.00176, "usage": { "prompt_tokens": 116, "completion_tokens": 12, "total_tokens": 128 }, "latencyMs": 3655 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00293, "billedUsd": 0.00293, "estimatedUsd": 0.00293, "usage": { "prompt_tokens": 108, "completion_tokens": 37, "total_tokens": 145 }, "latencyMs": 3897 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00143, "billedUsd": 0.00143, "estimatedUsd": 0.00143, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 2772 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03}", "spendUsd": 0.00439, "billedUsd": 0.00439, "estimatedUsd": 0.00439, "usage": { "prompt_tokens": 124, "completion_tokens": 63, "total_tokens": 187 }, "latencyMs": 3714 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00132, "billedUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 2888 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected fetch" }, "output": "", "spendUsd": 0, "billedUsd": 0, "estimatedUsd": 0, "usage": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }, "latencyMs": 1409 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00285, "billedUsd": 0.00285, "estimatedUsd": 0.00285, "usage": { "prompt_tokens": 85, "completion_tokens": 40, "total_tokens": 125 }, "latencyMs": 3460 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00091, "billedUsd": 0.00091, "estimatedUsd": 0.00091, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 2728 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0038000000000000004, "billedUsd": 0.0038, "estimatedUsd": 0.0038000000000000004, "usage": { "prompt_tokens": 70, "completion_tokens": 62, "total_tokens": 132 }, "latencyMs": 3096 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00136, "billedUsd": 0.00136, "estimatedUsd": 0.00136, "usage": { "prompt_tokens": 76, "completion_tokens": 12, "total_tokens": 88 }, "latencyMs": 2960 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00141, "billedUsd": 0.00141, "estimatedUsd": 0.00141, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 2766 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0010600000000000002, "billedUsd": 0.00106, "estimatedUsd": 0.0010600000000000002, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 2989 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0011, "billedUsd": 0.0011, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 2765 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00132, "billedUsd": 0.00132, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 2574 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0033299999999999996, "billedUsd": 0.00333, "estimatedUsd": 0.0033299999999999996, "usage": { "prompt_tokens": 98, "completion_tokens": 47, "total_tokens": 145 }, "latencyMs": 2883 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0025599999999999998, "billedUsd": 0.00256, "estimatedUsd": 0.0025599999999999998, "usage": { "prompt_tokens": 96, "completion_tokens": 32, "total_tokens": 128 }, "latencyMs": 3707 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0016100000000000003, "billedUsd": 0.00161, "estimatedUsd": 0.0016100000000000003, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 2685 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00117, "billedUsd": 0.00117, "estimatedUsd": 0.00117, "usage": { "prompt_tokens": 102, "completion_tokens": 3, "total_tokens": 105 }, "latencyMs": 2949 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0012100000000000001, "billedUsd": 0.00121, "estimatedUsd": 0.0012100000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 2598 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.005030000000000001, "billedUsd": 0.00503, "estimatedUsd": 0.005030000000000001, "usage": { "prompt_tokens": 123, "completion_tokens": 76, "total_tokens": 199 }, "latencyMs": 3530 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0011099999999999999, "billedUsd": 0.00111, "estimatedUsd": 0.0011099999999999999, "usage": { "prompt_tokens": 91, "completion_tokens": 4, "total_tokens": 95 }, "latencyMs": 2958 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0010899999999999998, "billedUsd": 0.00109, "estimatedUsd": 0.0010899999999999998, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 2666 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00104, "billedUsd": 0.00104, "estimatedUsd": 0.00104, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 2692 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-fable-5", "openrouterId": "anthropic/claude-fable-5", "label": "Claude Fable 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0014399999999999999, "billedUsd": 0.00144, "estimatedUsd": 0.0014399999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 9, "total_tokens": 108 }, "latencyMs": 2848 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001029, "billedUsd": 0.00051856, "estimatedUsd": 0.001029, "usage": { "prompt_tokens": 158, "completion_tokens": 37, "total_tokens": 195 }, "latencyMs": 512 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.002877, "billedUsd": 0.002669243, "estimatedUsd": 0.002877, "usage": { "prompt_tokens": 174, "completion_tokens": 157, "total_tokens": 331 }, "latencyMs": 1009 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.001467, "billedUsd": 0.0005528, "estimatedUsd": 0.001467, "usage": { "prompt_tokens": 149, "completion_tokens": 68, "total_tokens": 217 }, "latencyMs": 1752 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001359, "billedUsd": 0.00079868, "estimatedUsd": 0.001359, "usage": { "prompt_tokens": 148, "completion_tokens": 61, "total_tokens": 209 }, "latencyMs": 599 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.001398, "billedUsd": 0.001398, "estimatedUsd": 0.001398, "usage": { "prompt_tokens": 161, "completion_tokens": 61, "total_tokens": 222 }, "latencyMs": 1624 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001047, "billedUsd": 0.0008742, "estimatedUsd": 0.001047, "usage": { "prompt_tokens": 154, "completion_tokens": 39, "total_tokens": 193 }, "latencyMs": 1369 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.001137, "billedUsd": 0.0006116, "estimatedUsd": 0.001137, "usage": { "prompt_tokens": 154, "completion_tokens": 45, "total_tokens": 199 }, "latencyMs": 510 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.001242, "billedUsd": 0.000674176, "estimatedUsd": 0.001242, "usage": { "prompt_tokens": 154, "completion_tokens": 52, "total_tokens": 206 }, "latencyMs": 789 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.002694, "billedUsd": 0.00179034, "estimatedUsd": 0.002694, "usage": { "prompt_tokens": 183, "completion_tokens": 143, "total_tokens": 326 }, "latencyMs": 1089 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.002517, "billedUsd": 0.00167642, "estimatedUsd": 0.002517, "usage": { "prompt_tokens": 169, "completion_tokens": 134, "total_tokens": 303 }, "latencyMs": 1100 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.001524, "billedUsd": 0.0005746, "estimatedUsd": 0.001524, "usage": { "prompt_tokens": 158, "completion_tokens": 70, "total_tokens": 228 }, "latencyMs": 1562 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0018509999999999998, "billedUsd": 0.00175845, "estimatedUsd": 0.0018509999999999998, "usage": { "prompt_tokens": 172, "completion_tokens": 89, "total_tokens": 261 }, "latencyMs": 2161 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.002319, "billedUsd": 0.00152148, "estimatedUsd": 0.002319, "usage": { "prompt_tokens": 168, "completion_tokens": 121, "total_tokens": 289 }, "latencyMs": 880 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.001968, "billedUsd": 0.00180134, "estimatedUsd": 0.001968, "usage": { "prompt_tokens": 156, "completion_tokens": 100, "total_tokens": 256 }, "latencyMs": 822 }, { "taskId": "classify-001", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0014910000000000001, "billedUsd": 0.00088634, "estimatedUsd": 0.0014910000000000001, "usage": { "prompt_tokens": 157, "completion_tokens": 68, "total_tokens": 225 }, "latencyMs": 849 }, { "taskId": "classify-002", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.001677, "billedUsd": 0.001520999, "estimatedUsd": 0.001677, "usage": { "prompt_tokens": 154, "completion_tokens": 81, "total_tokens": 235 }, "latencyMs": 901 }, { "taskId": "classify-003", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0016680000000000002, "billedUsd": 0.0009174, "estimatedUsd": 0.0016680000000000002, "usage": { "prompt_tokens": 156, "completion_tokens": 80, "total_tokens": 236 }, "latencyMs": 1354 }, { "taskId": "classify-004", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.001869, "billedUsd": 0.00119322, "estimatedUsd": 0.001869, "usage": { "prompt_tokens": 153, "completion_tokens": 94, "total_tokens": 247 }, "latencyMs": 1131 }, { "taskId": "classify-005", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00198, "billedUsd": 0.001089, "estimatedUsd": 0.00198, "usage": { "prompt_tokens": 150, "completion_tokens": 102, "total_tokens": 252 }, "latencyMs": 1668 }, { "taskId": "classify-006", "category": "classification", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0017100000000000001, "billedUsd": 0.0010823, "estimatedUsd": 0.0017100000000000001, "usage": { "prompt_tokens": 145, "completion_tokens": 85, "total_tokens": 230 }, "latencyMs": 722 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "cheaper extraction succeeded but failed multi-step arithmetic tasks", "spendUsd": 0.002529, "billedUsd": 0.002333066, "estimatedUsd": 0.002529, "usage": { "prompt_tokens": 173, "completion_tokens": 134, "total_tokens": 307 }, "latencyMs": 1264 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.001887, "billedUsd": 0.001887, "estimatedUsd": 0.001887, "usage": { "prompt_tokens": 179, "completion_tokens": 90, "total_tokens": 269 }, "latencyMs": 1184 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity raises token cost, hurting dollars per correct answer.", "spendUsd": 0.0025050000000000003, "billedUsd": 0.00226216, "estimatedUsd": 0.0025050000000000003, "usage": { "prompt_tokens": 165, "completion_tokens": 134, "total_tokens": 299 }, "latencyMs": 1363 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.002625, "billedUsd": 0.002434617, "estimatedUsd": 0.002625, "usage": { "prompt_tokens": 160, "completion_tokens": 143, "total_tokens": 303 }, "latencyMs": 1438 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.003384, "billedUsd": 0.00235558, "estimatedUsd": 0.003384, "usage": { "prompt_tokens": 173, "completion_tokens": 191, "total_tokens": 364 }, "latencyMs": 995 }, { "taskId": "code-001", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0017040000000000002, "billedUsd": 0.001704, "estimatedUsd": 0.0017040000000000002, "usage": { "prompt_tokens": 163, "completion_tokens": 81, "total_tokens": 244 }, "latencyMs": 5017 }, { "taskId": "code-002", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.01221, "billedUsd": 0.009353236, "estimatedUsd": 0.01221, "usage": { "prompt_tokens": 165, "completion_tokens": 781, "total_tokens": 946 }, "latencyMs": 6245 }, { "taskId": "code-003", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.002358, "billedUsd": 0.0021852, "estimatedUsd": 0.002358, "usage": { "prompt_tokens": 156, "completion_tokens": 126, "total_tokens": 282 }, "latencyMs": 2307 }, { "taskId": "code-004", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.002913, "billedUsd": 0.00196137, "estimatedUsd": 0.002913, "usage": { "prompt_tokens": 166, "completion_tokens": 161, "total_tokens": 327 }, "latencyMs": 2822 }, { "taskId": "code-005", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.002127, "billedUsd": 0.0007948, "estimatedUsd": 0.002127, "usage": { "prompt_tokens": 149, "completion_tokens": 112, "total_tokens": 261 }, "latencyMs": 1960 }, { "taskId": "code-006", "category": "coding", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.001254, "billedUsd": 0.00070614, "estimatedUsd": 0.001254, "usage": { "prompt_tokens": 153, "completion_tokens": 53, "total_tokens": 206 }, "latencyMs": 714 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00132, "billedUsd": 0.000498, "estimatedUsd": 0.00132, "usage": { "prompt_tokens": 140, "completion_tokens": 60, "total_tokens": 200 }, "latencyMs": 1953 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00117, "billedUsd": 0.00090016, "estimatedUsd": 0.00117, "usage": { "prompt_tokens": 140, "completion_tokens": 50, "total_tokens": 190 }, "latencyMs": 818 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001524, "billedUsd": 0.001386845, "estimatedUsd": 0.001524, "usage": { "prompt_tokens": 133, "completion_tokens": 75, "total_tokens": 208 }, "latencyMs": 946 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.001752, "billedUsd": 0.0011362, "estimatedUsd": 0.001752, "usage": { "prompt_tokens": 134, "completion_tokens": 90, "total_tokens": 224 }, "latencyMs": 892 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.001446, "billedUsd": 0.001298734, "estimatedUsd": 0.001446, "usage": { "prompt_tokens": 152, "completion_tokens": 66, "total_tokens": 218 }, "latencyMs": 578 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0019260000000000002, "billedUsd": 0.001241056, "estimatedUsd": 0.0019260000000000002, "usage": { "prompt_tokens": 142, "completion_tokens": 100, "total_tokens": 242 }, "latencyMs": 1074 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.001656, "billedUsd": 0.00101269, "estimatedUsd": 0.001656, "usage": { "prompt_tokens": 142, "completion_tokens": 82, "total_tokens": 224 }, "latencyMs": 1749 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001836, "billedUsd": 0.0006884, "estimatedUsd": 0.001836, "usage": { "prompt_tokens": 152, "completion_tokens": 92, "total_tokens": 244 }, "latencyMs": 2083 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.001953, "billedUsd": 0.001244296, "estimatedUsd": 0.001953, "usage": { "prompt_tokens": 151, "completion_tokens": 100, "total_tokens": 251 }, "latencyMs": 962 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001443, "billedUsd": 0.0005437, "estimatedUsd": 0.001443, "usage": { "prompt_tokens": 146, "completion_tokens": 67, "total_tokens": 213 }, "latencyMs": 1692 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.002724, "billedUsd": 0.0010166, "estimatedUsd": 0.002724, "usage": { "prompt_tokens": 178, "completion_tokens": 146, "total_tokens": 324 }, "latencyMs": 3189 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.001503, "billedUsd": 0.00082665, "estimatedUsd": 0.001503, "usage": { "prompt_tokens": 161, "completion_tokens": 68, "total_tokens": 229 }, "latencyMs": 1656 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.001446, "billedUsd": 0.0011648, "estimatedUsd": 0.001446, "usage": { "prompt_tokens": 152, "completion_tokens": 66, "total_tokens": 218 }, "latencyMs": 957 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.002157, "billedUsd": 0.00140068, "estimatedUsd": 0.002157, "usage": { "prompt_tokens": 164, "completion_tokens": 111, "total_tokens": 275 }, "latencyMs": 1677 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0016319999999999998, "billedUsd": 0.0006133, "estimatedUsd": 0.0016319999999999998, "usage": { "prompt_tokens": 149, "completion_tokens": 79, "total_tokens": 228 }, "latencyMs": 1658 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001338, "billedUsd": 0.0007359, "estimatedUsd": 0.001338, "usage": { "prompt_tokens": 146, "completion_tokens": 60, "total_tokens": 206 }, "latencyMs": 1485 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.001317, "billedUsd": 0.00072435, "estimatedUsd": 0.001317, "usage": { "prompt_tokens": 149, "completion_tokens": 58, "total_tokens": 207 }, "latencyMs": 1096 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "kimi-k3", "openrouterId": "moonshotai/kimi-k3", "label": "Kimi K3", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.004788, "billedUsd": 0.003493396, "estimatedUsd": 0.004788, "usage": { "prompt_tokens": 151, "completion_tokens": 289, "total_tokens": 440 }, "latencyMs": 2293 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000605, "billedUsd": 0.000605, "estimatedUsd": 0.000605, "usage": { "prompt_tokens": 106, "completion_tokens": 3, "total_tokens": 109 }, "latencyMs": 801 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0007, "billedUsd": 0.0007, "estimatedUsd": 0.0007, "usage": { "prompt_tokens": 125, "completion_tokens": 3, "total_tokens": 128 }, "latencyMs": 779 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 704 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0005250000000000001, "billedUsd": 0.000525, "estimatedUsd": 0.0005250000000000001, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 750 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000625, "billedUsd": 0.000625, "estimatedUsd": 0.000625, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 832 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000615, "billedUsd": 0.000615, "estimatedUsd": 0.000615, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 812 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00062, "billedUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 925 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 943 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.002245, "billedUsd": 0.002245, "estimatedUsd": 0.002245, "usage": { "prompt_tokens": 144, "completion_tokens": 61, "total_tokens": 205 }, "latencyMs": 1253 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.002235, "billedUsd": 0.002235, "estimatedUsd": 0.002235, "usage": { "prompt_tokens": 127, "completion_tokens": 64, "total_tokens": 191 }, "latencyMs": 1149 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000705, "billedUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 725 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.0017850000000000001, "billedUsd": 0.001785, "estimatedUsd": 0.0017850000000000001, "usage": { "prompt_tokens": 132, "completion_tokens": 45, "total_tokens": 177 }, "latencyMs": 1331 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0014100000000000002, "billedUsd": 0.00141, "estimatedUsd": 0.0014100000000000002, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 1128 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00146, "billedUsd": 0.00146, "estimatedUsd": 0.00146, "usage": { "prompt_tokens": 107, "completion_tokens": 37, "total_tokens": 144 }, "latencyMs": 1031 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 894 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1013 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0006199999999999999, "billedUsd": 0.00062, "estimatedUsd": 0.0006199999999999999, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 797 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 997 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000575, "billedUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 911 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000535, "billedUsd": 0.000535, "estimatedUsd": 0.000535, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 995 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model passed extraction but failed arithmetic reasoning.", "spendUsd": 0.0013999999999999998, "billedUsd": 0.0014, "estimatedUsd": 0.0013999999999999998, "usage": { "prompt_tokens": 150, "completion_tokens": 26, "total_tokens": 176 }, "latencyMs": 1245 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0013, "billedUsd": 0.0013, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 768 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer.", "spendUsd": 0.00105, "billedUsd": 0.00105, "estimatedUsd": 0.00105, "usage": { "prompt_tokens": 125, "completion_tokens": 17, "total_tokens": 142 }, "latencyMs": 901 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost Per Correct Answer", "spendUsd": 0.000895, "billedUsd": 0.000895, "estimatedUsd": 0.000895, "usage": { "prompt_tokens": 114, "completion_tokens": 13, "total_tokens": 127 }, "latencyMs": 1012 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00227, "billedUsd": 0.00227, "estimatedUsd": 0.00227, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 1173 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000655, "billedUsd": 0.000655, "estimatedUsd": 0.000655, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 852 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000665, "billedUsd": 0.000665, "estimatedUsd": 0.000665, "usage": { "prompt_tokens": 108, "completion_tokens": 5, "total_tokens": 113 }, "latencyMs": 988 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000715, "billedUsd": 0.000715, "estimatedUsd": 0.000715, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 833 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0014199999999999998, "billedUsd": 0.00142, "estimatedUsd": 0.0014199999999999998, "usage": { "prompt_tokens": 124, "completion_tokens": 32, "total_tokens": 156 }, "latencyMs": 1144 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 929 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00062, "billedUsd": 0.00062, "estimatedUsd": 0.00062, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 759 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 1119 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000455, "billedUsd": 0.000455, "estimatedUsd": 0.000455, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 882 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000575, "billedUsd": 0.000575, "estimatedUsd": 0.000575, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 824 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0005300000000000001, "billedUsd": 0.00053, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 1139 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000705, "billedUsd": 0.000705, "estimatedUsd": 0.000705, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 777 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0005300000000000001, "billedUsd": 0.00053, "estimatedUsd": 0.0005300000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 777 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00055, "billedUsd": 0.00055, "estimatedUsd": 0.00055, "usage": { "prompt_tokens": 85, "completion_tokens": 5, "total_tokens": 90 }, "latencyMs": 950 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 1036 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000815, "billedUsd": 0.000815, "estimatedUsd": 0.000815, "usage": { "prompt_tokens": 98, "completion_tokens": 13, "total_tokens": 111 }, "latencyMs": 1140 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000555, "billedUsd": 0.000555, "estimatedUsd": 0.000555, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 700 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0008050000000000002, "billedUsd": 0.000805, "estimatedUsd": 0.0008050000000000002, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 836 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000585, "billedUsd": 0.000585, "estimatedUsd": 0.000585, "usage": { "prompt_tokens": 102, "completion_tokens": 3, "total_tokens": 105 }, "latencyMs": 720 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006050000000000001, "billedUsd": 0.000605, "estimatedUsd": 0.0006050000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 863 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0018150000000000002, "billedUsd": 0.001815, "estimatedUsd": 0.0018150000000000002, "usage": { "prompt_tokens": 123, "completion_tokens": 48, "total_tokens": 171 }, "latencyMs": 977 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00058, "billedUsd": 0.00058, "estimatedUsd": 0.00058, "usage": { "prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96 }, "latencyMs": 952 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0005449999999999999, "billedUsd": 0.000545, "estimatedUsd": 0.0005449999999999999, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 911 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00052, "billedUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 776 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-4.8", "openrouterId": "anthropic/claude-opus-4.8", "label": "Claude Opus 4.8", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0007199999999999999, "billedUsd": 0.00072, "estimatedUsd": 0.0007199999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 9, "total_tokens": 108 }, "latencyMs": 838 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00108, "billedUsd": 0.00108, "estimatedUsd": 0.00108, "usage": { "prompt_tokens": 106, "completion_tokens": 22, "total_tokens": 128 }, "latencyMs": 5013 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0024749999999999998, "billedUsd": 0.002475, "estimatedUsd": 0.0024749999999999998, "usage": { "prompt_tokens": 125, "completion_tokens": 74, "total_tokens": 199 }, "latencyMs": 3659 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000925, "billedUsd": 0.000925, "estimatedUsd": 0.000925, "usage": { "prompt_tokens": 90, "completion_tokens": 19, "total_tokens": 109 }, "latencyMs": 2691 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0005250000000000001, "billedUsd": 0.000525, "estimatedUsd": 0.0005250000000000001, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 1463 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.0013750000000000001, "billedUsd": 0.001375, "estimatedUsd": 0.0013750000000000001, "usage": { "prompt_tokens": 110, "completion_tokens": 33, "total_tokens": 143 }, "latencyMs": 2586 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00099, "billedUsd": 0.00099, "estimatedUsd": 0.00099, "usage": { "prompt_tokens": 108, "completion_tokens": 18, "total_tokens": 126 }, "latencyMs": 4492 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00087, "billedUsd": 0.00087, "estimatedUsd": 0.00087, "usage": { "prompt_tokens": 109, "completion_tokens": 13, "total_tokens": 122 }, "latencyMs": 3435 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 1662 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.00212, "billedUsd": 0.00212, "estimatedUsd": 0.00212, "usage": { "prompt_tokens": 144, "completion_tokens": 56, "total_tokens": 200 }, "latencyMs": 2968 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00211, "billedUsd": 0.00211, "estimatedUsd": 0.00211, "usage": { "prompt_tokens": 127, "completion_tokens": 59, "total_tokens": 186 }, "latencyMs": 5240 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00118, "billedUsd": 0.00118, "estimatedUsd": 0.00118, "usage": { "prompt_tokens": 116, "completion_tokens": 24, "total_tokens": 140 }, "latencyMs": 4499 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0022849999999999997, "billedUsd": 0.002285, "estimatedUsd": 0.0022849999999999997, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 2352 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00271, "billedUsd": 0.00271, "estimatedUsd": 0.00271, "usage": { "prompt_tokens": 117, "completion_tokens": 85, "total_tokens": 202 }, "latencyMs": 4161 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0017599999999999998, "billedUsd": 0.00176, "estimatedUsd": 0.0017599999999999998, "usage": { "prompt_tokens": 107, "completion_tokens": 49, "total_tokens": 156 }, "latencyMs": 2300 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 1496 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000645, "billedUsd": 0.000645, "estimatedUsd": 0.000645, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1313 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00082, "billedUsd": 0.00082, "estimatedUsd": 0.00082, "usage": { "prompt_tokens": 104, "completion_tokens": 12, "total_tokens": 116 }, "latencyMs": 3270 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00068, "billedUsd": 0.00068, "estimatedUsd": 0.00068, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 1299 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00145, "billedUsd": 0.00145, "estimatedUsd": 0.00145, "usage": { "prompt_tokens": 95, "completion_tokens": 39, "total_tokens": 134 }, "latencyMs": 4337 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0010600000000000002, "billedUsd": 0.00106, "estimatedUsd": 0.0010600000000000002, "usage": { "prompt_tokens": 92, "completion_tokens": 24, "total_tokens": 116 }, "latencyMs": 2899 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model aced extraction, failed arithmetic; choose parsing.", "spendUsd": 0.0042499999999999994, "billedUsd": 0.00425, "estimatedUsd": 0.0042499999999999994, "usage": { "prompt_tokens": 150, "completion_tokens": 140, "total_tokens": 290 }, "latencyMs": 4051 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0013, "billedUsd": 0.0013, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 1826 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity inflates cost per correct answer despite strong accuracy.", "spendUsd": 0.003, "billedUsd": 0.003, "estimatedUsd": 0.003, "usage": { "prompt_tokens": 125, "completion_tokens": 95, "total_tokens": 220 }, "latencyMs": 3655 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "The Cost of Each Correct Answer", "spendUsd": 0.00217, "billedUsd": 0.00217, "estimatedUsd": 0.00217, "usage": { "prompt_tokens": 114, "completion_tokens": 64, "total_tokens": 178 }, "latencyMs": 3751 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 27, "sentenceCount": 1 }, "output": "OpenRouter handles the routing, but costs are derived from AI Pricing Guru's direct-provider list prices, so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.003395, "billedUsd": 0.003395, "estimatedUsd": 0.003395, "usage": { "prompt_tokens": 149, "completion_tokens": 106, "total_tokens": 255 }, "latencyMs": 2721 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00108, "billedUsd": 0.00108, "estimatedUsd": 0.00108, "usage": { "prompt_tokens": 116, "completion_tokens": 20, "total_tokens": 136 }, "latencyMs": 3348 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00949, "billedUsd": 0.00949, "estimatedUsd": 0.00949, "usage": { "prompt_tokens": 108, "completion_tokens": 358, "total_tokens": 466 }, "latencyMs": 7260 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "`inputPerM`", "spendUsd": 0.001115, "billedUsd": 0.001115, "estimatedUsd": 0.001115, "usage": { "prompt_tokens": 108, "completion_tokens": 23, "total_tokens": 131 }, "latencyMs": 2996 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03}", "spendUsd": 0.00327, "billedUsd": 0.00327, "estimatedUsd": 0.00327, "usage": { "prompt_tokens": 124, "completion_tokens": 106, "total_tokens": 230 }, "latencyMs": 5663 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0016599999999999998, "billedUsd": 0.00166, "estimatedUsd": 0.0016599999999999998, "usage": { "prompt_tokens": 97, "completion_tokens": 47, "total_tokens": 144 }, "latencyMs": 3678 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.001045, "billedUsd": 0.001045, "estimatedUsd": 0.001045, "usage": { "prompt_tokens": 99, "completion_tokens": 22, "total_tokens": 121 }, "latencyMs": 3036 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0014, "billedUsd": 0.0014, "estimatedUsd": 0.0014, "usage": { "prompt_tokens": 85, "completion_tokens": 39, "total_tokens": 124 }, "latencyMs": 3128 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00078, "billedUsd": 0.00078, "estimatedUsd": 0.00078, "usage": { "prompt_tokens": 76, "completion_tokens": 16, "total_tokens": 92 }, "latencyMs": 2240 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.001175, "billedUsd": 0.001175, "estimatedUsd": 0.001175, "usage": { "prompt_tokens": 70, "completion_tokens": 33, "total_tokens": 103 }, "latencyMs": 2322 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0012799999999999999, "billedUsd": 0.00128, "estimatedUsd": 0.0012799999999999999, "usage": { "prompt_tokens": 76, "completion_tokens": 36, "total_tokens": 112 }, "latencyMs": 2973 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00113, "billedUsd": 0.00113, "estimatedUsd": 0.00113, "usage": { "prompt_tokens": 101, "completion_tokens": 25, "total_tokens": 126 }, "latencyMs": 3101 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.0014550000000000001, "billedUsd": 0.001455, "estimatedUsd": 0.0014550000000000001, "usage": { "prompt_tokens": 86, "completion_tokens": 41, "total_tokens": 127 }, "latencyMs": 4092 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0015, "billedUsd": 0.0015, "estimatedUsd": 0.0015, "usage": { "prompt_tokens": 85, "completion_tokens": 43, "total_tokens": 128 }, "latencyMs": 3235 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00066, "billedUsd": 0.00066, "estimatedUsd": 0.00066, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 1678 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.002615, "billedUsd": 0.002615, "estimatedUsd": 0.002615, "usage": { "prompt_tokens": 98, "completion_tokens": 85, "total_tokens": 183 }, "latencyMs": 4288 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0013050000000000002, "billedUsd": 0.001305, "estimatedUsd": 0.0013050000000000002, "usage": { "prompt_tokens": 96, "completion_tokens": 33, "total_tokens": 129 }, "latencyMs": 3955 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0008050000000000002, "billedUsd": 0.000805, "estimatedUsd": 0.0008050000000000002, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 1410 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.001935, "billedUsd": 0.001935, "estimatedUsd": 0.001935, "usage": { "prompt_tokens": 102, "completion_tokens": 57, "total_tokens": 159 }, "latencyMs": 3614 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006050000000000001, "billedUsd": 0.000605, "estimatedUsd": 0.0006050000000000001, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 1441 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001515, "billedUsd": 0.001515, "estimatedUsd": 0.001515, "usage": { "prompt_tokens": 123, "completion_tokens": 36, "total_tokens": 159 }, "latencyMs": 1750 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.001955, "billedUsd": 0.001955, "estimatedUsd": 0.001955, "usage": { "prompt_tokens": 91, "completion_tokens": 60, "total_tokens": 151 }, "latencyMs": 3238 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001145, "billedUsd": 0.001145, "estimatedUsd": 0.001145, "usage": { "prompt_tokens": 84, "completion_tokens": 29, "total_tokens": 113 }, "latencyMs": 2821 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00117, "billedUsd": 0.00117, "estimatedUsd": 0.00117, "usage": { "prompt_tokens": 89, "completion_tokens": 29, "total_tokens": 118 }, "latencyMs": 3260 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-opus-5", "openrouterId": "anthropic/claude-opus-5", "label": "Claude Opus 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0018449999999999999, "billedUsd": 0.001845, "estimatedUsd": 0.0018449999999999999, "usage": { "prompt_tokens": 99, "completion_tokens": 54, "total_tokens": 153 }, "latencyMs": 3434 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000432, "billedUsd": 0.000432, "estimatedUsd": 0.000432, "usage": { "prompt_tokens": 106, "completion_tokens": 22, "total_tokens": 128 }, "latencyMs": 3643 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0010600000000000002, "billedUsd": 0.00106, "estimatedUsd": 0.0010600000000000002, "usage": { "prompt_tokens": 125, "completion_tokens": 81, "total_tokens": 206 }, "latencyMs": 4668 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00022, "billedUsd": 0.00022, "estimatedUsd": 0.00022, "usage": { "prompt_tokens": 90, "completion_tokens": 4, "total_tokens": 94 }, "latencyMs": 1723 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00021, "billedUsd": 0.00021, "estimatedUsd": 0.00021, "usage": { "prompt_tokens": 90, "completion_tokens": 3, "total_tokens": 93 }, "latencyMs": 1438 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00025, "billedUsd": 0.00025, "estimatedUsd": 0.00025, "usage": { "prompt_tokens": 110, "completion_tokens": 3, "total_tokens": 113 }, "latencyMs": 1612 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000246, "billedUsd": 0.000246, "estimatedUsd": 0.000246, "usage": { "prompt_tokens": 108, "completion_tokens": 3, "total_tokens": 111 }, "latencyMs": 1683 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000248, "billedUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 109, "completion_tokens": 3, "total_tokens": 112 }, "latencyMs": 1652 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000258, "billedUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 104, "completion_tokens": 5, "total_tokens": 109 }, "latencyMs": 1579 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0008979999999999999, "billedUsd": 0.000898, "estimatedUsd": 0.0008979999999999999, "usage": { "prompt_tokens": 144, "completion_tokens": 61, "total_tokens": 205 }, "latencyMs": 1923 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0008939999999999999, "billedUsd": 0.000894, "estimatedUsd": 0.0008939999999999999, "usage": { "prompt_tokens": 127, "completion_tokens": 64, "total_tokens": 191 }, "latencyMs": 1798 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000282, "billedUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 116, "completion_tokens": 5, "total_tokens": 121 }, "latencyMs": 1761 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.000914, "billedUsd": 0.000914, "estimatedUsd": 0.000914, "usage": { "prompt_tokens": 132, "completion_tokens": 65, "total_tokens": 197 }, "latencyMs": 1760 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.000564, "billedUsd": 0.000564, "estimatedUsd": 0.000564, "usage": { "prompt_tokens": 117, "completion_tokens": 33, "total_tokens": 150 }, "latencyMs": 1593 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.000704, "billedUsd": 0.000704, "estimatedUsd": 0.000704, "usage": { "prompt_tokens": 107, "completion_tokens": 49, "total_tokens": 156 }, "latencyMs": 1796 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000272, "billedUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 111, "completion_tokens": 5, "total_tokens": 116 }, "latencyMs": 1612 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000258, "billedUsd": 0.000258, "estimatedUsd": 0.000258, "usage": { "prompt_tokens": 109, "completion_tokens": 4, "total_tokens": 113 }, "latencyMs": 1460 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00024799999999999996, "billedUsd": 0.000248, "estimatedUsd": 0.00024799999999999996, "usage": { "prompt_tokens": 104, "completion_tokens": 4, "total_tokens": 108 }, "latencyMs": 2040 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000272, "billedUsd": 0.000272, "estimatedUsd": 0.000272, "usage": { "prompt_tokens": 106, "completion_tokens": 6, "total_tokens": 112 }, "latencyMs": 1499 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 95, "completion_tokens": 4, "total_tokens": 99 }, "latencyMs": 2300 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000214, "billedUsd": 0.000214, "estimatedUsd": 0.000214, "usage": { "prompt_tokens": 92, "completion_tokens": 3, "total_tokens": 95 }, "latencyMs": 1729 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model: good extraction, failed multi-step arithmetic reasoning.", "spendUsd": 0.0006000000000000001, "billedUsd": 0.0006, "estimatedUsd": 0.0006000000000000001, "usage": { "prompt_tokens": 150, "completion_tokens": 30, "total_tokens": 180 }, "latencyMs": 1885 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00052, "billedUsd": 0.00052, "estimatedUsd": 0.00052, "usage": { "prompt_tokens": 155, "completion_tokens": 21, "total_tokens": 176 }, "latencyMs": 1781 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 9, "sentenceCount": 1 }, "output": "High verbosity increases cost, reducing dollars per correct answer.", "spendUsd": 0.00049, "billedUsd": 0.00049, "estimatedUsd": 0.00049, "usage": { "prompt_tokens": 125, "completion_tokens": 24, "total_tokens": 149 }, "latencyMs": 1421 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct Answer Comparison", "spendUsd": 0.000398, "billedUsd": 0.000398, "estimatedUsd": 0.000398, "usage": { "prompt_tokens": 114, "completion_tokens": 17, "total_tokens": 131 }, "latencyMs": 1870 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.000908, "billedUsd": 0.000908, "estimatedUsd": 0.000908, "usage": { "prompt_tokens": 149, "completion_tokens": 61, "total_tokens": 210 }, "latencyMs": 1838 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000262, "billedUsd": 0.000262, "estimatedUsd": 0.000262, "usage": { "prompt_tokens": 116, "completion_tokens": 3, "total_tokens": 119 }, "latencyMs": 1806 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0006559999999999999, "billedUsd": 0.000656, "estimatedUsd": 0.0006559999999999999, "usage": { "prompt_tokens": 108, "completion_tokens": 44, "total_tokens": 152 }, "latencyMs": 4753 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000286, "billedUsd": 0.000286, "estimatedUsd": 0.000286, "usage": { "prompt_tokens": 108, "completion_tokens": 7, "total_tokens": 115 }, "latencyMs": 1659 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001188, "billedUsd": 0.001188, "estimatedUsd": 0.001188, "usage": { "prompt_tokens": 124, "completion_tokens": 94, "total_tokens": 218 }, "latencyMs": 3710 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000264, "billedUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 97, "completion_tokens": 7, "total_tokens": 104 }, "latencyMs": 1670 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000248, "billedUsd": 0.000248, "estimatedUsd": 0.000248, "usage": { "prompt_tokens": 99, "completion_tokens": 5, "total_tokens": 104 }, "latencyMs": 1534 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00057, "billedUsd": 0.00057, "estimatedUsd": 0.00057, "usage": { "prompt_tokens": 85, "completion_tokens": 40, "total_tokens": 125 }, "latencyMs": 2308 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000182, "billedUsd": 0.000182, "estimatedUsd": 0.000182, "usage": { "prompt_tokens": 76, "completion_tokens": 3, "total_tokens": 79 }, "latencyMs": 1856 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00023, "billedUsd": 0.00023, "estimatedUsd": 0.00023, "usage": { "prompt_tokens": 70, "completion_tokens": 9, "total_tokens": 79 }, "latencyMs": 1616 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000212, "billedUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 76, "completion_tokens": 6, "total_tokens": 82 }, "latencyMs": 1566 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000282, "billedUsd": 0.000282, "estimatedUsd": 0.000282, "usage": { "prompt_tokens": 101, "completion_tokens": 8, "total_tokens": 109 }, "latencyMs": 1507 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000212, "billedUsd": 0.000212, "estimatedUsd": 0.000212, "usage": { "prompt_tokens": 86, "completion_tokens": 4, "total_tokens": 90 }, "latencyMs": 1893 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00039, "billedUsd": 0.00039, "estimatedUsd": 0.00039, "usage": { "prompt_tokens": 85, "completion_tokens": 22, "total_tokens": 107 }, "latencyMs": 4218 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.000264, "billedUsd": 0.000264, "estimatedUsd": 0.000264, "usage": { "prompt_tokens": 92, "completion_tokens": 8, "total_tokens": 100 }, "latencyMs": 1775 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.000326, "billedUsd": 0.000326, "estimatedUsd": 0.000326, "usage": { "prompt_tokens": 98, "completion_tokens": 13, "total_tokens": 111 }, "latencyMs": 1743 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000222, "billedUsd": 0.000222, "estimatedUsd": 0.000222, "usage": { "prompt_tokens": 96, "completion_tokens": 3, "total_tokens": 99 }, "latencyMs": 1271 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000322, "billedUsd": 0.000322, "estimatedUsd": 0.000322, "usage": { "prompt_tokens": 131, "completion_tokens": 6, "total_tokens": 137 }, "latencyMs": 1686 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000684, "billedUsd": 0.000684, "estimatedUsd": 0.000684, "usage": { "prompt_tokens": 102, "completion_tokens": 48, "total_tokens": 150 }, "latencyMs": 4267 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000242, "billedUsd": 0.000242, "estimatedUsd": 0.000242, "usage": { "prompt_tokens": 101, "completion_tokens": 4, "total_tokens": 105 }, "latencyMs": 1707 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001206, "billedUsd": 0.001206, "estimatedUsd": 0.001206, "usage": { "prompt_tokens": 123, "completion_tokens": 96, "total_tokens": 219 }, "latencyMs": 4145 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.000222, "billedUsd": 0.000222, "estimatedUsd": 0.000222, "usage": { "prompt_tokens": 91, "completion_tokens": 4, "total_tokens": 95 }, "latencyMs": 1799 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00021799999999999999, "billedUsd": 0.000218, "estimatedUsd": 0.00021799999999999999, "usage": { "prompt_tokens": 84, "completion_tokens": 5, "total_tokens": 89 }, "latencyMs": 1794 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000208, "billedUsd": 0.000208, "estimatedUsd": 0.000208, "usage": { "prompt_tokens": 89, "completion_tokens": 3, "total_tokens": 92 }, "latencyMs": 1671 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-sonnet-5", "openrouterId": "anthropic/claude-sonnet-5", "label": "Claude Sonnet 5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "Cost per correct answer", "spendUsd": 0.000298, "billedUsd": 0.000298, "estimatedUsd": 0.000298, "usage": { "prompt_tokens": 99, "completion_tokens": 10, "total_tokens": 109 }, "latencyMs": 1533 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.000099, "billedUsd": 0.000099, "estimatedUsd": 0.000099, "usage": { "prompt_tokens": 74, "completion_tokens": 5, "total_tokens": 79 }, "latencyMs": 853 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 6, "detail": "expected 8" }, "output": "6", "spendUsd": 0.000115, "billedUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 90, "completion_tokens": 5, "total_tokens": 95 }, "latencyMs": 989 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 62, "completion_tokens": 6, "total_tokens": 68 }, "latencyMs": 1020 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000087, "billedUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 1000 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "actual": 18, "detail": "expected 20" }, "output": "18", "spendUsd": 0.00009999999999999999, "billedUsd": 0.0001, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 75, "completion_tokens": 5, "total_tokens": 80 }, "latencyMs": 832 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00009499999999999999, "billedUsd": 0.000095, "estimatedUsd": 0.00009499999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 5, "total_tokens": 75 }, "latencyMs": 857 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000093, "billedUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 1084 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000104, "billedUsd": 0.000104, "estimatedUsd": 0.000104, "usage": { "prompt_tokens": 69, "completion_tokens": 7, "total_tokens": 76 }, "latencyMs": 1022 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.000375, "billedUsd": 0.000375, "estimatedUsd": 0.000375, "usage": { "prompt_tokens": 100, "completion_tokens": 55, "total_tokens": 155 }, "latencyMs": 1067 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.000371, "billedUsd": 0.000371, "estimatedUsd": 0.000371, "usage": { "prompt_tokens": 86, "completion_tokens": 57, "total_tokens": 143 }, "latencyMs": 1104 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.000094, "billedUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 1122 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.00038700000000000003, "billedUsd": 0.000387, "estimatedUsd": 0.00038700000000000003, "usage": { "prompt_tokens": 92, "completion_tokens": 59, "total_tokens": 151 }, "latencyMs": 942 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00021700000000000002, "billedUsd": 0.000217, "estimatedUsd": 0.00021700000000000002, "usage": { "prompt_tokens": 82, "completion_tokens": 27, "total_tokens": 109 }, "latencyMs": 907 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.00029299999999999997, "billedUsd": 0.000293, "estimatedUsd": 0.00029299999999999997, "usage": { "prompt_tokens": 73, "completion_tokens": 44, "total_tokens": 117 }, "latencyMs": 1037 }, { "taskId": "classify-001", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000094, "billedUsd": 0.000094, "estimatedUsd": 0.000094, "usage": { "prompt_tokens": 74, "completion_tokens": 4, "total_tokens": 78 }, "latencyMs": 997 }, { "taskId": "classify-002", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000091, "billedUsd": 0.000091, "estimatedUsd": 0.000091, "usage": { "prompt_tokens": 71, "completion_tokens": 4, "total_tokens": 75 }, "latencyMs": 907 }, { "taskId": "classify-003", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.000093, "billedUsd": 0.000093, "estimatedUsd": 0.000093, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 884 }, { "taskId": "classify-004", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.000089, "billedUsd": 0.000089, "estimatedUsd": 0.000089, "usage": { "prompt_tokens": 69, "completion_tokens": 4, "total_tokens": 73 }, "latencyMs": 827 }, { "taskId": "classify-005", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00007999999999999999, "billedUsd": 0.00008, "estimatedUsd": 0.00007999999999999999, "usage": { "prompt_tokens": 60, "completion_tokens": 4, "total_tokens": 64 }, "latencyMs": 1574 }, { "taskId": "classify-006", "category": "classification", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.000078, "billedUsd": 0.000078, "estimatedUsd": 0.000078, "usage": { "prompt_tokens": 58, "completion_tokens": 4, "total_tokens": 62 }, "latencyMs": 842 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction model failed complex arithmetic reasoning tasks.", "spendUsd": 0.000154, "billedUsd": 0.000154, "estimatedUsd": 0.000154, "usage": { "prompt_tokens": 89, "completion_tokens": 13, "total_tokens": 102 }, "latencyMs": 1140 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.000165, "billedUsd": 0.000165, "estimatedUsd": 0.000165, "usage": { "prompt_tokens": 95, "completion_tokens": 14, "total_tokens": 109 }, "latencyMs": 894 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity increases cost per correct answer despite model accuracy.", "spendUsd": 0.000156, "billedUsd": 0.000156, "estimatedUsd": 0.000156, "usage": { "prompt_tokens": 81, "completion_tokens": 15, "total_tokens": 96 }, "latencyMs": 1035 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost to Correct AI Benchmark Answers", "spendUsd": 0.000134, "billedUsd": 0.000134, "estimatedUsd": 0.000134, "usage": { "prompt_tokens": 74, "completion_tokens": 12, "total_tokens": 86 }, "latencyMs": 1082 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00028399999999999996, "billedUsd": 0.000284, "estimatedUsd": 0.00028399999999999996, "usage": { "prompt_tokens": 89, "completion_tokens": 39, "total_tokens": 128 }, "latencyMs": 969 }, { "taskId": "code-001", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000106, "billedUsd": 0.000106, "estimatedUsd": 0.000106, "usage": { "prompt_tokens": 81, "completion_tokens": 5, "total_tokens": 86 }, "latencyMs": 1229 }, { "taskId": "code-002", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000115, "billedUsd": 0.000115, "estimatedUsd": 0.000115, "usage": { "prompt_tokens": 80, "completion_tokens": 7, "total_tokens": 87 }, "latencyMs": 995 }, { "taskId": "code-003", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00009999999999999999, "billedUsd": 0.0001, "estimatedUsd": 0.00009999999999999999, "usage": { "prompt_tokens": 70, "completion_tokens": 6, "total_tokens": 76 }, "latencyMs": 953 }, { "taskId": "code-004", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.000299, "billedUsd": 0.000299, "estimatedUsd": 0.000299, "usage": { "prompt_tokens": 84, "completion_tokens": 43, "total_tokens": 127 }, "latencyMs": 1488 }, { "taskId": "code-005", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00010899999999999999, "billedUsd": 0.000109, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 9, "total_tokens": 73 }, "latencyMs": 1088 }, { "taskId": "code-006", "category": "coding", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.000088, "billedUsd": 0.000088, "estimatedUsd": 0.000088, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 919 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.000084, "billedUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 54, "completion_tokens": 6, "total_tokens": 60 }, "latencyMs": 936 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00007199999999999999, "billedUsd": 0.000072, "estimatedUsd": 0.00007199999999999999, "usage": { "prompt_tokens": 52, "completion_tokens": 4, "total_tokens": 56 }, "latencyMs": 933 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000096, "billedUsd": 0.000096, "estimatedUsd": 0.000096, "usage": { "prompt_tokens": 46, "completion_tokens": 10, "total_tokens": 56 }, "latencyMs": 5939 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.000077, "billedUsd": 0.000077, "estimatedUsd": 0.000077, "usage": { "prompt_tokens": 47, "completion_tokens": 6, "total_tokens": 53 }, "latencyMs": 1126 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00010899999999999999, "billedUsd": 0.000109, "estimatedUsd": 0.00010899999999999999, "usage": { "prompt_tokens": 69, "completion_tokens": 8, "total_tokens": 77 }, "latencyMs": 1132 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00007500000000000001, "billedUsd": 0.000075, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 981 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00007500000000000001, "billedUsd": 0.000075, "estimatedUsd": 0.00007500000000000001, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 1024 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00011300000000000001, "billedUsd": 0.000113, "estimatedUsd": 0.00011300000000000001, "usage": { "prompt_tokens": 68, "completion_tokens": 9, "total_tokens": 77 }, "latencyMs": 1234 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00013099999999999999, "billedUsd": 0.000131, "estimatedUsd": 0.00013099999999999999, "usage": { "prompt_tokens": 66, "completion_tokens": 13, "total_tokens": 79 }, "latencyMs": 924 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000087, "billedUsd": 0.000087, "estimatedUsd": 0.000087, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 871 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000116, "billedUsd": 0.000116, "estimatedUsd": 0.000116, "usage": { "prompt_tokens": 96, "completion_tokens": 4, "total_tokens": 100 }, "latencyMs": 933 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 72, "completion_tokens": 4, "total_tokens": 76 }, "latencyMs": 1133 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.000097, "billedUsd": 0.000097, "estimatedUsd": 0.000097, "usage": { "prompt_tokens": 67, "completion_tokens": 6, "total_tokens": 73 }, "latencyMs": 1167 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.000315, "billedUsd": 0.000315, "estimatedUsd": 0.000315, "usage": { "prompt_tokens": 85, "completion_tokens": 46, "total_tokens": 131 }, "latencyMs": 1104 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes.", "spendUsd": 0.000086, "billedUsd": 0.000086, "estimatedUsd": 0.000086, "usage": { "prompt_tokens": 61, "completion_tokens": 5, "total_tokens": 66 }, "latencyMs": 1074 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.000092, "billedUsd": 0.000092, "estimatedUsd": 0.000092, "usage": { "prompt_tokens": 57, "completion_tokens": 7, "total_tokens": 64 }, "latencyMs": 1123 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000084, "billedUsd": 0.000084, "estimatedUsd": 0.000084, "usage": { "prompt_tokens": 59, "completion_tokens": 5, "total_tokens": 64 }, "latencyMs": 927 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "claude-haiku-4.5", "openrouterId": "anthropic/claude-haiku-4.5", "label": "Claude Haiku 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000098, "billedUsd": 0.000098, "estimatedUsd": 0.000098, "usage": { "prompt_tokens": 63, "completion_tokens": 7, "total_tokens": 70 }, "latencyMs": 971 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0005279999999999999, "billedUsd": 0.000528, "estimatedUsd": 0.0005279999999999999, "usage": { "prompt_tokens": 64, "completion_tokens": 128, "total_tokens": 192 }, "latencyMs": 1527 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00098025, "billedUsd": 0.00098025, "estimatedUsd": 0.00098025, "usage": { "prompt_tokens": 87, "completion_tokens": 244, "total_tokens": 331 }, "latencyMs": 2428 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0008235, "billedUsd": 0.0008235, "estimatedUsd": 0.0008235, "usage": { "prompt_tokens": 58, "completion_tokens": 208, "total_tokens": 266 }, "latencyMs": 2001 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000618, "billedUsd": 0.000618, "estimatedUsd": 0.000618, "usage": { "prompt_tokens": 54, "completion_tokens": 154, "total_tokens": 208 }, "latencyMs": 1533 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00068475, "billedUsd": 0.00068475, "estimatedUsd": 0.00068475, "usage": { "prompt_tokens": 68, "completion_tokens": 169, "total_tokens": 237 }, "latencyMs": 1800 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0006742499999999999, "billedUsd": 0.00067425, "estimatedUsd": 0.0006742499999999999, "usage": { "prompt_tokens": 59, "completion_tokens": 168, "total_tokens": 227 }, "latencyMs": 1494 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00035625, "billedUsd": 0.00035625, "estimatedUsd": 0.00035625, "usage": { "prompt_tokens": 60, "completion_tokens": 83, "total_tokens": 143 }, "latencyMs": 1283 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00066375, "billedUsd": 0.00066375, "estimatedUsd": 0.00066375, "usage": { "prompt_tokens": 60, "completion_tokens": 165, "total_tokens": 225 }, "latencyMs": 1337 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.00178875, "billedUsd": 0.00178875, "estimatedUsd": 0.00178875, "usage": { "prompt_tokens": 95, "completion_tokens": 458, "total_tokens": 553 }, "latencyMs": 2862 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.003177, "billedUsd": 0.003177, "estimatedUsd": 0.003177, "usage": { "prompt_tokens": 76, "completion_tokens": 832, "total_tokens": 908 }, "latencyMs": 5206 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0009584999999999999, "billedUsd": 0.0009585, "estimatedUsd": 0.0009584999999999999, "usage": { "prompt_tokens": 63, "completion_tokens": 243, "total_tokens": 306 }, "latencyMs": 2300 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.0012907499999999998, "billedUsd": 0.00129075, "estimatedUsd": 0.0012907499999999998, "usage": { "prompt_tokens": 86, "completion_tokens": 327, "total_tokens": 413 }, "latencyMs": 2194 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00203475, "billedUsd": 0.00203475, "estimatedUsd": 0.00203475, "usage": { "prompt_tokens": 73, "completion_tokens": 528, "total_tokens": 601 }, "latencyMs": 3506 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0011775, "billedUsd": 0.0011775, "estimatedUsd": 0.0011775, "usage": { "prompt_tokens": 60, "completion_tokens": 302, "total_tokens": 362 }, "latencyMs": 2349 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0003945, "billedUsd": 0.0003945, "estimatedUsd": 0.0003945, "usage": { "prompt_tokens": 61, "completion_tokens": 93, "total_tokens": 154 }, "latencyMs": 1093 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0007920000000000001, "billedUsd": 0.000792, "estimatedUsd": 0.0007920000000000001, "usage": { "prompt_tokens": 56, "completion_tokens": 200, "total_tokens": 256 }, "latencyMs": 1803 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00047025, "billedUsd": 0.00047025, "estimatedUsd": 0.00047025, "usage": { "prompt_tokens": 57, "completion_tokens": 114, "total_tokens": 171 }, "latencyMs": 1019 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00077775, "billedUsd": 0.00077775, "estimatedUsd": 0.00077775, "usage": { "prompt_tokens": 57, "completion_tokens": 196, "total_tokens": 253 }, "latencyMs": 1567 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0009195, "billedUsd": 0.0009195, "estimatedUsd": 0.0009195, "usage": { "prompt_tokens": 51, "completion_tokens": 235, "total_tokens": 286 }, "latencyMs": 1689 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000603, "billedUsd": 0.000603, "estimatedUsd": 0.000603, "usage": { "prompt_tokens": 49, "completion_tokens": 151, "total_tokens": 200 }, "latencyMs": 1498 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model passed extraction but failed arithmetic.", "spendUsd": 0.003351, "billedUsd": 0.003351, "estimatedUsd": 0.003351, "usage": { "prompt_tokens": 78, "completion_tokens": 878, "total_tokens": 956 }, "latencyMs": 4407 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0011347500000000001, "billedUsd": 0.00113475, "estimatedUsd": 0.0011347500000000001, "usage": { "prompt_tokens": 83, "completion_tokens": 286, "total_tokens": 369 }, "latencyMs": 2000 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "High verbosity increases cost.", "spendUsd": 0.002649, "billedUsd": 0.002649, "estimatedUsd": 0.002649, "usage": { "prompt_tokens": 72, "completion_tokens": 692, "total_tokens": 764 }, "latencyMs": 3791 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct AI Answer", "spendUsd": 0.0026392499999999997, "billedUsd": 0.00263925, "estimatedUsd": 0.0026392499999999997, "usage": { "prompt_tokens": 64, "completion_tokens": 691, "total_tokens": 755 }, "latencyMs": 4257 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0013717500000000001, "billedUsd": 0.00137175, "estimatedUsd": 0.0013717500000000001, "usage": { "prompt_tokens": 79, "completion_tokens": 350, "total_tokens": 429 }, "latencyMs": 3373 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00112275, "billedUsd": 0.00112275, "estimatedUsd": 0.00112275, "usage": { "prompt_tokens": 72, "completion_tokens": 285, "total_tokens": 357 }, "latencyMs": 2265 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000666, "billedUsd": 0.000666, "estimatedUsd": 0.000666, "usage": { "prompt_tokens": 68, "completion_tokens": 164, "total_tokens": 232 }, "latencyMs": 1342 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00084225, "billedUsd": 0.00084225, "estimatedUsd": 0.00084225, "usage": { "prompt_tokens": 63, "completion_tokens": 212, "total_tokens": 275 }, "latencyMs": 1651 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0024232500000000005, "billedUsd": 0.00242325, "estimatedUsd": 0.0024232500000000005, "usage": { "prompt_tokens": 76, "completion_tokens": 631, "total_tokens": 707 }, "latencyMs": 3363 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0006299999999999999, "billedUsd": 0.00063, "estimatedUsd": 0.0006299999999999999, "usage": { "prompt_tokens": 55, "completion_tokens": 157, "total_tokens": 212 }, "latencyMs": 1527 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00038925, "billedUsd": 0.00038925, "estimatedUsd": 0.00038925, "usage": { "prompt_tokens": 59, "completion_tokens": 92, "total_tokens": 151 }, "latencyMs": 1167 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0005872499999999999, "billedUsd": 0.00058725, "estimatedUsd": 0.0005872499999999999, "usage": { "prompt_tokens": 43, "completion_tokens": 148, "total_tokens": 191 }, "latencyMs": 1317 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00057675, "billedUsd": 0.00057675, "estimatedUsd": 0.00057675, "usage": { "prompt_tokens": 44, "completion_tokens": 145, "total_tokens": 189 }, "latencyMs": 1256 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00056775, "billedUsd": 0.00056775, "estimatedUsd": 0.00056775, "usage": { "prompt_tokens": 37, "completion_tokens": 144, "total_tokens": 181 }, "latencyMs": 1646 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0005909999999999999, "billedUsd": 0.000591, "estimatedUsd": 0.0005909999999999999, "usage": { "prompt_tokens": 38, "completion_tokens": 150, "total_tokens": 188 }, "latencyMs": 1470 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00087825, "billedUsd": 0.00087825, "estimatedUsd": 0.00087825, "usage": { "prompt_tokens": 56, "completion_tokens": 223, "total_tokens": 279 }, "latencyMs": 1992 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00050775, "billedUsd": 0.00050775, "estimatedUsd": 0.00050775, "usage": { "prompt_tokens": 47, "completion_tokens": 126, "total_tokens": 173 }, "latencyMs": 1237 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0006165, "billedUsd": 0.0006165, "estimatedUsd": 0.0006165, "usage": { "prompt_tokens": 47, "completion_tokens": 155, "total_tokens": 202 }, "latencyMs": 1623 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0010530000000000001, "billedUsd": 0.001053, "estimatedUsd": 0.0010530000000000001, "usage": { "prompt_tokens": 64, "completion_tokens": 268, "total_tokens": 332 }, "latencyMs": 1978 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0006907500000000001, "billedUsd": 0.00069075, "estimatedUsd": 0.0006907500000000001, "usage": { "prompt_tokens": 56, "completion_tokens": 173, "total_tokens": 229 }, "latencyMs": 2422 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00089925, "billedUsd": 0.00089925, "estimatedUsd": 0.00089925, "usage": { "prompt_tokens": 49, "completion_tokens": 230, "total_tokens": 279 }, "latencyMs": 1597 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0010095, "billedUsd": 0.0010095, "estimatedUsd": 0.0010095, "usage": { "prompt_tokens": 91, "completion_tokens": 251, "total_tokens": 342 }, "latencyMs": 1884 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000615, "billedUsd": 0.000615, "estimatedUsd": 0.000615, "usage": { "prompt_tokens": 65, "completion_tokens": 151, "total_tokens": 216 }, "latencyMs": 1810 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0006180000000000001, "billedUsd": 0.000618, "estimatedUsd": 0.0006180000000000001, "usage": { "prompt_tokens": 59, "completion_tokens": 153, "total_tokens": 212 }, "latencyMs": 1526 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0020355, "billedUsd": 0.0020355, "estimatedUsd": 0.0020355, "usage": { "prompt_tokens": 74, "completion_tokens": 528, "total_tokens": 602 }, "latencyMs": 2952 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00087975, "billedUsd": 0.00087975, "estimatedUsd": 0.00087975, "usage": { "prompt_tokens": 53, "completion_tokens": 224, "total_tokens": 277 }, "latencyMs": 1818 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0006674999999999999, "billedUsd": 0.0006675, "estimatedUsd": 0.0006674999999999999, "usage": { "prompt_tokens": 50, "completion_tokens": 168, "total_tokens": 218 }, "latencyMs": 1935 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00051075, "billedUsd": 0.00051075, "estimatedUsd": 0.00051075, "usage": { "prompt_tokens": 51, "completion_tokens": 126, "total_tokens": 177 }, "latencyMs": 1568 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.6-flash", "openrouterId": "google/gemini-3.6-flash", "label": "Gemini 3.6 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.000975, "billedUsd": 0.000975, "estimatedUsd": 0.000975, "usage": { "prompt_tokens": 55, "completion_tokens": 249, "total_tokens": 304 }, "latencyMs": 2551 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0000267, "billedUsd": 0.0000267, "estimatedUsd": 0.0000267, "usage": { "prompt_tokens": 64, "completion_tokens": 3, "total_tokens": 67 }, "latencyMs": 441 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "actual": null, "detail": "expected 8" }, "output": "To find the total cost in cents: 1. **Input tokens cost:** - Rate: $0.30 per 1,000,000 tokens - Usage: 100,000 tokens (which is 0.1 million) - Cost = $0.30 * 0.1 = $0.03 2. **Output tokens cost:** - Rate: $2.50 per 1,000,000 tokens - Usage: 20,000 tokens (which is 0.02 million) - Cost = $2.50 * 0.02 = $0.05 3. **Total cost:** - Total in dollars = $0.03 + $0.05 = $0.08 - Total in cents = 0.08 * 100 = 8 8", "spendUsd": 0.0005511000000000001, "billedUsd": 0.0005511, "estimatedUsd": 0.0005511000000000001, "usage": { "prompt_tokens": 87, "completion_tokens": 210, "total_tokens": 297 }, "latencyMs": 870 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000029900000000000002, "billedUsd": 0.0000299, "estimatedUsd": 0.000029900000000000002, "usage": { "prompt_tokens": 58, "completion_tokens": 5, "total_tokens": 63 }, "latencyMs": 664 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.000021199999999999997, "billedUsd": 0.0000212, "estimatedUsd": 0.000021199999999999997, "usage": { "prompt_tokens": 54, "completion_tokens": 2, "total_tokens": 56 }, "latencyMs": 418 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.000025399999999999997, "billedUsd": 0.0000254, "estimatedUsd": 0.000025399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 2, "total_tokens": 70 }, "latencyMs": 496 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.0000227, "billedUsd": 0.0000227, "estimatedUsd": 0.0000227, "usage": { "prompt_tokens": 59, "completion_tokens": 2, "total_tokens": 61 }, "latencyMs": 714 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.000023, "billedUsd": 0.000023, "estimatedUsd": 0.000023, "usage": { "prompt_tokens": 60, "completion_tokens": 2, "total_tokens": 62 }, "latencyMs": 584 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.000028, "billedUsd": 0.000028, "estimatedUsd": 0.000028, "usage": { "prompt_tokens": 60, "completion_tokens": 4, "total_tokens": 64 }, "latencyMs": 499 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.00017099999999999998, "billedUsd": 0.000171, "estimatedUsd": 0.00017099999999999998, "usage": { "prompt_tokens": 95, "completion_tokens": 57, "total_tokens": 152 }, "latencyMs": 674 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.00015030000000000002, "billedUsd": 0.0001503, "estimatedUsd": 0.00015030000000000002, "usage": { "prompt_tokens": 76, "completion_tokens": 51, "total_tokens": 127 }, "latencyMs": 527 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0000214, "billedUsd": 0.0000214, "estimatedUsd": 0.0000214, "usage": { "prompt_tokens": 63, "completion_tokens": 1, "total_tokens": 64 }, "latencyMs": 504 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0001608, "billedUsd": 0.0001608, "estimatedUsd": 0.0001608, "usage": { "prompt_tokens": 86, "completion_tokens": 54, "total_tokens": 140 }, "latencyMs": 600 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0000844, "billedUsd": 0.0000844, "estimatedUsd": 0.0000844, "usage": { "prompt_tokens": 73, "completion_tokens": 25, "total_tokens": 98 }, "latencyMs": 563 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0001155, "billedUsd": 0.0001155, "estimatedUsd": 0.0001155, "usage": { "prompt_tokens": 60, "completion_tokens": 39, "total_tokens": 99 }, "latencyMs": 612 }, { "taskId": "classify-001", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000020799999999999997, "billedUsd": 0.0000208, "estimatedUsd": 0.000020799999999999997, "usage": { "prompt_tokens": 61, "completion_tokens": 1, "total_tokens": 62 }, "latencyMs": 965 }, { "taskId": "classify-002", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.000019299999999999998, "billedUsd": 0.0000193, "estimatedUsd": 0.000019299999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 1, "total_tokens": 57 }, "latencyMs": 653 }, { "taskId": "classify-003", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.0000221, "billedUsd": 0.0000221, "estimatedUsd": 0.0000221, "usage": { "prompt_tokens": 57, "completion_tokens": 2, "total_tokens": 59 }, "latencyMs": 370 }, { "taskId": "classify-004", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0000196, "billedUsd": 0.0000196, "estimatedUsd": 0.0000196, "usage": { "prompt_tokens": 57, "completion_tokens": 1, "total_tokens": 58 }, "latencyMs": 544 }, { "taskId": "classify-005", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000178, "billedUsd": 0.0000178, "estimatedUsd": 0.0000178, "usage": { "prompt_tokens": 51, "completion_tokens": 1, "total_tokens": 52 }, "latencyMs": 599 }, { "taskId": "classify-006", "category": "classification", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.000017199999999999998, "billedUsd": 0.0000172, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 673 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model passed extraction, failed arithmetic reasoning tests.", "spendUsd": 0.0000509, "billedUsd": 0.0000509, "estimatedUsd": 0.0000509, "usage": { "prompt_tokens": 78, "completion_tokens": 11, "total_tokens": 89 }, "latencyMs": 686 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0000499, "billedUsd": 0.0000499, "estimatedUsd": 0.0000499, "usage": { "prompt_tokens": 83, "completion_tokens": 10, "total_tokens": 93 }, "latencyMs": 563 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Excessive output verbosity compounds cost per correct answer.", "spendUsd": 0.0000491, "billedUsd": 0.0000491, "estimatedUsd": 0.0000491, "usage": { "prompt_tokens": 72, "completion_tokens": 11, "total_tokens": 83 }, "latencyMs": 483 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 6, "sentenceCount": 1 }, "output": "Cost and Correct AI Benchmark Comparison", "spendUsd": 0.0000342, "billedUsd": 0.0000342, "estimatedUsd": 0.0000342, "usage": { "prompt_tokens": 64, "completion_tokens": 6, "total_tokens": 70 }, "latencyMs": 861 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.00011119999999999998, "billedUsd": 0.0001112, "estimatedUsd": 0.00011119999999999998, "usage": { "prompt_tokens": 79, "completion_tokens": 35, "total_tokens": 114 }, "latencyMs": 452 }, { "taskId": "code-001", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0000266, "billedUsd": 0.0000266, "estimatedUsd": 0.0000266, "usage": { "prompt_tokens": 72, "completion_tokens": 2, "total_tokens": 74 }, "latencyMs": 687 }, { "taskId": "code-002", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000030399999999999997, "billedUsd": 0.0000304, "estimatedUsd": 0.000030399999999999997, "usage": { "prompt_tokens": 68, "completion_tokens": 4, "total_tokens": 72 }, "latencyMs": 533 }, { "taskId": "code-003", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000026399999999999998, "billedUsd": 0.0000264, "estimatedUsd": 0.000026399999999999998, "usage": { "prompt_tokens": 63, "completion_tokens": 3, "total_tokens": 66 }, "latencyMs": 560 }, { "taskId": "code-004", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{ \"correct\": 2, \"totalCost\": 0.06, \"costPerCorrect\": 0.03 }", "spendUsd": 0.00011029999999999998, "billedUsd": 0.0001103, "estimatedUsd": 0.00011029999999999998, "usage": { "prompt_tokens": 76, "completion_tokens": 35, "total_tokens": 111 }, "latencyMs": 500 }, { "taskId": "code-005", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.000029, "billedUsd": 0.000029, "estimatedUsd": 0.000029, "usage": { "prompt_tokens": 55, "completion_tokens": 5, "total_tokens": 60 }, "latencyMs": 528 }, { "taskId": "code-006", "category": "coding", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0000202, "billedUsd": 0.0000202, "estimatedUsd": 0.0000202, "usage": { "prompt_tokens": 59, "completion_tokens": 1, "total_tokens": 60 }, "latencyMs": 568 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": false, "grade": { "correct": false, "detail": "expected Luna" }, "output": "Sonnet", "spendUsd": 0.0000179, "billedUsd": 0.0000179, "estimatedUsd": 0.0000179, "usage": { "prompt_tokens": 43, "completion_tokens": 2, "total_tokens": 45 }, "latencyMs": 671 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0000157, "billedUsd": 0.0000157, "estimatedUsd": 0.0000157, "usage": { "prompt_tokens": 44, "completion_tokens": 1, "total_tokens": 45 }, "latencyMs": 405 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000026099999999999997, "billedUsd": 0.0000261, "estimatedUsd": 0.000026099999999999997, "usage": { "prompt_tokens": 37, "completion_tokens": 6, "total_tokens": 43 }, "latencyMs": 577 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000164, "billedUsd": 0.0000164, "estimatedUsd": 0.0000164, "usage": { "prompt_tokens": 38, "completion_tokens": 2, "total_tokens": 40 }, "latencyMs": 344 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000026799999999999998, "billedUsd": 0.0000268, "estimatedUsd": 0.000026799999999999998, "usage": { "prompt_tokens": 56, "completion_tokens": 4, "total_tokens": 60 }, "latencyMs": 592 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000016599999999999997, "billedUsd": 0.0000166, "estimatedUsd": 0.000016599999999999997, "usage": { "prompt_tokens": 47, "completion_tokens": 1, "total_tokens": 48 }, "latencyMs": 404 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.000016599999999999997, "billedUsd": 0.0000166, "estimatedUsd": 0.000016599999999999997, "usage": { "prompt_tokens": 47, "completion_tokens": 1, "total_tokens": 48 }, "latencyMs": 740 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0000367, "billedUsd": 0.0000367, "estimatedUsd": 0.0000367, "usage": { "prompt_tokens": 64, "completion_tokens": 7, "total_tokens": 71 }, "latencyMs": 459 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000393, "billedUsd": 0.0000393, "estimatedUsd": 0.0000393, "usage": { "prompt_tokens": 56, "completion_tokens": 9, "total_tokens": 65 }, "latencyMs": 717 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.000017199999999999998, "billedUsd": 0.0000172, "estimatedUsd": 0.000017199999999999998, "usage": { "prompt_tokens": 49, "completion_tokens": 1, "total_tokens": 50 }, "latencyMs": 504 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0000323, "billedUsd": 0.0000323, "estimatedUsd": 0.0000323, "usage": { "prompt_tokens": 91, "completion_tokens": 2, "total_tokens": 93 }, "latencyMs": 592 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.000021999999999999996, "billedUsd": 0.000022, "estimatedUsd": 0.000021999999999999996, "usage": { "prompt_tokens": 65, "completion_tokens": 1, "total_tokens": 66 }, "latencyMs": 658 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0000277, "billedUsd": 0.0000277, "estimatedUsd": 0.0000277, "usage": { "prompt_tokens": 59, "completion_tokens": 4, "total_tokens": 63 }, "latencyMs": 1118 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.0001372, "billedUsd": 0.0001372, "estimatedUsd": 0.0001372, "usage": { "prompt_tokens": 74, "completion_tokens": 46, "total_tokens": 120 }, "latencyMs": 482 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0000184, "billedUsd": 0.0000184, "estimatedUsd": 0.0000184, "usage": { "prompt_tokens": 53, "completion_tokens": 1, "total_tokens": 54 }, "latencyMs": 650 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0000225, "billedUsd": 0.0000225, "estimatedUsd": 0.0000225, "usage": { "prompt_tokens": 50, "completion_tokens": 3, "total_tokens": 53 }, "latencyMs": 708 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0000228, "billedUsd": 0.0000228, "estimatedUsd": 0.0000228, "usage": { "prompt_tokens": 51, "completion_tokens": 3, "total_tokens": 54 }, "latencyMs": 498 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "gemini-3.5-flash-lite", "openrouterId": "google/gemini-3.5-flash-lite", "label": "Gemini 3.5 Flash-Lite", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0000265, "billedUsd": 0.0000265, "estimatedUsd": 0.0000265, "usage": { "prompt_tokens": 55, "completion_tokens": 4, "total_tokens": 59 }, "latencyMs": 504 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00022572, "billedUsd": 0.0001974, "estimatedUsd": 0.00022572, "usage": { "prompt_tokens": 66, "completion_tokens": 92, "total_tokens": 158 }, "latencyMs": 1186 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0005847599999999999, "billedUsd": 0.0004758, "estimatedUsd": 0.0005847599999999999, "usage": { "prompt_tokens": 82, "completion_tokens": 268, "total_tokens": 350 }, "latencyMs": 2761 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.00018083999999999998, "billedUsd": 0.0001602, "estimatedUsd": 0.00018083999999999998, "usage": { "prompt_tokens": 58, "completion_tokens": 72, "total_tokens": 130 }, "latencyMs": 787 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00015642, "billedUsd": 0.00013728, "estimatedUsd": 0.00015642, "usage": { "prompt_tokens": 135, "completion_tokens": 34, "total_tokens": 169 }, "latencyMs": 1692 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00024948, "billedUsd": 0.00044, "estimatedUsd": 0.00024948, "usage": { "prompt_tokens": 69, "completion_tokens": 103, "total_tokens": 172 }, "latencyMs": 2908 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00020394, "billedUsd": 0.0001797, "estimatedUsd": 0.00020394, "usage": { "prompt_tokens": 63, "completion_tokens": 82, "total_tokens": 145 }, "latencyMs": 1079 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00017556, "billedUsd": 0.00014886135, "estimatedUsd": 0.00017556, "usage": { "prompt_tokens": 62, "completion_tokens": 68, "total_tokens": 130 }, "latencyMs": 2226 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00018546, "billedUsd": 0.0001653, "estimatedUsd": 0.00018546, "usage": { "prompt_tokens": 62, "completion_tokens": 73, "total_tokens": 135 }, "latencyMs": 1079 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576}", "spendUsd": 0.0004917, "billedUsd": 0.0004089, "estimatedUsd": 0.0004917, "usage": { "prompt_tokens": 91, "completion_tokens": 218, "total_tokens": 309 }, "latencyMs": 2311 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40}", "spendUsd": 0.00049038, "billedUsd": 0.0004023, "estimatedUsd": 0.00049038, "usage": { "prompt_tokens": 77, "completion_tokens": 222, "total_tokens": 299 }, "latencyMs": 3004 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.00017225999999999998, "billedUsd": 0.0001569, "estimatedUsd": 0.00017225999999999998, "usage": { "prompt_tokens": 66, "completion_tokens": 65, "total_tokens": 131 }, "latencyMs": 1339 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true}", "spendUsd": 0.00045078, "billedUsd": 0.0003723, "estimatedUsd": 0.00045078, "usage": { "prompt_tokens": 77, "completion_tokens": 202, "total_tokens": 279 }, "latencyMs": 1978 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00031284000000000005, "billedUsd": 0.0002682, "estimatedUsd": 0.00031284000000000005, "usage": { "prompt_tokens": 78, "completion_tokens": 132, "total_tokens": 210 }, "latencyMs": 2298 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\"}", "spendUsd": 0.00030228000000000004, "billedUsd": 0.000271214235, "estimatedUsd": 0.00030228000000000004, "usage": { "prompt_tokens": 65, "completion_tokens": 131, "total_tokens": 196 }, "latencyMs": 3443 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.00004884, "billedUsd": 0.00007455, "estimatedUsd": 0.00004884, "usage": { "prompt_tokens": 65, "completion_tokens": 3, "total_tokens": 68 }, "latencyMs": 719 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00019074, "billedUsd": 0.000384, "estimatedUsd": 0.00019074, "usage": { "prompt_tokens": 142, "completion_tokens": 49, "total_tokens": 191 }, "latencyMs": 1532 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00014124, "billedUsd": 0.0001314, "estimatedUsd": 0.00014124, "usage": { "prompt_tokens": 61, "completion_tokens": 51, "total_tokens": 112 }, "latencyMs": 1242 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00012671999999999998, "billedUsd": 0.00012, "estimatedUsd": 0.00012671999999999998, "usage": { "prompt_tokens": 60, "completion_tokens": 44, "total_tokens": 104 }, "latencyMs": 1258 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0001815, "billedUsd": 0.0001599, "estimatedUsd": 0.0001815, "usage": { "prompt_tokens": 56, "completion_tokens": 73, "total_tokens": 129 }, "latencyMs": 1865 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00014784, "billedUsd": 0.0001332, "estimatedUsd": 0.00014784, "usage": { "prompt_tokens": 53, "completion_tokens": 57, "total_tokens": 110 }, "latencyMs": 951 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "The cheaper model's extraction succeeded but arithmetic failed.", "spendUsd": 0.00072072, "billedUsd": 0.0005784, "estimatedUsd": 0.00072072, "usage": { "prompt_tokens": 81, "completion_tokens": 337, "total_tokens": 418 }, "latencyMs": 3958 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00033858, "billedUsd": 0.0002913, "estimatedUsd": 0.00033858, "usage": { "prompt_tokens": 87, "completion_tokens": 142, "total_tokens": 229 }, "latencyMs": 1683 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 7, "sentenceCount": 1 }, "output": "High verbosity raises cost per correct answer.", "spendUsd": 0.00081972, "billedUsd": 0.000651, "estimatedUsd": 0.00081972, "usage": { "prompt_tokens": 75, "completion_tokens": 389, "total_tokens": 464 }, "latencyMs": 3537 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost per Correct Answer", "spendUsd": 0.00005478, "billedUsd": 0.0000819, "estimatedUsd": 0.00005478, "usage": { "prompt_tokens": 68, "completion_tokens": 5, "total_tokens": 73 }, "latencyMs": 530 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0004026, "billedUsd": 0.0003378, "estimatedUsd": 0.0004026, "usage": { "prompt_tokens": 82, "completion_tokens": 176, "total_tokens": 258 }, "latencyMs": 1376 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.00033396, "billedUsd": 0.0002814, "estimatedUsd": 0.00033396, "usage": { "prompt_tokens": 71, "completion_tokens": 145, "total_tokens": 216 }, "latencyMs": 2274 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00033198000000000006, "billedUsd": 0.0002811, "estimatedUsd": 0.00033198000000000006, "usage": { "prompt_tokens": 74, "completion_tokens": 143, "total_tokens": 217 }, "latencyMs": 2465 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000049500000000000004, "billedUsd": 0.00007455, "estimatedUsd": 0.000049500000000000004, "usage": { "prompt_tokens": 63, "completion_tokens": 4, "total_tokens": 67 }, "latencyMs": 4540 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00045144, "billedUsd": 0.0003732, "estimatedUsd": 0.00045144, "usage": { "prompt_tokens": 78, "completion_tokens": 202, "total_tokens": 280 }, "latencyMs": 3104 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00029633999999999997, "billedUsd": 0.0002481, "estimatedUsd": 0.00029633999999999997, "usage": { "prompt_tokens": 59, "completion_tokens": 130, "total_tokens": 189 }, "latencyMs": 1808 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00015972, "billedUsd": 0.0002477, "estimatedUsd": 0.00015972, "usage": { "prompt_tokens": 140, "completion_tokens": 34, "total_tokens": 174 }, "latencyMs": 1410 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.00026532, "billedUsd": 0.0002202, "estimatedUsd": 0.00026532, "usage": { "prompt_tokens": 48, "completion_tokens": 118, "total_tokens": 166 }, "latencyMs": 2497 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00014058, "billedUsd": 0.0001257, "estimatedUsd": 0.00014058, "usage": { "prompt_tokens": 48, "completion_tokens": 55, "total_tokens": 103 }, "latencyMs": 1032 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00014387999999999998, "billedUsd": 0.0001254, "estimatedUsd": 0.00014387999999999998, "usage": { "prompt_tokens": 41, "completion_tokens": 59, "total_tokens": 100 }, "latencyMs": 890 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00018216, "billedUsd": 0.0001548, "estimatedUsd": 0.00018216, "usage": { "prompt_tokens": 42, "completion_tokens": 78, "total_tokens": 120 }, "latencyMs": 853 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00019668, "billedUsd": 0.0001734, "estimatedUsd": 0.00019668, "usage": { "prompt_tokens": 61, "completion_tokens": 79, "total_tokens": 140 }, "latencyMs": 1990 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00017622, "billedUsd": 0.0001539, "estimatedUsd": 0.00017622, "usage": { "prompt_tokens": 51, "completion_tokens": 72, "total_tokens": 123 }, "latencyMs": 1502 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00035442000000000004, "billedUsd": 0.0002889, "estimatedUsd": 0.00035442000000000004, "usage": { "prompt_tokens": 51, "completion_tokens": 162, "total_tokens": 213 }, "latencyMs": 1467 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.00023166000000000001, "billedUsd": 0.0001995, "estimatedUsd": 0.00023166000000000001, "usage": { "prompt_tokens": 60, "completion_tokens": 97, "total_tokens": 157 }, "latencyMs": 2240 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000594, "billedUsd": 0.000104, "estimatedUsd": 0.0000594, "usage": { "prompt_tokens": 60, "completion_tokens": 10, "total_tokens": 70 }, "latencyMs": 1034 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.00019865999999999997, "billedUsd": 0.0001713, "estimatedUsd": 0.00019865999999999997, "usage": { "prompt_tokens": 52, "completion_tokens": 83, "total_tokens": 135 }, "latencyMs": 791 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00034584, "billedUsd": 0.0002964, "estimatedUsd": 0.00034584, "usage": { "prompt_tokens": 86, "completion_tokens": 146, "total_tokens": 232 }, "latencyMs": 1120 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00027522, "billedUsd": 0.0002349, "estimatedUsd": 0.00027522, "usage": { "prompt_tokens": 66, "completion_tokens": 117, "total_tokens": 183 }, "latencyMs": 1298 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00024024, "billedUsd": 0.0002064, "estimatedUsd": 0.00024024, "usage": { "prompt_tokens": 61, "completion_tokens": 101, "total_tokens": 162 }, "latencyMs": 1045 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00032076, "billedUsd": 0.00058812, "estimatedUsd": 0.00032076, "usage": { "prompt_tokens": 72, "completion_tokens": 138, "total_tokens": 210 }, "latencyMs": 3143 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.0002739, "billedUsd": 0.00039355, "estimatedUsd": 0.0002739, "usage": { "prompt_tokens": 136, "completion_tokens": 93, "total_tokens": 229 }, "latencyMs": 2062 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00031085999999999993, "billedUsd": 0.0002559, "estimatedUsd": 0.00031085999999999993, "usage": { "prompt_tokens": 51, "completion_tokens": 140, "total_tokens": 191 }, "latencyMs": 2456 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00019998, "billedUsd": 0.0001731, "estimatedUsd": 0.00019998, "usage": { "prompt_tokens": 54, "completion_tokens": 83, "total_tokens": 137 }, "latencyMs": 3082 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-pro", "openrouterId": "deepseek/deepseek-v4-pro", "label": "DeepSeek V4 Pro", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00032868000000000003, "billedUsd": 0.0006048, "estimatedUsd": 0.00032868000000000003, "usage": { "prompt_tokens": 138, "completion_tokens": 120, "total_tokens": 258 }, "latencyMs": 3595 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0000324, "billedUsd": 0.000031641, "estimatedUsd": 0.0000324, "usage": { "prompt_tokens": 92, "completion_tokens": 31, "total_tokens": 123 }, "latencyMs": 432 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.00005819999999999999, "billedUsd": 0.00011532, "estimatedUsd": 0.00005819999999999999, "usage": { "prompt_tokens": 108, "completion_tokens": 70, "total_tokens": 178 }, "latencyMs": 652 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.000029399999999999996, "billedUsd": 0.000028672, "estimatedUsd": 0.000029399999999999996, "usage": { "prompt_tokens": 84, "completion_tokens": 28, "total_tokens": 112 }, "latencyMs": 455 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.0000363, "billedUsd": 0.000037046, "estimatedUsd": 0.0000363, "usage": { "prompt_tokens": 82, "completion_tokens": 40, "total_tokens": 122 }, "latencyMs": 528 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00004785, "billedUsd": 0.0000862, "estimatedUsd": 0.00004785, "usage": { "prompt_tokens": 95, "completion_tokens": 56, "total_tokens": 151 }, "latencyMs": 415 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000026549999999999998, "billedUsd": 0.000044528, "estimatedUsd": 0.000026549999999999998, "usage": { "prompt_tokens": 89, "completion_tokens": 22, "total_tokens": 111 }, "latencyMs": 1236 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.00003075, "billedUsd": 0.0000615, "estimatedUsd": 0.00003075, "usage": { "prompt_tokens": 89, "completion_tokens": 29, "total_tokens": 118 }, "latencyMs": 871 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.0000336, "billedUsd": 0.000033374, "estimatedUsd": 0.0000336, "usage": { "prompt_tokens": 88, "completion_tokens": 34, "total_tokens": 122 }, "latencyMs": 539 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.00010334999999999999, "billedUsd": 0.000113685, "estimatedUsd": 0.00010334999999999999, "usage": { "prompt_tokens": 117, "completion_tokens": 143, "total_tokens": 260 }, "latencyMs": 1122 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.00009585, "billedUsd": 0.000106419, "estimatedUsd": 0.00009585, "usage": { "prompt_tokens": 103, "completion_tokens": 134, "total_tokens": 237 }, "latencyMs": 1545 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0000426, "billedUsd": 0.00003044, "estimatedUsd": 0.0000426, "usage": { "prompt_tokens": 92, "completion_tokens": 48, "total_tokens": 140 }, "latencyMs": 494 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00005984999999999999, "billedUsd": 0.0001197, "estimatedUsd": 0.00005984999999999999, "usage": { "prompt_tokens": 103, "completion_tokens": 74, "total_tokens": 177 }, "latencyMs": 382 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0000864, "billedUsd": 0.0000923, "estimatedUsd": 0.0000864, "usage": { "prompt_tokens": 104, "completion_tokens": 118, "total_tokens": 222 }, "latencyMs": 662 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.00006585, "billedUsd": 0.00004987, "estimatedUsd": 0.00006585, "usage": { "prompt_tokens": 91, "completion_tokens": 87, "total_tokens": 178 }, "latencyMs": 939 }, { "taskId": "classify-001", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000024449999999999998, "billedUsd": 0.000042832, "estimatedUsd": 0.000024449999999999998, "usage": { "prompt_tokens": 91, "completion_tokens": 18, "total_tokens": 109 }, "latencyMs": 462 }, { "taskId": "classify-002", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00002895, "billedUsd": 0.000046928, "estimatedUsd": 0.00002895, "usage": { "prompt_tokens": 89, "completion_tokens": 26, "total_tokens": 115 }, "latencyMs": 610 }, { "taskId": "classify-003", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00003825, "billedUsd": 0.000034425, "estimatedUsd": 0.00003825, "usage": { "prompt_tokens": 87, "completion_tokens": 42, "total_tokens": 129 }, "latencyMs": 1452 }, { "taskId": "classify-004", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0000387, "billedUsd": 0.000039603, "estimatedUsd": 0.0000387, "usage": { "prompt_tokens": 86, "completion_tokens": 43, "total_tokens": 129 }, "latencyMs": 469 }, { "taskId": "classify-005", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0000345, "billedUsd": 0.000034901, "estimatedUsd": 0.0000345, "usage": { "prompt_tokens": 82, "completion_tokens": 37, "total_tokens": 119 }, "latencyMs": 637 }, { "taskId": "classify-006", "category": "classification", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.00003165, "billedUsd": 0.000034815, "estimatedUsd": 0.00003165, "usage": { "prompt_tokens": 79, "completion_tokens": 33, "total_tokens": 112 }, "latencyMs": 826 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper extraction model failed arithmetic, good for parsing.", "spendUsd": 0.00013665000000000001, "billedUsd": 0.00015098, "estimatedUsd": 0.00013665000000000001, "usage": { "prompt_tokens": 107, "completion_tokens": 201, "total_tokens": 308 }, "latencyMs": 1023 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.00005715, "billedUsd": 0.000059544, "estimatedUsd": 0.00005715, "usage": { "prompt_tokens": 113, "completion_tokens": 67, "total_tokens": 180 }, "latencyMs": 606 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 10, "sentenceCount": 1 }, "output": "High verbosity raises token cost, hurting dollars per correct answer.", "spendUsd": 0.00012375, "billedUsd": 0.0002475, "estimatedUsd": 0.00012375, "usage": { "prompt_tokens": 101, "completion_tokens": 181, "total_tokens": 282 }, "latencyMs": 661 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Correct Answers, Real Cost", "spendUsd": 0.0000687, "billedUsd": 0.000087688, "estimatedUsd": 0.0000687, "usage": { "prompt_tokens": 94, "completion_tokens": 91, "total_tokens": 185 }, "latencyMs": 1917 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0001128, "billedUsd": 0.00008806, "estimatedUsd": 0.0001128, "usage": { "prompt_tokens": 108, "completion_tokens": 161, "total_tokens": 269 }, "latencyMs": 1266 }, { "taskId": "code-001", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.000042749999999999996, "billedUsd": 0.000043596, "estimatedUsd": 0.000042749999999999996, "usage": { "prompt_tokens": 97, "completion_tokens": 47, "total_tokens": 144 }, "latencyMs": 456 }, { "taskId": "code-002", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.000051599999999999994, "billedUsd": 0.00005865, "estimatedUsd": 0.000051599999999999994, "usage": { "prompt_tokens": 100, "completion_tokens": 61, "total_tokens": 161 }, "latencyMs": 597 }, { "taskId": "code-003", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.000049949999999999994, "billedUsd": 0.000067928, "estimatedUsd": 0.000049949999999999994, "usage": { "prompt_tokens": 89, "completion_tokens": 61, "total_tokens": 150 }, "latencyMs": 866 }, { "taskId": "code-004", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.00006720000000000001, "billedUsd": 0.00007566, "estimatedUsd": 0.00006720000000000001, "usage": { "prompt_tokens": 104, "completion_tokens": 86, "total_tokens": 190 }, "latencyMs": 596 }, { "taskId": "code-005", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.00003675, "billedUsd": 0.00005392, "estimatedUsd": 0.00003675, "usage": { "prompt_tokens": 85, "completion_tokens": 40, "total_tokens": 125 }, "latencyMs": 835 }, { "taskId": "code-006", "category": "coding", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00003465, "billedUsd": 0.00002409, "estimatedUsd": 0.00003465, "usage": { "prompt_tokens": 87, "completion_tokens": 36, "total_tokens": 123 }, "latencyMs": 1062 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0000387, "billedUsd": 0.000053648, "estimatedUsd": 0.0000387, "usage": { "prompt_tokens": 74, "completion_tokens": 46, "total_tokens": 120 }, "latencyMs": 600 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.00003149999999999999, "billedUsd": 0.00002218, "estimatedUsd": 0.00003149999999999999, "usage": { "prompt_tokens": 74, "completion_tokens": 34, "total_tokens": 108 }, "latencyMs": 692 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.000054449999999999995, "billedUsd": 0.000059895, "estimatedUsd": 0.000054449999999999995, "usage": { "prompt_tokens": 67, "completion_tokens": 74, "total_tokens": 141 }, "latencyMs": 892 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0000312, "billedUsd": 0.00003567, "estimatedUsd": 0.0000312, "usage": { "prompt_tokens": 68, "completion_tokens": 35, "total_tokens": 103 }, "latencyMs": 680 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.000049649999999999994, "billedUsd": 0.00005618, "estimatedUsd": 0.000049649999999999994, "usage": { "prompt_tokens": 87, "completion_tokens": 61, "total_tokens": 148 }, "latencyMs": 653 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.000033749999999999994, "billedUsd": 0.000049304, "estimatedUsd": 0.000033749999999999994, "usage": { "prompt_tokens": 77, "completion_tokens": 37, "total_tokens": 114 }, "latencyMs": 634 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0000435, "billedUsd": 0.000087, "estimatedUsd": 0.0000435, "usage": { "prompt_tokens": 78, "completion_tokens": 53, "total_tokens": 131 }, "latencyMs": 350 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0000441, "billedUsd": 0.000046038, "estimatedUsd": 0.0000441, "usage": { "prompt_tokens": 86, "completion_tokens": 52, "total_tokens": 138 }, "latencyMs": 428 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0000555, "billedUsd": 0.000111, "estimatedUsd": 0.0000555, "usage": { "prompt_tokens": 86, "completion_tokens": 71, "total_tokens": 157 }, "latencyMs": 448 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0000393, "billedUsd": 0.00002846, "estimatedUsd": 0.0000393, "usage": { "prompt_tokens": 78, "completion_tokens": 46, "total_tokens": 124 }, "latencyMs": 798 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.000048, "billedUsd": 0.00003752, "estimatedUsd": 0.000048, "usage": { "prompt_tokens": 112, "completion_tokens": 52, "total_tokens": 164 }, "latencyMs": 725 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.00010515000000000001, "billedUsd": 0.0002103, "estimatedUsd": 0.00010515000000000001, "usage": { "prompt_tokens": 93, "completion_tokens": 152, "total_tokens": 245 }, "latencyMs": 564 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00003945, "billedUsd": 0.00004513, "estimatedUsd": 0.00003945, "usage": { "prompt_tokens": 87, "completion_tokens": 44, "total_tokens": 131 }, "latencyMs": 816 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.00006645, "billedUsd": 0.0001329, "estimatedUsd": 0.00006645, "usage": { "prompt_tokens": 99, "completion_tokens": 86, "total_tokens": 185 }, "latencyMs": 606 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00004005, "billedUsd": 0.00004235, "estimatedUsd": 0.00004005, "usage": { "prompt_tokens": 83, "completion_tokens": 46, "total_tokens": 129 }, "latencyMs": 409 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.00006075, "billedUsd": 0.000066825, "estimatedUsd": 0.00006075, "usage": { "prompt_tokens": 77, "completion_tokens": 82, "total_tokens": 159 }, "latencyMs": 1190 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.000036, "billedUsd": 0.0000396, "estimatedUsd": 0.000036, "usage": { "prompt_tokens": 80, "completion_tokens": 40, "total_tokens": 120 }, "latencyMs": 1144 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "deepseek-v4-flash", "openrouterId": "deepseek/deepseek-v4.1-flash", "label": "DeepSeek V4.1 Flash", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.00004455, "billedUsd": 0.0000506, "estimatedUsd": 0.00004455, "usage": { "prompt_tokens": 85, "completion_tokens": 53, "total_tokens": 138 }, "latencyMs": 591 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.00041740000000000006, "billedUsd": 0.00021832, "estimatedUsd": 0.00041740000000000006, "usage": { "prompt_tokens": 75, "completion_tokens": 71, "total_tokens": 146 }, "latencyMs": 510 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.0005146, "billedUsd": 0.00026938, "estimatedUsd": 0.0005146, "usage": { "prompt_tokens": 91, "completion_tokens": 88, "total_tokens": 179 }, "latencyMs": 631 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0003168, "billedUsd": 0.00016434, "estimatedUsd": 0.0003168, "usage": { "prompt_tokens": 66, "completion_tokens": 51, "total_tokens": 117 }, "latencyMs": 542 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.00017460000000000002, "billedUsd": 0.00008628, "estimatedUsd": 0.00017460000000000002, "usage": { "prompt_tokens": 65, "completion_tokens": 19, "total_tokens": 84 }, "latencyMs": 514 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00046560000000000004, "billedUsd": 0.00024438, "estimatedUsd": 0.00046560000000000004, "usage": { "prompt_tokens": 78, "completion_tokens": 81, "total_tokens": 159 }, "latencyMs": 512 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.00037220000000000005, "billedUsd": 0.00019406, "estimatedUsd": 0.00037220000000000005, "usage": { "prompt_tokens": 71, "completion_tokens": 62, "total_tokens": 133 }, "latencyMs": 632 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.0006934, "billedUsd": 0.00037072, "estimatedUsd": 0.0006934, "usage": { "prompt_tokens": 71, "completion_tokens": 135, "total_tokens": 206 }, "latencyMs": 719 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.00040740000000000004, "billedUsd": 0.00021342, "estimatedUsd": 0.00040740000000000004, "usage": { "prompt_tokens": 71, "completion_tokens": 70, "total_tokens": 141 }, "latencyMs": 684 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{ \"model\": \"Gemini Flash\", \"inputPerM\": 0.30, \"outputPerM\": 2.50, \"context\": 1048576 }", "spendUsd": 0.0016124000000000002, "billedUsd": 0.00087152, "estimatedUsd": 0.0016124000000000002, "usage": { "prompt_tokens": 102, "completion_tokens": 334, "total_tokens": 436 }, "latencyMs": 1298 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{ \"provider\": \"Z.ai\", \"model\": \"GLM-5.2\", \"inputPerM\": 1.40, \"outputPerM\": 4.40 }", "spendUsd": 0.0012644000000000002, "billedUsd": 0.00068252, "estimatedUsd": 0.0012644000000000002, "usage": { "prompt_tokens": 86, "completion_tokens": 260, "total_tokens": 346 }, "latencyMs": 819 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.0003852, "billedUsd": 0.00020076, "estimatedUsd": 0.0003852, "usage": { "prompt_tokens": 74, "completion_tokens": 64, "total_tokens": 138 }, "latencyMs": 558 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.00029200000000000005, "billedUsd": 0.0001477, "estimatedUsd": 0.00029200000000000005, "usage": { "prompt_tokens": 86, "completion_tokens": 39, "total_tokens": 125 }, "latencyMs": 456 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0005664, "billedUsd": 0.00029892, "estimatedUsd": 0.0005664, "usage": { "prompt_tokens": 84, "completion_tokens": 102, "total_tokens": 186 }, "latencyMs": 826 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{ \"provider\": \"Anthropic\", \"model\": \"Claude Haiku 4.5\", \"status\": \"active\" }", "spendUsd": 0.0009954, "billedUsd": 0.00053652, "estimatedUsd": 0.0009954, "usage": { "prompt_tokens": 73, "completion_tokens": 203, "total_tokens": 276 }, "latencyMs": 881 }, { "taskId": "classify-001", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0012358000000000002, "billedUsd": 0.00066844, "estimatedUsd": 0.0012358000000000002, "usage": { "prompt_tokens": 75, "completion_tokens": 257, "total_tokens": 332 }, "latencyMs": 1022 }, { "taskId": "classify-002", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.0010528, "billedUsd": 0.0010528, "estimatedUsd": 0.0010528, "usage": { "prompt_tokens": 70, "completion_tokens": 217, "total_tokens": 287 }, "latencyMs": 1000 }, { "taskId": "classify-003", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.00111, "billedUsd": 0.00111, "estimatedUsd": 0.00111, "usage": { "prompt_tokens": 70, "completion_tokens": 230, "total_tokens": 300 }, "latencyMs": 1033 }, { "taskId": "classify-004", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.0009824, "billedUsd": 0.00052982, "estimatedUsd": 0.0009824, "usage": { "prompt_tokens": 70, "completion_tokens": 201, "total_tokens": 271 }, "latencyMs": 905 }, { "taskId": "classify-005", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.00022600000000000002, "billedUsd": 0.0001147, "estimatedUsd": 0.00022600000000000002, "usage": { "prompt_tokens": 64, "completion_tokens": 31, "total_tokens": 95 }, "latencyMs": 711 }, { "taskId": "classify-006", "category": "classification", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "No", "spendUsd": 0.0001088, "billedUsd": 0.00005054, "estimatedUsd": 0.0001088, "usage": { "prompt_tokens": 62, "completion_tokens": 5, "total_tokens": 67 }, "latencyMs": 371 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model succeeded at extraction but failed arithmetic.", "spendUsd": 0.0005366, "billedUsd": 0.00028148, "estimatedUsd": 0.0005366, "usage": { "prompt_tokens": 91, "completion_tokens": 93, "total_tokens": 184 }, "latencyMs": 658 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0011112000000000001, "billedUsd": 0.00059676, "estimatedUsd": 0.0011112000000000001, "usage": { "prompt_tokens": 96, "completion_tokens": 222, "total_tokens": 318 }, "latencyMs": 841 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "High verbosity directly increases cost.", "spendUsd": 0.0011884, "billedUsd": 0.00064132, "estimatedUsd": 0.0011884, "usage": { "prompt_tokens": 82, "completion_tokens": 244, "total_tokens": 326 }, "latencyMs": 1038 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct AI Answer", "spendUsd": 0.0022154, "billedUsd": 0.00120692, "estimatedUsd": 0.0022154, "usage": { "prompt_tokens": 77, "completion_tokens": 479, "total_tokens": 556 }, "latencyMs": 1456 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0014636000000000002, "billedUsd": 0.00079148, "estimatedUsd": 0.0014636000000000002, "usage": { "prompt_tokens": 90, "completion_tokens": 304, "total_tokens": 394 }, "latencyMs": 1111 }, { "taskId": "code-001", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.0001282, "billedUsd": 0.00005866, "estimatedUsd": 0.0001282, "usage": { "prompt_tokens": 79, "completion_tokens": 4, "total_tokens": 83 }, "latencyMs": 339 }, { "taskId": "code-002", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0003408, "billedUsd": 0.00017544, "estimatedUsd": 0.0003408, "usage": { "prompt_tokens": 80, "completion_tokens": 52, "total_tokens": 132 }, "latencyMs": 467 }, { "taskId": "code-003", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.0008136, "billedUsd": 0.00043668, "estimatedUsd": 0.0008136, "usage": { "prompt_tokens": 72, "completion_tokens": 162, "total_tokens": 234 }, "latencyMs": 798 }, { "taskId": "code-004", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0006134000000000001, "billedUsd": 0.00032492, "estimatedUsd": 0.0006134000000000001, "usage": { "prompt_tokens": 83, "completion_tokens": 113, "total_tokens": 196 }, "latencyMs": 750 }, { "taskId": "code-005", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0012232000000000002, "billedUsd": 0.00066286, "estimatedUsd": 0.0012232000000000002, "usage": { "prompt_tokens": 66, "completion_tokens": 257, "total_tokens": 323 }, "latencyMs": 1216 }, { "taskId": "code-006", "category": "coding", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00027100000000000003, "billedUsd": 0.0004065, "estimatedUsd": 0.00027100000000000003, "usage": { "prompt_tokens": 71, "completion_tokens": 39, "total_tokens": 110 }, "latencyMs": 499 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0007354, "billedUsd": 0.00039592, "estimatedUsd": 0.0007354, "usage": { "prompt_tokens": 57, "completion_tokens": 149, "total_tokens": 206 }, "latencyMs": 618 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.0005726, "billedUsd": 0.00030638, "estimatedUsd": 0.0005726, "usage": { "prompt_tokens": 57, "completion_tokens": 112, "total_tokens": 169 }, "latencyMs": 625 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0001668, "billedUsd": 0.00008424, "estimatedUsd": 0.0001668, "usage": { "prompt_tokens": 50, "completion_tokens": 22, "total_tokens": 72 }, "latencyMs": 527 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0008722000000000001, "billedUsd": 0.00047206, "estimatedUsd": 0.0008722000000000001, "usage": { "prompt_tokens": 51, "completion_tokens": 182, "total_tokens": 233 }, "latencyMs": 699 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.0003342, "billedUsd": 0.00017346, "estimatedUsd": 0.0003342, "usage": { "prompt_tokens": 69, "completion_tokens": 54, "total_tokens": 123 }, "latencyMs": 466 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00025120000000000003, "billedUsd": 0.00012916, "estimatedUsd": 0.00025120000000000003, "usage": { "prompt_tokens": 60, "completion_tokens": 38, "total_tokens": 98 }, "latencyMs": 688 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.00037440000000000005, "billedUsd": 0.00019692, "estimatedUsd": 0.00037440000000000005, "usage": { "prompt_tokens": 60, "completion_tokens": 66, "total_tokens": 126 }, "latencyMs": 635 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.0003342, "billedUsd": 0.00017346, "estimatedUsd": 0.0003342, "usage": { "prompt_tokens": 69, "completion_tokens": 54, "total_tokens": 123 }, "latencyMs": 531 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.0003870000000000001, "billedUsd": 0.0002025, "estimatedUsd": 0.0003870000000000001, "usage": { "prompt_tokens": 69, "completion_tokens": 66, "total_tokens": 135 }, "latencyMs": 764 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0007174000000000001, "billedUsd": 0.00038512, "estimatedUsd": 0.0007174000000000001, "usage": { "prompt_tokens": 63, "completion_tokens": 143, "total_tokens": 206 }, "latencyMs": 618 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0002122, "billedUsd": 0.00010246, "estimatedUsd": 0.0002122, "usage": { "prompt_tokens": 95, "completion_tokens": 18, "total_tokens": 113 }, "latencyMs": 441 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0002942, "billedUsd": 0.00015056, "estimatedUsd": 0.0002942, "usage": { "prompt_tokens": 75, "completion_tokens": 43, "total_tokens": 118 }, "latencyMs": 393 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00043100000000000007, "billedUsd": 0.0002267, "estimatedUsd": 0.00043100000000000007, "usage": { "prompt_tokens": 69, "completion_tokens": 76, "total_tokens": 145 }, "latencyMs": 594 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{ \"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004 }", "spendUsd": 0.000505, "billedUsd": 0.0002525, "estimatedUsd": 0.000505, "usage": { "prompt_tokens": 81, "completion_tokens": 89, "total_tokens": 170 }, "latencyMs": 1304 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "Yes", "spendUsd": 0.0008272, "billedUsd": 0.00044506, "estimatedUsd": 0.0008272, "usage": { "prompt_tokens": 66, "completion_tokens": 167, "total_tokens": 233 }, "latencyMs": 1185 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0003348, "billedUsd": 0.00017514, "estimatedUsd": 0.0003348, "usage": { "prompt_tokens": 60, "completion_tokens": 57, "total_tokens": 117 }, "latencyMs": 693 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.00045779999999999996, "billedUsd": 0.0004578, "estimatedUsd": 0.00045779999999999996, "usage": { "prompt_tokens": 63, "completion_tokens": 84, "total_tokens": 147 }, "latencyMs": 613 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "z-ai-glm-5.2", "openrouterId": "z-ai/glm-5.2", "label": "GLM-5.2", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.0002976, "billedUsd": 0.00015348, "estimatedUsd": 0.0002976, "usage": { "prompt_tokens": 68, "completion_tokens": 46, "total_tokens": 114 }, "latencyMs": 449 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.0012079999999999999, "billedUsd": 0.001016, "estimatedUsd": 0.0012079999999999999, "usage": { "prompt_tokens": 271, "completion_tokens": 111, "total_tokens": 382 }, "latencyMs": 1604 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.002704, "billedUsd": 0.002512, "estimatedUsd": 0.002704, "usage": { "prompt_tokens": 287, "completion_tokens": 355, "total_tokens": 642 }, "latencyMs": 3864 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0016539999999999999, "billedUsd": 0.001462, "estimatedUsd": 0.0016539999999999999, "usage": { "prompt_tokens": 263, "completion_tokens": 188, "total_tokens": 451 }, "latencyMs": 2579 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001522, "billedUsd": 0.00133, "estimatedUsd": 0.001522, "usage": { "prompt_tokens": 263, "completion_tokens": 166, "total_tokens": 429 }, "latencyMs": 2342 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.00147, "billedUsd": 0.001278, "estimatedUsd": 0.00147, "usage": { "prompt_tokens": 276, "completion_tokens": 153, "total_tokens": 429 }, "latencyMs": 2173 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.001352, "billedUsd": 0.00116, "estimatedUsd": 0.001352, "usage": { "prompt_tokens": 268, "completion_tokens": 136, "total_tokens": 404 }, "latencyMs": 2122 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.0012599999999999998, "billedUsd": 0.001068, "estimatedUsd": 0.0012599999999999998, "usage": { "prompt_tokens": 267, "completion_tokens": 121, "total_tokens": 388 }, "latencyMs": 1767 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.002214, "billedUsd": 0.002022, "estimatedUsd": 0.002214, "usage": { "prompt_tokens": 267, "completion_tokens": 280, "total_tokens": 547 }, "latencyMs": 3373 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.0022619999999999997, "billedUsd": 0.00207, "estimatedUsd": 0.0022619999999999997, "usage": { "prompt_tokens": 297, "completion_tokens": 278, "total_tokens": 575 }, "latencyMs": 3056 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.002296, "billedUsd": 0.002104, "estimatedUsd": 0.002296, "usage": { "prompt_tokens": 281, "completion_tokens": 289, "total_tokens": 570 }, "latencyMs": 3205 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.001676, "billedUsd": 0.001484, "estimatedUsd": 0.001676, "usage": { "prompt_tokens": 271, "completion_tokens": 189, "total_tokens": 460 }, "latencyMs": 2507 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{\"model\":\"deepseek-v4-flash\",\"prompt\":120,\"completion\":38,\"latency_ms\":910,\"ok\":true}", "spendUsd": 0.002114, "billedUsd": 0.001922, "estimatedUsd": 0.002114, "usage": { "prompt_tokens": 286, "completion_tokens": 257, "total_tokens": 543 }, "latencyMs": 2912 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.00149, "billedUsd": 0.001298, "estimatedUsd": 0.00149, "usage": { "prompt_tokens": 283, "completion_tokens": 154, "total_tokens": 437 }, "latencyMs": 1842 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.001732, "billedUsd": 0.00154, "estimatedUsd": 0.001732, "usage": { "prompt_tokens": 269, "completion_tokens": 199, "total_tokens": 468 }, "latencyMs": 2168 }, { "taskId": "classify-001", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.0017699999999999999, "billedUsd": 0.001578, "estimatedUsd": 0.0017699999999999999, "usage": { "prompt_tokens": 270, "completion_tokens": 205, "total_tokens": 475 }, "latencyMs": 2912 }, { "taskId": "classify-002", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.001954, "billedUsd": 0.001762, "estimatedUsd": 0.001954, "usage": { "prompt_tokens": 266, "completion_tokens": 237, "total_tokens": 503 }, "latencyMs": 2985 }, { "taskId": "classify-003", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.001876, "billedUsd": 0.001684, "estimatedUsd": 0.001876, "usage": { "prompt_tokens": 266, "completion_tokens": 224, "total_tokens": 490 }, "latencyMs": 3356 }, { "taskId": "classify-004", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.001548, "billedUsd": 0.001356, "estimatedUsd": 0.001548, "usage": { "prompt_tokens": 267, "completion_tokens": 169, "total_tokens": 436 }, "latencyMs": 2110 }, { "taskId": "classify-005", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001294, "billedUsd": 0.001102, "estimatedUsd": 0.001294, "usage": { "prompt_tokens": 260, "completion_tokens": 129, "total_tokens": 389 }, "latencyMs": 1836 }, { "taskId": "classify-006", "category": "classification", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.00177, "billedUsd": 0.001578, "estimatedUsd": 0.00177, "usage": { "prompt_tokens": 258, "completion_tokens": 209, "total_tokens": 467 }, "latencyMs": 3138 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model passed extraction but failed arithmetic reasoning.", "spendUsd": 0.004906, "billedUsd": 0.004714, "estimatedUsd": 0.004906, "usage": { "prompt_tokens": 287, "completion_tokens": 722, "total_tokens": 1009 }, "latencyMs": 8530 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.002024, "billedUsd": 0.001832, "estimatedUsd": 0.002024, "usage": { "prompt_tokens": 292, "completion_tokens": 240, "total_tokens": 532 }, "latencyMs": 2913 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "High verbosity compounds cost.", "spendUsd": 0.006334, "billedUsd": 0.006142, "estimatedUsd": 0.006334, "usage": { "prompt_tokens": 278, "completion_tokens": 963, "total_tokens": 1241 }, "latencyMs": 13380 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 5, "sentenceCount": 1 }, "output": "Cost Per Correct AI Answer", "spendUsd": 0.004902, "billedUsd": 0.00471, "estimatedUsd": 0.004902, "usage": { "prompt_tokens": 273, "completion_tokens": 726, "total_tokens": 999 }, "latencyMs": 8880 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.002956, "billedUsd": 0.002764, "estimatedUsd": 0.002956, "usage": { "prompt_tokens": 287, "completion_tokens": 397, "total_tokens": 684 }, "latencyMs": 4335 }, { "taskId": "code-001", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.002162, "billedUsd": 0.00197, "estimatedUsd": 0.002162, "usage": { "prompt_tokens": 280, "completion_tokens": 267, "total_tokens": 547 }, "latencyMs": 3245 }, { "taskId": "code-002", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.00234, "billedUsd": 0.002148, "estimatedUsd": 0.00234, "usage": { "prompt_tokens": 279, "completion_tokens": 297, "total_tokens": 576 }, "latencyMs": 3151 }, { "taskId": "code-003", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00208, "billedUsd": 0.001888, "estimatedUsd": 0.00208, "usage": { "prompt_tokens": 269, "completion_tokens": 257, "total_tokens": 526 }, "latencyMs": 5704 }, { "taskId": "code-004", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.0020940000000000004, "billedUsd": 0.001902, "estimatedUsd": 0.0020940000000000004, "usage": { "prompt_tokens": 282, "completion_tokens": 255, "total_tokens": 537 }, "latencyMs": 3041 }, { "taskId": "code-005", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.0018080000000000001, "billedUsd": 0.001616, "estimatedUsd": 0.0018080000000000001, "usage": { "prompt_tokens": 262, "completion_tokens": 214, "total_tokens": 476 }, "latencyMs": 2861 }, { "taskId": "code-006", "category": "coding", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.00128, "billedUsd": 0.001088, "estimatedUsd": 0.00128, "usage": { "prompt_tokens": 268, "completion_tokens": 124, "total_tokens": 392 }, "latencyMs": 1715 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.001718, "billedUsd": 0.001526, "estimatedUsd": 0.001718, "usage": { "prompt_tokens": 253, "completion_tokens": 202, "total_tokens": 455 }, "latencyMs": 2569 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.001448, "billedUsd": 0.001256, "estimatedUsd": 0.001448, "usage": { "prompt_tokens": 253, "completion_tokens": 157, "total_tokens": 410 }, "latencyMs": 2261 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.0021060000000000002, "billedUsd": 0.001914, "estimatedUsd": 0.0021060000000000002, "usage": { "prompt_tokens": 246, "completion_tokens": 269, "total_tokens": 515 }, "latencyMs": 2945 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.0014299999999999998, "billedUsd": 0.001238, "estimatedUsd": 0.0014299999999999998, "usage": { "prompt_tokens": 247, "completion_tokens": 156, "total_tokens": 403 }, "latencyMs": 2042 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.002398, "billedUsd": 0.002206, "estimatedUsd": 0.002398, "usage": { "prompt_tokens": 266, "completion_tokens": 311, "total_tokens": 577 }, "latencyMs": 3699 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.002192, "billedUsd": 0.002, "estimatedUsd": 0.002192, "usage": { "prompt_tokens": 256, "completion_tokens": 280, "total_tokens": 536 }, "latencyMs": 3156 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.0018180000000000002, "billedUsd": 0.001626, "estimatedUsd": 0.0018180000000000002, "usage": { "prompt_tokens": 255, "completion_tokens": 218, "total_tokens": 473 }, "latencyMs": 2889 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001396, "billedUsd": 0.001204, "estimatedUsd": 0.001396, "usage": { "prompt_tokens": 269, "completion_tokens": 143, "total_tokens": 412 }, "latencyMs": 1826 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.00204, "billedUsd": 0.001848, "estimatedUsd": 0.00204, "usage": { "prompt_tokens": 264, "completion_tokens": 252, "total_tokens": 516 }, "latencyMs": 3080 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.001412, "billedUsd": 0.00122, "estimatedUsd": 0.001412, "usage": { "prompt_tokens": 259, "completion_tokens": 149, "total_tokens": 408 }, "latencyMs": 1979 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.0022, "billedUsd": 0.002008, "estimatedUsd": 0.0022, "usage": { "prompt_tokens": 296, "completion_tokens": 268, "total_tokens": 564 }, "latencyMs": 3641 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.0035960000000000002, "billedUsd": 0.003404, "estimatedUsd": 0.0035960000000000002, "usage": { "prompt_tokens": 271, "completion_tokens": 509, "total_tokens": 780 }, "latencyMs": 7029 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.0014320000000000001, "billedUsd": 0.00124, "estimatedUsd": 0.0014320000000000001, "usage": { "prompt_tokens": 266, "completion_tokens": 150, "total_tokens": 416 }, "latencyMs": 1966 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\":0.001,\"outputCost\":0.003,\"totalCost\":0.004}", "spendUsd": 0.002316, "billedUsd": 0.002124, "estimatedUsd": 0.002316, "usage": { "prompt_tokens": 279, "completion_tokens": 293, "total_tokens": 572 }, "latencyMs": 3277 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001628, "billedUsd": 0.001436, "estimatedUsd": 0.001628, "usage": { "prompt_tokens": 262, "completion_tokens": 184, "total_tokens": 446 }, "latencyMs": 2643 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.0010739999999999999, "billedUsd": 0.000882, "estimatedUsd": 0.0010739999999999999, "usage": { "prompt_tokens": 258, "completion_tokens": 93, "total_tokens": 351 }, "latencyMs": 1627 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.0011840000000000002, "billedUsd": 0.000992, "estimatedUsd": 0.0011840000000000002, "usage": { "prompt_tokens": 259, "completion_tokens": 111, "total_tokens": 370 }, "latencyMs": 1743 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "grok-4.6", "openrouterId": "x-ai/grok-4.6", "label": "Grok 4.6", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.002262, "billedUsd": 0.00207, "estimatedUsd": 0.002262, "usage": { "prompt_tokens": 264, "completion_tokens": 289, "total_tokens": 553 }, "latencyMs": 5047 }, { "taskId": "math-001", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 186, "detail": "expected 186" }, "output": "186", "spendUsd": 0.001094, "billedUsd": 0.0008764, "estimatedUsd": 0.001094, "usage": { "prompt_tokens": 271, "completion_tokens": 92, "total_tokens": 363 }, "latencyMs": 1687 }, { "taskId": "math-002", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 8, "detail": "expected 8" }, "output": "8", "spendUsd": 0.001588, "billedUsd": 0.0013704, "estimatedUsd": 0.001588, "usage": { "prompt_tokens": 287, "completion_tokens": 169, "total_tokens": 456 }, "latencyMs": 2954 }, { "taskId": "math-003", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 48000, "detail": "expected 48000" }, "output": "48000", "spendUsd": 0.0011020000000000001, "billedUsd": 0.0008844, "estimatedUsd": 0.0011020000000000001, "usage": { "prompt_tokens": 263, "completion_tokens": 96, "total_tokens": 359 }, "latencyMs": 1931 }, { "taskId": "math-004", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 50, "detail": "expected 50" }, "output": "50", "spendUsd": 0.001174, "billedUsd": 0.0009564, "estimatedUsd": 0.001174, "usage": { "prompt_tokens": 263, "completion_tokens": 108, "total_tokens": 371 }, "latencyMs": 2266 }, { "taskId": "math-005", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 20, "detail": "expected 20" }, "output": "20", "spendUsd": 0.001278, "billedUsd": 0.0010604, "estimatedUsd": 0.001278, "usage": { "prompt_tokens": 276, "completion_tokens": 121, "total_tokens": 397 }, "latencyMs": 2154 }, { "taskId": "math-006", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 18, "detail": "expected 18" }, "output": "18", "spendUsd": 0.000914, "billedUsd": 0.0006964, "estimatedUsd": 0.000914, "usage": { "prompt_tokens": 268, "completion_tokens": 63, "total_tokens": 331 }, "latencyMs": 1338 }, { "taskId": "math-007", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 74, "detail": "expected 74" }, "output": "74", "spendUsd": 0.001086, "billedUsd": 0.0008684, "estimatedUsd": 0.001086, "usage": { "prompt_tokens": 267, "completion_tokens": 92, "total_tokens": 359 }, "latencyMs": 1953 }, { "taskId": "math-008", "category": "reasoning", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 0.03, "detail": "expected 0.03" }, "output": "0.03", "spendUsd": 0.001104, "billedUsd": 0.0008864, "estimatedUsd": 0.001104, "usage": { "prompt_tokens": 267, "completion_tokens": 95, "total_tokens": 362 }, "latencyMs": 1871 }, { "taskId": "extract-001", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "Gemini Flash", "inputPerM": 0.3, "outputPerM": 2.5, "context": 1048576 }, "missing": [] }, "output": "{\"model\":\"Gemini Flash\",\"inputPerM\":0.30,\"outputPerM\":2.50,\"context\":1048576}", "spendUsd": 0.001914, "billedUsd": 0.0016964, "estimatedUsd": 0.001914, "usage": { "prompt_tokens": 297, "completion_tokens": 220, "total_tokens": 517 }, "latencyMs": 3145 }, { "taskId": "extract-002", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Z.ai", "model": "GLM-5.2", "inputPerM": 1.4, "outputPerM": 4.4 }, "missing": [] }, "output": "{\"provider\":\"Z.ai\",\"model\":\"GLM-5.2\",\"inputPerM\":1.40,\"outputPerM\":4.40}", "spendUsd": 0.0013, "billedUsd": 0.0010824, "estimatedUsd": 0.0013, "usage": { "prompt_tokens": 281, "completion_tokens": 123, "total_tokens": 404 }, "latencyMs": 1942 }, { "taskId": "extract-003", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Pro" }, "output": "Pro", "spendUsd": 0.001094, "billedUsd": 0.0008764, "estimatedUsd": 0.001094, "usage": { "prompt_tokens": 271, "completion_tokens": 92, "total_tokens": 363 }, "latencyMs": 1891 }, { "taskId": "extract-004", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "model": "deepseek-v4-flash", "prompt": 120, "completion": 38, "latency_ms": 910, "ok": true }, "missing": [] }, "output": "{ \"model\": \"deepseek-v4-flash\", \"prompt\": 120, \"completion\": 38, \"latency_ms\": 910, \"ok\": true }", "spendUsd": 0.0018260000000000001, "billedUsd": 0.0016084, "estimatedUsd": 0.0018260000000000001, "usage": { "prompt_tokens": 286, "completion_tokens": 209, "total_tokens": 495 }, "latencyMs": 3013 }, { "taskId": "extract-005", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^gpt-5\\.6-luna\\s*,\\s*claude-sonnet-5\\s*,\\s*gemini-2\\.5-flash$" }, "output": "gpt-5.6-luna, claude-sonnet-5, gemini-2.5-flash", "spendUsd": 0.0015079999999999998, "billedUsd": 0.0012904, "estimatedUsd": 0.0015079999999999998, "usage": { "prompt_tokens": 283, "completion_tokens": 157, "total_tokens": 440 }, "latencyMs": 2316 }, { "taskId": "extract-006", "category": "extraction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "provider": "Anthropic", "model": "Claude Haiku 4.5", "status": "active" }, "missing": [] }, "output": "{\"provider\":\"Anthropic\",\"model\":\"Claude Haiku 4.5\",\"status\":\"active\"}", "spendUsd": 0.001084, "billedUsd": 0.0008664, "estimatedUsd": 0.001084, "usage": { "prompt_tokens": 269, "completion_tokens": 91, "total_tokens": 360 }, "latencyMs": 1512 }, { "taskId": "classify-001", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected pricing" }, "output": "pricing", "spendUsd": 0.000876, "billedUsd": 0.0006584, "estimatedUsd": 0.000876, "usage": { "prompt_tokens": 270, "completion_tokens": 56, "total_tokens": 326 }, "latencyMs": 1314 }, { "taskId": "classify-002", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected launch" }, "output": "launch", "spendUsd": 0.00133, "billedUsd": 0.0011124, "estimatedUsd": 0.00133, "usage": { "prompt_tokens": 266, "completion_tokens": 133, "total_tokens": 399 }, "latencyMs": 2493 }, { "taskId": "classify-003", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected outage" }, "output": "outage", "spendUsd": 0.001018, "billedUsd": 0.0008004, "estimatedUsd": 0.001018, "usage": { "prompt_tokens": 266, "completion_tokens": 81, "total_tokens": 347 }, "latencyMs": 1721 }, { "taskId": "classify-004", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected tutorial" }, "output": "tutorial", "spendUsd": 0.00129, "billedUsd": 0.0010724, "estimatedUsd": 0.00129, "usage": { "prompt_tokens": 267, "completion_tokens": 126, "total_tokens": 393 }, "latencyMs": 2373 }, { "taskId": "classify-005", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.001, "billedUsd": 0.0007824, "estimatedUsd": 0.001, "usage": { "prompt_tokens": 260, "completion_tokens": 80, "total_tokens": 340 }, "latencyMs": 1581 }, { "taskId": "classify-006", "category": "classification", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected no" }, "output": "no", "spendUsd": 0.0010799999999999998, "billedUsd": 0.0008624, "estimatedUsd": 0.0010799999999999998, "usage": { "prompt_tokens": 258, "completion_tokens": 94, "total_tokens": 352 }, "latencyMs": 1978 }, { "taskId": "summ-001", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 8, "sentenceCount": 1 }, "output": "Cheaper model passed extraction but failed arithmetic tasks", "spendUsd": 0.0032619999999999997, "billedUsd": 0.0030444, "estimatedUsd": 0.0032619999999999997, "usage": { "prompt_tokens": 287, "completion_tokens": 448, "total_tokens": 735 }, "latencyMs": 6292 }, { "taskId": "summ-002", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^ai api pricing\\s*,\\s*cheapest ai api\\s*,\\s*token calculator$" }, "output": "ai api pricing, cheapest ai api, token calculator", "spendUsd": 0.0012200000000000002, "billedUsd": 0.00122, "estimatedUsd": 0.0012200000000000002, "usage": { "prompt_tokens": 292, "completion_tokens": 106, "total_tokens": 398 }, "latencyMs": 1785 }, { "taskId": "summ-003", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "High verbosity compounds cost.", "spendUsd": 0.0039640000000000005, "billedUsd": 0.0037464, "estimatedUsd": 0.0039640000000000005, "usage": { "prompt_tokens": 278, "completion_tokens": 568, "total_tokens": 846 }, "latencyMs": 8557 }, { "taskId": "summ-004", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 4, "sentenceCount": 1 }, "output": "Cost per Correct Answer", "spendUsd": 0.0018960000000000001, "billedUsd": 0.0016784, "estimatedUsd": 0.0018960000000000001, "usage": { "prompt_tokens": 273, "completion_tokens": 225, "total_tokens": 498 }, "latencyMs": 3871 }, { "taskId": "summ-005", "category": "summarization", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "missing": [], "constraintFailures": [], "wordCount": 28, "sentenceCount": 1 }, "output": "OpenRouter is used for routing, but costs are computed from AI Pricing Guru's direct-provider list prices so the metric reflects what API buyers see on provider pricing pages.", "spendUsd": 0.0017559999999999997, "billedUsd": 0.0015384, "estimatedUsd": 0.0017559999999999997, "usage": { "prompt_tokens": 287, "completion_tokens": 197, "total_tokens": 484 }, "latencyMs": 3202 }, { "taskId": "code-001", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 12" }, "output": "12", "spendUsd": 0.001346, "billedUsd": 0.0011284, "estimatedUsd": 0.001346, "usage": { "prompt_tokens": 280, "completion_tokens": 131, "total_tokens": 411 }, "latencyMs": 2336 }, { "taskId": "code-002", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 0.85" }, "output": "0.85", "spendUsd": 0.0014759999999999999, "billedUsd": 0.0012584, "estimatedUsd": 0.0014759999999999999, "usage": { "prompt_tokens": 279, "completion_tokens": 153, "total_tokens": 432 }, "latencyMs": 2363 }, { "taskId": "code-003", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern \\binputPerM\\b" }, "output": "inputPerM", "spendUsd": 0.00163, "billedUsd": 0.0014124, "estimatedUsd": 0.00163, "usage": { "prompt_tokens": 269, "completion_tokens": 182, "total_tokens": 451 }, "latencyMs": 3381 }, { "taskId": "code-004", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "correct": 2, "totalCost": 0.06, "costPerCorrect": 0.03 }, "missing": [] }, "output": "{\"correct\":2,\"totalCost\":0.06,\"costPerCorrect\":0.03}", "spendUsd": 0.001794, "billedUsd": 0.0015764, "estimatedUsd": 0.001794, "usage": { "prompt_tokens": 282, "completion_tokens": 205, "total_tokens": 487 }, "latencyMs": 2919 }, { "taskId": "code-005", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 3-6-3" }, "output": "3-6-3", "spendUsd": 0.001274, "billedUsd": 0.0010564, "estimatedUsd": 0.001274, "usage": { "prompt_tokens": 262, "completion_tokens": 125, "total_tokens": 387 }, "latencyMs": 2742 }, { "taskId": "code-006", "category": "coding", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected fetch" }, "output": "fetch", "spendUsd": 0.0008719999999999999, "billedUsd": 0.0006544, "estimatedUsd": 0.0008719999999999999, "usage": { "prompt_tokens": 268, "completion_tokens": 56, "total_tokens": 324 }, "latencyMs": 1173 }, { "taskId": "logic-001", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected Luna" }, "output": "Luna", "spendUsd": 0.0015079999999999998, "billedUsd": 0.0012904, "estimatedUsd": 0.0015079999999999998, "usage": { "prompt_tokens": 253, "completion_tokens": 167, "total_tokens": 420 }, "latencyMs": 2656 }, { "taskId": "logic-002", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected token" }, "output": "token", "spendUsd": 0.000926, "billedUsd": 0.0007084, "estimatedUsd": 0.000926, "usage": { "prompt_tokens": 253, "completion_tokens": 70, "total_tokens": 323 }, "latencyMs": 2166 }, { "taskId": "logic-003", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 2.5-mlg" }, "output": "2.5-mlg", "spendUsd": 0.00126, "billedUsd": 0.0010424, "estimatedUsd": 0.00126, "usage": { "prompt_tokens": 246, "completion_tokens": 128, "total_tokens": 374 }, "latencyMs": 2351 }, { "taskId": "logic-004", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected CPCA" }, "output": "CPCA", "spendUsd": 0.00089, "billedUsd": 0.0006724, "estimatedUsd": 0.00089, "usage": { "prompt_tokens": 247, "completion_tokens": 66, "total_tokens": 313 }, "latencyMs": 1526 }, { "taskId": "logic-005", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "pattern ^Google\\s*,\\s*Groq$" }, "output": "Google, Groq", "spendUsd": 0.00127, "billedUsd": 0.0010524, "estimatedUsd": 0.00127, "usage": { "prompt_tokens": 266, "completion_tokens": 123, "total_tokens": 389 }, "latencyMs": 2127 }, { "taskId": "logic-006", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected valid" }, "output": "valid", "spendUsd": 0.00131, "billedUsd": 0.0010924, "estimatedUsd": 0.00131, "usage": { "prompt_tokens": 256, "completion_tokens": 133, "total_tokens": 389 }, "latencyMs": 2334 }, { "taskId": "logic-007", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected invalid" }, "output": "invalid", "spendUsd": 0.001668, "billedUsd": 0.0014504, "estimatedUsd": 0.001668, "usage": { "prompt_tokens": 255, "completion_tokens": 193, "total_tokens": 448 }, "latencyMs": 3164 }, { "taskId": "logic-008", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected $0.14/M" }, "output": "$0.14/M", "spendUsd": 0.001204, "billedUsd": 0.0009864, "estimatedUsd": 0.001204, "usage": { "prompt_tokens": 269, "completion_tokens": 111, "total_tokens": 380 }, "latencyMs": 2053 }, { "taskId": "logic-009", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected z-ai-glm-5.2" }, "output": "z-ai-glm-5.2", "spendUsd": 0.001128, "billedUsd": 0.0009104, "estimatedUsd": 0.001128, "usage": { "prompt_tokens": 264, "completion_tokens": 100, "total_tokens": 364 }, "latencyMs": 1869 }, { "taskId": "logic-010", "category": "instruction", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 4, "detail": "expected 4" }, "output": "4", "spendUsd": 0.0011, "billedUsd": 0.0008824, "estimatedUsd": 0.0011, "usage": { "prompt_tokens": 259, "completion_tokens": 97, "total_tokens": 356 }, "latencyMs": 1750 }, { "taskId": "pricing-001", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cheaper" }, "output": "cheaper", "spendUsd": 0.00262, "billedUsd": 0.0024024, "estimatedUsd": 0.00262, "usage": { "prompt_tokens": 296, "completion_tokens": 338, "total_tokens": 634 }, "latencyMs": 4893 }, { "taskId": "pricing-002", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected B" }, "output": "B", "spendUsd": 0.002606, "billedUsd": 0.0023884, "estimatedUsd": 0.002606, "usage": { "prompt_tokens": 271, "completion_tokens": 344, "total_tokens": 615 }, "latencyMs": 5682 }, { "taskId": "pricing-003", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 1000, "detail": "expected 1000" }, "output": "1000", "spendUsd": 0.00109, "billedUsd": 0.0008724, "estimatedUsd": 0.00109, "usage": { "prompt_tokens": 266, "completion_tokens": 93, "total_tokens": 359 }, "latencyMs": 1885 }, { "taskId": "pricing-004", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": { "inputCost": 0.001, "outputCost": 0.003, "totalCost": 0.004 }, "missing": [] }, "output": "{\"inputCost\": 0.001, \"outputCost\": 0.003, \"totalCost\": 0.004}", "spendUsd": 0.001422, "billedUsd": 0.0012044, "estimatedUsd": 0.001422, "usage": { "prompt_tokens": 279, "completion_tokens": 144, "total_tokens": 423 }, "latencyMs": 2639 }, { "taskId": "pricing-005", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected yes" }, "output": "yes", "spendUsd": 0.00098, "billedUsd": 0.0007624, "estimatedUsd": 0.00098, "usage": { "prompt_tokens": 262, "completion_tokens": 76, "total_tokens": 338 }, "latencyMs": 1874 }, { "taskId": "pricing-006", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected 5:1" }, "output": "5:1", "spendUsd": 0.001176, "billedUsd": 0.0009584, "estimatedUsd": 0.001176, "usage": { "prompt_tokens": 258, "completion_tokens": 110, "total_tokens": 368 }, "latencyMs": 2363 }, { "taskId": "pricing-007", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "actual": 325, "detail": "expected 325" }, "output": "325", "spendUsd": 0.001124, "billedUsd": 0.0009064, "estimatedUsd": 0.001124, "usage": { "prompt_tokens": 259, "completion_tokens": 101, "total_tokens": 360 }, "latencyMs": 1823 }, { "taskId": "pricing-008", "category": "pricing", "pricingId": "grok-4.5", "openrouterId": "x-ai/grok-4.5", "label": "Grok 4.5", "status": "ok", "correct": true, "grade": { "correct": true, "detail": "expected cost per correct answer" }, "output": "cost per correct answer", "spendUsd": 0.001026, "billedUsd": 0.0008084, "estimatedUsd": 0.001026, "usage": { "prompt_tokens": 264, "completion_tokens": 83, "total_tokens": 347 }, "latencyMs": 2177 } ] }