{"dataset":{"name":"OpenUI language and model benchmark","canonical_url":"https://www.openui.com/benchmarks/language","benchmark_type":"OpenUI language output across models","version":"v1","updated":"2026-08-25"},"question":"How reliably and economically do different models generate structurally valid OpenUI?","metric":{"y":"Structural validity percentage","x":"Measured cost per task in USD at provider list prices where comparable"},"scope":{"models":30,"briefs":46,"generations_per_brief":4},"caveats":["Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16, whose parser validates enum and scalar prop values. The website and the benchmark repository are on the same regime; raw outputs and per-run verdicts are committed for independent rescoring.","Self-hosted cost is null because no comparable API list price is available; null does not mean free.","Five low-validity compact or local models are hidden in the default visual view but remain present in this dataset.","The website publishes summary rows for this board; row-level evidence links are not yet attached to every displayed value."],"methodology":"https://www.openui.com/benchmarks/methodology","data":[{"model_id":"grok-4-6","model_name":"Grok 4.6","provider":"xAI","family_id":"grok-4-6","family_name":"Grok 4.6","family_order":null,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":99.5,"cost_per_task_usd":0.01847826086956522,"cost_per_46_screen_pass_usd":0.85,"cost_type":"provider-list-price","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gpt-5-6-sol","model_name":"GPT-5.6 Sol","provider":"OpenAI","family_id":"gpt-5-6","family_name":"GPT-5.6","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":99.5,"cost_per_task_usd":0.04760869565217391,"cost_per_46_screen_pass_usd":2.19,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"claude-opus-4-8","model_name":"Claude Opus 4.8","provider":"Anthropic","family_id":"claude","family_name":"Claude","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":98.9,"cost_per_task_usd":0.04934782608695652,"cost_per_46_screen_pass_usd":2.27,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gemini-3-7-flash","model_name":"Gemini 3.7 Flash","provider":"Google","family_id":"gemini-flash","family_name":"Gemini Flash","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":98.9,"cost_per_task_usd":0.01,"cost_per_46_screen_pass_usd":0.46,"cost_type":"provider-list-price","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"claude-sonnet-5","model_name":"Claude Sonnet 5","provider":"Anthropic","family_id":"claude","family_name":"Claude","family_order":3,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":98.4,"cost_per_task_usd":0.020217391304347826,"cost_per_46_screen_pass_usd":0.93,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gpt-5-6-terra","model_name":"GPT-5.6 Terra","provider":"OpenAI","family_id":"gpt-5-6","family_name":"GPT-5.6","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":98.4,"cost_per_task_usd":0.0241304347826087,"cost_per_46_screen_pass_usd":1.11,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"kimi-k3","model_name":"Kimi K3","provider":"Moonshot","family_id":"kimi-k3","family_name":"Kimi K3","family_order":null,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":96.2,"cost_per_task_usd":0.03326086956521739,"cost_per_46_screen_pass_usd":1.53,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"claude-opus-5","model_name":"Claude Opus 5","provider":"Anthropic","family_id":"claude","family_name":"Claude","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":96.2,"cost_per_task_usd":0.07739130434782608,"cost_per_46_screen_pass_usd":3.56,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"muse-spark-1-2","model_name":"Muse Spark 1.2","provider":"Meta","family_id":"muse-spark-1-2","family_name":"Muse Spark 1.2","family_order":null,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":96.2,"cost_per_task_usd":0.015652173913043476,"cost_per_46_screen_pass_usd":0.72,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"claude-sonnet-4-6","model_name":"Claude Sonnet 4.6","provider":"Anthropic","family_id":"claude","family_name":"Claude","family_order":4,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":92.9,"cost_per_task_usd":0.045,"cost_per_46_screen_pass_usd":2.07,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"qwen-3-8-2-4t","model_name":"Qwen3.8 2.4T","provider":"Alibaba","family_id":"qwen-3-8","family_name":"Qwen3.8","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":91.8,"cost_per_task_usd":0.016956521739130436,"cost_per_46_screen_pass_usd":0.78,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"glm-5-3","model_name":"GLM-5.3","provider":"Zhipu","family_id":"glm-5-3","family_name":"GLM-5.3","family_order":null,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":90.8,"cost_per_task_usd":0.011956521739130435,"cost_per_46_screen_pass_usd":0.55,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"inkling-small","model_name":"Inkling Small","provider":"Thinking Machines","family_id":"inkling","family_name":"Inkling","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":88.6,"cost_per_task_usd":0.003260869565217391,"cost_per_46_screen_pass_usd":0.15,"cost_type":"provider-list-price","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"deepseek-v4-flash","model_name":"DeepSeek V4 Flash","provider":"DeepSeek","family_id":"deepseek-v4","family_name":"DeepSeek V4","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":85.8,"cost_per_task_usd":0.0004347826086956522,"cost_per_46_screen_pass_usd":0.02,"cost_type":"provider-list-price","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"deepseek-v4-pro","model_name":"DeepSeek V4 Pro","provider":"DeepSeek","family_id":"deepseek-v4","family_name":"DeepSeek V4","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":84.2,"cost_per_task_usd":0.008913043478260869,"cost_per_46_screen_pass_usd":0.41,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gpt-5-6-luna","model_name":"GPT-5.6 Luna","provider":"OpenAI","family_id":"gpt-5-6","family_name":"GPT-5.6","family_order":3,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":83.7,"cost_per_task_usd":0.002173913043478261,"cost_per_46_screen_pass_usd":0.1,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"qwen-3-8-27b","model_name":"Qwen3.8 27B","provider":"Alibaba","family_id":"qwen-3-8","family_name":"Qwen3.8","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":78.8,"cost_per_task_usd":0.005434782608695652,"cost_per_46_screen_pass_usd":0.25,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gemini-3-6-flash","model_name":"Gemini 3.6 Flash","provider":"Google","family_id":"gemini-flash","family_name":"Gemini Flash","family_order":2,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":78.8,"cost_per_task_usd":0.009130434782608695,"cost_per_46_screen_pass_usd":0.42,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gemini-3-5-flash-lite","model_name":"Gemini 3.5 Flash-Lite","provider":"Google","family_id":"gemini-flash","family_name":"Gemini Flash","family_order":3,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":78.3,"cost_per_task_usd":0.004130434782608695,"cost_per_46_screen_pass_usd":0.19,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"inkling","model_name":"Inkling","provider":"Thinking Machines","family_id":"inkling","family_name":"Inkling","family_order":1,"connected_family":false,"default_selected":true,"format":"openui","validity_score_percent":73.4,"cost_per_task_usd":0.008043478260869565,"cost_per_46_screen_pass_usd":0.37,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"qwen-3-6-27b","model_name":"Qwen3.6 27B","provider":"Alibaba","family_id":"qwen-3-6","family_name":"Qwen3.6","family_order":1,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":68.5,"cost_per_task_usd":0.006304347826086956,"cost_per_46_screen_pass_usd":0.29,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"qwen-3-6-35b-a3b","model_name":"Qwen3.6 35B-A3B","provider":"Alibaba","family_id":"qwen-3-6","family_name":"Qwen3.6","family_order":2,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":61.4,"cost_per_task_usd":0.0017391304347826088,"cost_per_46_screen_pass_usd":0.08,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gemma-4-31b","model_name":"Gemma 4 31B","provider":"Google","family_id":"gemma-4","family_name":"Gemma 4","family_order":1,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":46.7,"cost_per_task_usd":0.0008695652173913044,"cost_per_46_screen_pass_usd":0.04,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"phi-4","model_name":"Phi-4","provider":"Microsoft","family_id":"phi-4","family_name":"Phi-4","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":44,"cost_per_task_usd":0.0004347826086956522,"cost_per_46_screen_pass_usd":0.02,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"gemma-4-26b-a4b","model_name":"Gemma 4 26B-A4B","provider":"Google","family_id":"gemma-4","family_name":"Gemma 4","family_order":2,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":29.9,"cost_per_task_usd":0.0006521739130434782,"cost_per_46_screen_pass_usd":0.03,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"ministral-8b","model_name":"Ministral 8B","provider":"Mistral","family_id":"ministral-8b","family_name":"Ministral 8B","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":27.2,"cost_per_task_usd":0.0008695652173913044,"cost_per_46_screen_pass_usd":0.04,"cost_type":"provider-list-price","serving":null,"pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"granite-4-1-8b","model_name":"Granite 4.1 8B","provider":"IBM","family_id":"granite-4-1-8b","family_name":"Granite 4.1 8B","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":14.7,"cost_per_task_usd":0.0002173913043478261,"cost_per_46_screen_pass_usd":0.01,"cost_type":"provider-list-price","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"lfm-2-5-2-6b","model_name":"LFM 2.5 2.6B","provider":"Liquid","family_id":"lfm-2-5-2-6b","family_name":"LFM 2.5 2.6B","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":3.3,"cost_per_task_usd":0,"cost_per_46_screen_pass_usd":0,"cost_type":"free","serving":null,"pareto_frontier":true,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"diffusion-gemma-26b-a4b","model_name":"DiffusionGemma 26B-A4B","provider":"Google","family_id":"diffusion-gemma-26b-a4b","family_name":"DiffusionGemma 26B-A4B","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":13,"cost_per_task_usd":null,"cost_per_46_screen_pass_usd":null,"cost_type":"self-hosted","serving":"Self-hosted · A100 FP8","pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"},{"model_id":"ling-3-tiny","model_name":"Ling 3.0 Tiny","provider":"InclusionAI","family_id":"ling-3-tiny","family_name":"Ling 3.0 Tiny","family_order":null,"connected_family":false,"default_selected":false,"format":"openui","validity_score_percent":9.8,"cost_per_task_usd":null,"cost_per_46_screen_pass_usd":null,"cost_type":"self-hosted","serving":"Self-hosted · llama.cpp Q8","pareto_frontier":false,"briefs":46,"generations_per_brief":4,"dataset_updated":"2026-08-25"}],"distributions":{"json":"https://www.openui.com/benchmarks/language/data.json","csv":"https://www.openui.com/benchmarks/language/data.csv","markdown":"https://www.openui.com/benchmarks/language/agent.md","combined_dataset":"https://www.openui.com/benchmarks/data.json"}}