{"dataset":{"name":"Generative UI framework benchmark","canonical_url":"https://www.openui.com/benchmarks/framework","benchmark_type":"OpenUI versus A2UI versus json-render","version":"v1","updated":"2026-08-18"},"question":"How do OpenUI, Google A2UI, and Vercel json-render compare under the same briefs and generation condition?","scope":{"models":6,"formats":3,"briefs":46,"generations_per_brief":4,"runs_per_format":1104,"total_scored_runs":3312},"definitions":{"structural_validity":"Parses, has a root, resolves every reference, has no orphaned or invented components, uses valid props, is not truncated, and meets the component-count floor.","render_success":"The format's own shipped renderer produced a non-blank screen.","generation_condition":"Four generations per brief, a 16,384-token output ceiling, temperature 0.7 where the provider accepts it, and minimal or no reasoning. Anthropic runs use the model default temperature because that API rejects the benchmark temperature setting for the tested model.","scorer_regime":"Every row is scored by one build of the shipped OpenUI parser, lang-core 0.2.16, whose parser validates enum and scalar prop values. The website and the benchmark repository are on the same regime; raw outputs and per-run verdicts are committed for independent rescoring."},"methodology":"https://www.openui.com/benchmarks/methodology","evidence":{"evidence_status":"published","scorer_regime":"lang-core 0.2.16","raw_results":"https://github.com/thesysdev/generative-ui-bench/tree/main/results","raw_outputs":"https://github.com/thesysdev/generative-ui-bench/tree/main/raw","harness":"https://github.com/thesysdev/generative-ui-bench","briefs":"https://github.com/thesysdev/generative-ui-bench/tree/main/briefs/briefs.ts"},"format_summary":[{"format_id":"openui","format_name":"OpenUI","validity_score_percent":96.92028985507247,"render_rate_percent":99.90942028985508,"blank_screens":1,"runs":1104,"system_prompt_tokens":5031,"mean_output_tokens_per_screen":1362,"estimated_stream_seconds_at_50_tps":27.24},{"format_id":"a2ui","format_name":"A2UI","validity_score_percent":95.56159420289855,"render_rate_percent":96.64855072463769,"blank_screens":37,"runs":1104,"system_prompt_tokens":12610,"mean_output_tokens_per_screen":2823,"estimated_stream_seconds_at_50_tps":56.46},{"format_id":"jsonRender","format_name":"json-render","validity_score_percent":82.78985507246377,"render_rate_percent":99.63768115942028,"blank_screens":4,"runs":1104,"system_prompt_tokens":7651,"mean_output_tokens_per_screen":3258,"estimated_stream_seconds_at_50_tps":65.16}],"data":[{"model_id":"sol","model_name":"GPT-5.6 Sol","provider":"OpenAI","format_id":"openui","format_name":"OpenUI","valid_runs":183,"rendered_runs":184,"total_runs":184,"validity_score_percent":99.45652173913044,"render_rate_percent":100,"cost_per_46_screen_pass_usd":3.08},{"model_id":"sol","model_name":"GPT-5.6 Sol","provider":"OpenAI","format_id":"a2ui","format_name":"A2UI","valid_runs":177,"rendered_runs":179,"total_runs":184,"validity_score_percent":96.19565217391305,"render_rate_percent":97.28260869565217,"cost_per_46_screen_pass_usd":6.84},{"model_id":"sol","model_name":"GPT-5.6 Sol","provider":"OpenAI","format_id":"jsonRender","format_name":"json-render","valid_runs":152,"rendered_runs":184,"total_runs":184,"validity_score_percent":82.6086956521739,"render_rate_percent":100,"cost_per_46_screen_pass_usd":6.3},{"model_id":"opus","model_name":"Claude Opus 4.8","provider":"Anthropic","format_id":"openui","format_name":"OpenUI","valid_runs":182,"rendered_runs":184,"total_runs":184,"validity_score_percent":98.91304347826086,"render_rate_percent":100,"cost_per_46_screen_pass_usd":2.27},{"model_id":"opus","model_name":"Claude Opus 4.8","provider":"Anthropic","format_id":"a2ui","format_name":"A2UI","valid_runs":183,"rendered_runs":184,"total_runs":184,"validity_score_percent":99.45652173913044,"render_rate_percent":100,"cost_per_46_screen_pass_usd":5.85},{"model_id":"opus","model_name":"Claude Opus 4.8","provider":"Anthropic","format_id":"jsonRender","format_name":"json-render","valid_runs":161,"rendered_runs":184,"total_runs":184,"validity_score_percent":87.5,"render_rate_percent":100,"cost_per_46_screen_pass_usd":4.88},{"model_id":"kimi","model_name":"Kimi K3","provider":"Moonshot","format_id":"openui","format_name":"OpenUI","valid_runs":177,"rendered_runs":184,"total_runs":184,"validity_score_percent":96.19565217391305,"render_rate_percent":100,"cost_per_46_screen_pass_usd":1.53},{"model_id":"kimi","model_name":"Kimi K3","provider":"Moonshot","format_id":"a2ui","format_name":"A2UI","valid_runs":176,"rendered_runs":179,"total_runs":184,"validity_score_percent":95.65217391304348,"render_rate_percent":97.28260869565217,"cost_per_46_screen_pass_usd":3.16},{"model_id":"kimi","model_name":"Kimi K3","provider":"Moonshot","format_id":"jsonRender","format_name":"json-render","valid_runs":131,"rendered_runs":184,"total_runs":184,"validity_score_percent":71.19565217391305,"render_rate_percent":100,"cost_per_46_screen_pass_usd":2.91},{"model_id":"gemini","model_name":"Gemini 3.7 Flash","provider":"Google","format_id":"openui","format_name":"OpenUI","valid_runs":182,"rendered_runs":184,"total_runs":184,"validity_score_percent":98.91304347826086,"render_rate_percent":100,"cost_per_46_screen_pass_usd":0.42},{"model_id":"gemini","model_name":"Gemini 3.7 Flash","provider":"Google","format_id":"a2ui","format_name":"A2UI","valid_runs":173,"rendered_runs":178,"total_runs":184,"validity_score_percent":94.02173913043478,"render_rate_percent":96.73913043478261,"cost_per_46_screen_pass_usd":0.95},{"model_id":"gemini","model_name":"Gemini 3.7 Flash","provider":"Google","format_id":"jsonRender","format_name":"json-render","valid_runs":171,"rendered_runs":184,"total_runs":184,"validity_score_percent":92.93478260869566,"render_rate_percent":100,"cost_per_46_screen_pass_usd":0.85},{"model_id":"qwen","model_name":"Qwen3.8 2.4T","provider":"Alibaba","format_id":"openui","format_name":"OpenUI","valid_runs":169,"rendered_runs":183,"total_runs":184,"validity_score_percent":91.84782608695652,"render_rate_percent":99.45652173913044,"cost_per_46_screen_pass_usd":0.81},{"model_id":"qwen","model_name":"Qwen3.8 2.4T","provider":"Alibaba","format_id":"a2ui","format_name":"A2UI","valid_runs":168,"rendered_runs":168,"total_runs":184,"validity_score_percent":91.30434782608695,"render_rate_percent":91.30434782608695,"cost_per_46_screen_pass_usd":1.9},{"model_id":"qwen","model_name":"Qwen3.8 2.4T","provider":"Alibaba","format_id":"jsonRender","format_name":"json-render","valid_runs":149,"rendered_runs":180,"total_runs":184,"validity_score_percent":80.97826086956522,"render_rate_percent":97.82608695652173,"cost_per_46_screen_pass_usd":1.46},{"model_id":"muse","model_name":"Muse Spark 1.2","provider":"Meta","format_id":"openui","format_name":"OpenUI","valid_runs":177,"rendered_runs":184,"total_runs":184,"validity_score_percent":96.19565217391305,"render_rate_percent":100,"cost_per_46_screen_pass_usd":0.71},{"model_id":"muse","model_name":"Muse Spark 1.2","provider":"Meta","format_id":"a2ui","format_name":"A2UI","valid_runs":178,"rendered_runs":179,"total_runs":184,"validity_score_percent":96.73913043478261,"render_rate_percent":97.28260869565217,"cost_per_46_screen_pass_usd":1.38},{"model_id":"muse","model_name":"Muse Spark 1.2","provider":"Meta","format_id":"jsonRender","format_name":"json-render","valid_runs":150,"rendered_runs":184,"total_runs":184,"validity_score_percent":81.52173913043478,"render_rate_percent":100,"cost_per_46_screen_pass_usd":1.34}],"distributions":{"json":"https://www.openui.com/benchmarks/framework/data.json","csv":"https://www.openui.com/benchmarks/framework/data.csv","markdown":"https://www.openui.com/benchmarks/framework/agent.md","combined_dataset":"https://www.openui.com/benchmarks/data.json"}}