# Generated by scripts/publish_leaderboard.py; do not edit by hand.
benchmark: CADClamp
generated: '2026-09-27'
commit: e28dd5c
versions:
  prompt_sets:
    v0.2: 0.2.3
    trackc: trackc-0.1.3
  task: 3
  harness: '0.3'
  engine: 0.2.2
headline: On each prompt, a part that meets every requirement scores its printability divided by the reference solution's,
  capped at 1. A part that misses any requirement scores 0. Each language's score is the average over its 47 prompts.
main:
- model: claude-opus-5-5
  baseline: false
  languages: 7
  average: 0.9086
  by_language:
    build123d: 0.9209
    openscad: 0.9685
    cadquery: 0.8814
    freecad: 0.9253
    rhino: 0.8464
    fusion: 0.9313
    blender: 0.8861
  public: 0.9186
  heldout: 0.8714
- model: gpt-6-astra
  baseline: false
  languages: 7
  average: 0.9
  by_language:
    build123d: 0.9145
    openscad: 0.9251
    cadquery: 0.9252
    freecad: 0.8986
    rhino: 0.8929
    fusion: 0.8397
    blender: 0.9037
  public: 0.9038
  heldout: 0.8856
- model: claude-fable-5-1
  baseline: false
  languages: 7
  average: 0.8983
  by_language:
    build123d: 0.9062
    openscad: 0.9703
    cadquery: 0.9145
    freecad: 0.9074
    rhino: 0.7846
    fusion: 0.9478
    blender: 0.8575
  public: 0.9082
  heldout: 0.8618
- model: gpt-6-sol
  baseline: false
  languages: 7
  average: 0.8415
  by_language:
    build123d: 0.9463
    openscad: 0.8827
    cadquery: 0.9039
    freecad: 0.9038
    rhino: 0.5954
    fusion: 0.8184
    blender: 0.8397
  public: 0.8411
  heldout: 0.8426
- model: grok-4.6
  baseline: false
  languages: 7
  average: 0.8157
  by_language:
    build123d: 0.8068
    openscad: 0.9219
    cadquery: 0.7947
    freecad: 0.9011
    rhino: 0.7546
    fusion: 0.6695
    blender: 0.8611
  public: 0.816
  heldout: 0.8143
- model: grok-4.7
  baseline: false
  languages: 7
  average: 0.8137
  by_language:
    build123d: 0.7733
    openscad: 0.901
    cadquery: 0.901
    freecad: 0.8736
    rhino: 0.7048
    fusion: 0.7337
    blender: 0.8085
  public: 0.8213
  heldout: 0.7856
- model: gpt-6-luna-pro
  baseline: false
  languages: 7
  average: 0.756
  by_language:
    build123d: 0.901
    openscad: 0.8161
    cadquery: 0.901
    freecad: 0.8693
    rhino: 0.4465
    fusion: 0.5206
    blender: 0.8372
  public: 0.7595
  heldout: 0.7428
- model: claude-opus-5
  baseline: false
  languages: 7
  average: 0.7284
  by_language:
    build123d: 0.7835
    openscad: 0.8206
    cadquery: 0.6684
    freecad: 0.8783
    rhino: 0.6751
    fusion: 0.542
    blender: 0.7311
  public: 0.7514
  heldout: 0.7093
- model: kimi-k3
  baseline: false
  languages: 7
  average: 0.7259
  by_language:
    build123d: 0.7333
    openscad: 0.8189
    cadquery: 0.8401
    freecad: 0.9457
    rhino: 0.6167
    fusion: 0.3719
    blender: 0.7546
  public: 0.7291
  heldout: 0.7139
- model: gpt-6-luna
  baseline: false
  languages: 7
  average: 0.6865
  by_language:
    build123d: 0.8159
    openscad: 0.752
    cadquery: 0.8827
    freecad: 0.8951
    rhino: 0.3082
    fusion: 0.425
    blender: 0.7263
  public: 0.6848
  heldout: 0.6925
- model: gpt-5.1
  baseline: false
  languages: 7
  average: 0.1897
  by_language:
    build123d: 0.0426
    openscad: 0.3344
    cadquery: 0.2082
    freecad: 0.3384
    rhino: 0.2341
    fusion: 0.0426
    blender: 0.1277
  public: 0.1869
  heldout: 0.2
- model: qwen2.5-coder:7b
  baseline: false
  languages: 7
  average: 0.0091
  by_language:
    build123d: 0.0
    openscad: 0.0213
    cadquery: 0.0213
    freecad: 0.0213
    rhino: 0.0
    fusion: 0.0
    blender: 0.0
  public: 0.0039
  heldout: 0.0286
- model: baseline:reference
  baseline: true
  languages: 1
  average: 0.9972
  by_language:
    openscad: 0.9972
  public: 0.9965
  heldout: 1.0
- model: baseline:cube
  baseline: true
  languages: 1
  average: 0.0
  by_language:
    openscad: 0.0
  public: 0.0
  heldout: 0.0
rows:
- model: claude-opus-5
  harness: claude-cli
  language: build123d
  attempts: 1
  n: 60
  prompts: 20
  score: 0.802
  ci95:
  - 0.676
  - 0.9125
  spec_pass: 0.8667
  requirements: 0.9
  valid: 0.9167
  printability: 0.852
  epochs: 3
  pass_all: 0.75
  pass_any: 0.95
  score_public: 0.8462
  score_heldout: 0.625
  parametric: 1.0
- model: claude-opus-5
  harness: claude-cli
  language: openscad
  attempts: 1
  n: 60
  prompts: 20
  score: 0.7864
  ci95:
  - 0.6316
  - 0.9282
  spec_pass: 0.8333
  requirements: 0.9056
  valid: 0.95
  printability: 0.8781
  epochs: 3
  pass_all: 0.75
  pass_any: 0.9
  score_public: 0.8268
  score_heldout: 0.625
- model: gpt-6-astra
  harness: openrouter
  language: blender
  attempts: 1
  n: 94
  prompts: 47
  score: 0.9037
  ci95:
  - 0.8295
  - 0.9678
  spec_pass: 0.9149
  requirements: 0.9215
  valid: 0.9681
  printability: 0.892
  epochs: 2
  pass_all: 0.8723
  pass_any: 0.9574
  score_public: 0.9047
  score_heldout: 0.9
  parametric: 0.8877
- model: claude-opus-5-5
  harness: claude-cli
  language: blender
  attempts: 1
  n: 141
  prompts: 47
  score: 0.8861
  ci95:
  - 0.8188
  - 0.9455
  spec_pass: 0.8936
  requirements: 0.9137
  valid: 0.9645
  printability: 0.9053
  epochs: 3
  pass_all: 0.766
  pass_any: 0.9787
  score_public: 0.9273
  score_heldout: 0.7333
  parametric: 0.886
- model: grok-4.6
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8611
  ci95:
  - 0.7652
  - 0.9566
  spec_pass: 0.8723
  requirements: 0.9
  valid: 0.8936
  printability: 0.8268
  epochs: 1
  pass_all: 0.8723
  pass_any: 0.8723
  score_public: 0.8505
  score_heldout: 0.9
  parametric: 0.9683
- model: claude-fable-5-1
  harness: claude-cli
  language: blender
  attempts: 1
  n: 141
  prompts: 47
  score: 0.8575
  ci95:
  - 0.7939
  - 0.9215
  spec_pass: 0.8582
  requirements: 0.8975
  valid: 0.9645
  printability: 0.9105
  epochs: 3
  pass_all: 0.7021
  pass_any: 0.9787
  score_public: 0.891
  score_heldout: 0.7333
  parametric: 0.8799
- model: gpt-6-sol
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8397
  ci95:
  - 0.7435
  - 0.9253
  spec_pass: 0.8511
  requirements: 0.8862
  valid: 0.9574
  printability: 0.8656
  epochs: 1
  pass_all: 0.8511
  pass_any: 0.8511
  score_public: 0.9047
  score_heldout: 0.599
  parametric: 0.9185
- model: gpt-6-luna-pro
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8372
  ci95:
  - 0.7337
  - 0.9356
  spec_pass: 0.8511
  requirements: 0.8981
  valid: 0.9574
  printability: 0.8766
  epochs: 1
  pass_all: 0.8511
  pass_any: 0.8511
  score_public: 0.7933
  score_heldout: 1.0
  parametric: 0.8926
- model: grok-4.7
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8085
  ci95:
  - 0.701
  - 0.9149
  spec_pass: 0.8298
  requirements: 0.8727
  valid: 0.9149
  printability: 0.8254
  epochs: 1
  pass_all: 0.8298
  pass_any: 0.8298
  score_public: 0.8378
  score_heldout: 0.7
  parametric: 0.907
- model: kimi-k3
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7546
  ci95:
  - 0.6373
  - 0.8723
  spec_pass: 0.766
  requirements: 0.8088
  valid: 0.9362
  printability: 0.8406
  epochs: 1
  pass_all: 0.766
  pass_any: 0.766
  score_public: 0.7699
  score_heldout: 0.698
  parametric: 0.8182
- model: claude-opus-5
  harness: claude-cli
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7311
  ci95:
  - 0.6035
  - 0.8588
  spec_pass: 0.7447
  requirements: 0.7971
  valid: 0.8723
  printability: 0.7982
  epochs: 1
  pass_all: 0.7447
  pass_any: 0.7447
  score_public: 0.7567
  score_heldout: 0.6362
  parametric: 0.8902
- model: gpt-6-luna
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7263
  ci95:
  - 0.5957
  - 0.8498
  spec_pass: 0.766
  requirements: 0.8396
  valid: 0.9149
  printability: 0.8252
  epochs: 1
  pass_all: 0.766
  pass_any: 0.766
  score_public: 0.7068
  score_heldout: 0.7985
  parametric: 0.8798
- model: gpt-5.1
  harness: openrouter
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.1277
  ci95:
  - 0.0426
  - 0.234
  spec_pass: 0.1277
  requirements: 0.3034
  valid: 0.6809
  printability: 0.4908
  epochs: 1
  pass_all: 0.1277
  pass_any: 0.1277
  score_public: 0.1351
  score_heldout: 0.1
  parametric: 0.7708
- model: qwen2.5-coder:7b
  harness: ollama
  language: blender
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0
  ci95:
  - 0.0
  - 0.0
  spec_pass: 0.0
  requirements: 0.0241
  valid: 0.0851
  printability: 0.0638
  epochs: 1
  pass_all: 0.0
  pass_any: 0.0
  score_public: 0.0
  score_heldout: 0.0
  parametric: 0.625
- model: gpt-6-astra
  harness: openrouter
  language: blender@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.7259
  ci95:
  - 0.4433
  - 0.9267
  spec_pass: 0.8
  requirements: 0.8336
  valid: 0.8
  printability: 0.7252
  epochs: 1
  pass_all: 0.8
  pass_any: 0.8
- model: claude-fable-5-1
  harness: claude-cli
  language: blender@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.5953
  ci95:
  - 0.3
  - 0.8953
  spec_pass: 0.6
  requirements: 0.6904
  valid: 0.9
  printability: 0.7967
  epochs: 1
  pass_all: 0.6
  pass_any: 0.6
- model: claude-opus-5-5
  harness: claude-cli
  language: blender@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.4988
  ci95:
  - 0.1989
  - 0.7987
  spec_pass: 0.5
  requirements: 0.6804
  valid: 0.9
  printability: 0.7908
  epochs: 1
  pass_all: 0.5
  pass_any: 0.5
- model: kimi-k3
  harness: openrouter
  language: blender@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.3974
  ci95:
  - 0.1
  - 0.6965
  spec_pass: 0.4
  requirements: 0.5569
  valid: 0.6
  printability: 0.5485
  epochs: 1
  pass_all: 0.4
  pass_any: 0.4
- model: gpt-6-sol
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9463
  ci95:
  - 0.8825
  - 0.9998
  spec_pass: 0.9574
  requirements: 0.9811
  valid: 1.0
  printability: 0.9152
  epochs: 1
  pass_all: 0.9574
  pass_any: 0.9574
  score_public: 0.9319
  score_heldout: 0.9999
  parametric: 0.9149
- model: claude-opus-5-5
  harness: claude-cli
  language: build123d
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9209
  ci95:
  - 0.8429
  - 0.9837
  spec_pass: 0.922
  requirements: 0.9522
  valid: 0.9787
  printability: 0.926
  epochs: 3
  pass_all: 0.9149
  pass_any: 0.9362
  score_public: 0.9265
  score_heldout: 0.9
  parametric: 0.8841
- model: gpt-6-astra
  harness: openrouter
  language: build123d
  attempts: 1
  n: 94
  prompts: 47
  score: 0.9145
  ci95:
  - 0.8398
  - 0.9786
  spec_pass: 0.9255
  requirements: 0.9564
  valid: 0.9787
  printability: 0.9095
  epochs: 2
  pass_all: 0.9149
  pass_any: 0.9362
  score_public: 0.9184
  score_heldout: 0.9
  parametric: 0.8967
- model: claude-fable-5-1
  harness: claude-cli
  language: build123d
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9062
  ci95:
  - 0.8479
  - 0.9567
  spec_pass: 0.9149
  requirements: 0.9335
  valid: 0.9645
  printability: 0.8804
  epochs: 3
  pass_all: 0.8085
  pass_any: 0.9787
  score_public: 0.9079
  score_heldout: 0.9
  parametric: 0.8983
- model: gpt-6-luna-pro
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.901
  ci95:
  - 0.8159
  - 0.9757
  spec_pass: 0.9149
  requirements: 0.9645
  valid: 1.0
  printability: 0.9172
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9013
  score_heldout: 0.9
  parametric: 0.8865
- model: gpt-6-luna
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8159
  ci95:
  - 0.7089
  - 0.9193
  spec_pass: 0.8511
  requirements: 0.964
  valid: 1.0
  printability: 0.9058
  epochs: 1
  pass_all: 0.8511
  pass_any: 0.8511
  score_public: 0.8472
  score_heldout: 0.7
  parametric: 0.8901
- model: grok-4.6
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8068
  ci95:
  - 0.6924
  - 0.912
  spec_pass: 0.8298
  requirements: 0.9013
  valid: 0.9362
  printability: 0.7916
  epochs: 1
  pass_all: 0.8298
  pass_any: 0.8298
  score_public: 0.8086
  score_heldout: 0.8
  parametric: 0.9242
- model: claude-opus-5
  harness: claude-cli
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7835
  ci95:
  - 0.6748
  - 0.8901
  spec_pass: 0.8085
  requirements: 0.8553
  valid: 0.8936
  printability: 0.8113
  epochs: 1
  pass_all: 0.8085
  pass_any: 0.8085
  score_public: 0.7793
  score_heldout: 0.7993
  parametric: 0.873
- model: grok-4.7
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7733
  ci95:
  - 0.656
  - 0.8827
  spec_pass: 0.7872
  requirements: 0.8209
  valid: 0.8511
  printability: 0.7679
  epochs: 1
  pass_all: 0.7872
  pass_any: 0.7872
  score_public: 0.7661
  score_heldout: 0.8
  parametric: 0.9208
- model: kimi-k3
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7333
  ci95:
  - 0.6061
  - 0.8401
  spec_pass: 0.7447
  requirements: 0.8013
  valid: 0.8298
  printability: 0.7622
  epochs: 1
  pass_all: 0.7447
  pass_any: 0.7447
  score_public: 0.7422
  score_heldout: 0.7
  parametric: 0.7778
- model: gpt-5.1
  harness: openrouter
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0426
  ci95:
  - 0.0
  - 0.1064
  spec_pass: 0.0426
  requirements: 0.0776
  valid: 0.1064
  printability: 0.1007
  epochs: 1
  pass_all: 0.0426
  pass_any: 0.0426
  score_public: 0.027
  score_heldout: 0.1
  parametric: 0.8667
- model: qwen2.5-coder:7b
  harness: ollama
  language: build123d
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0
  ci95:
  - 0.0
  - 0.0
  spec_pass: 0.0
  requirements: 0.0
  valid: 0.0
  printability: 0.0
  epochs: 1
  pass_all: 0.0
  pass_any: 0.0
  score_public: 0.0
  score_heldout: 0.0
- model: claude-fable-5-1
  harness: claude-cli
  language: build123d
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9645
  ci95:
  - 0.9153
  - 0.9992
  spec_pass: 0.9787
  requirements: 0.984
  valid: 1.0
  printability: 0.914
  epochs: 1
  pass_all: 0.9787
  pass_any: 0.9787
  score_public: 0.955
  score_heldout: 1.0
  parametric: 0.9184
- model: claude-opus-5-5
  harness: claude-cli
  language: build123d
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9332
  ci95:
  - 0.8635
  - 0.9969
  spec_pass: 0.9362
  requirements: 0.9722
  valid: 1.0
  printability: 0.9376
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.9422
  score_heldout: 0.9
  parametric: 0.8723
- model: gpt-6-astra
  harness: openrouter
  language: build123d
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9251
  ci95:
  - 0.8504
  - 0.9889
  spec_pass: 0.9362
  requirements: 0.9586
  valid: 0.9787
  printability: 0.9072
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.9319
  score_heldout: 0.9
  parametric: 0.8986
- model: claude-fable-5-1
  harness: claude-cli
  language: build123d@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.9034
  ci95:
  - 0.7907
  - 0.9899
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.8772
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
- model: claude-opus-5-5
  harness: claude-cli
  language: build123d@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8885
  ci95:
  - 0.7615
  - 0.9856
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.8713
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
- model: gpt-6-astra
  harness: openrouter
  language: build123d@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8425
  ci95:
  - 0.6417
  - 0.9852
  spec_pass: 0.9
  requirements: 0.9875
  valid: 1.0
  printability: 0.8306
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: kimi-k3
  harness: openrouter
  language: build123d@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.3922
  ci95:
  - 0.1557
  - 0.6563
  spec_pass: 0.5
  requirements: 0.6767
  valid: 0.8
  printability: 0.6402
  epochs: 1
  pass_all: 0.5
  pass_any: 0.5
- model: gpt-6-astra
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 94
  prompts: 47
  score: 0.9252
  ci95:
  - 0.8504
  - 0.989
  spec_pass: 0.9362
  requirements: 0.9577
  valid: 0.9787
  printability: 0.9078
  epochs: 2
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.932
  score_heldout: 0.8999
  parametric: 0.9293
- model: claude-fable-5-1
  harness: claude-cli
  language: cadquery
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9145
  ci95:
  - 0.8368
  - 0.9781
  spec_pass: 0.9149
  requirements: 0.9486
  valid: 0.9787
  printability: 0.9172
  epochs: 3
  pass_all: 0.8723
  pass_any: 0.9362
  score_public: 0.9185
  score_heldout: 0.9
  parametric: 0.8913
- model: gpt-6-sol
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9039
  ci95:
  - 0.8188
  - 0.9786
  spec_pass: 0.9149
  requirements: 0.9396
  valid: 0.9574
  printability: 0.8798
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9049
  score_heldout: 0.8999
  parametric: 0.9222
- model: grok-4.7
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.901
  ci95:
  - 0.8154
  - 0.9786
  spec_pass: 0.9149
  requirements: 0.956
  valid: 0.9787
  printability: 0.8893
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.8743
  score_heldout: 1.0
  parametric: 0.942
- model: gpt-6-luna-pro
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.901
  ci95:
  - 0.8158
  - 0.9757
  spec_pass: 0.9149
  requirements: 0.9839
  valid: 1.0
  printability: 0.8936
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9013
  score_heldout: 0.8999
  parametric: 0.9326
- model: gpt-6-luna
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8827
  ci95:
  - 0.7872
  - 0.9574
  spec_pass: 0.8936
  requirements: 0.9527
  valid: 0.9787
  printability: 0.8829
  epochs: 1
  pass_all: 0.8936
  pass_any: 0.8936
  score_public: 0.878
  score_heldout: 0.9
  parametric: 0.9022
- model: claude-opus-5-5
  harness: claude-cli
  language: cadquery
  attempts: 1
  n: 141
  prompts: 47
  score: 0.8814
  ci95:
  - 0.8175
  - 0.9422
  spec_pass: 0.8865
  requirements: 0.918
  valid: 0.9362
  printability: 0.8787
  epochs: 3
  pass_all: 0.766
  pass_any: 0.9787
  score_public: 0.8764
  score_heldout: 0.9
  parametric: 0.8939
- model: kimi-k3
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8401
  ci95:
  - 0.7337
  - 0.9361
  spec_pass: 0.8511
  requirements: 0.8771
  valid: 0.8936
  printability: 0.8268
  epochs: 1
  pass_all: 0.8511
  pass_any: 0.8511
  score_public: 0.8509
  score_heldout: 0.8
  parametric: 0.9286
- model: grok-4.6
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7947
  ci95:
  - 0.6803
  - 0.904
  spec_pass: 0.8085
  requirements: 0.8672
  valid: 0.8936
  printability: 0.8345
  epochs: 1
  pass_all: 0.8085
  pass_any: 0.8085
  score_public: 0.7932
  score_heldout: 0.8
  parametric: 0.9286
- model: claude-opus-5
  harness: claude-cli
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.6684
  ci95:
  - 0.5422
  - 0.7961
  spec_pass: 0.6809
  requirements: 0.7539
  valid: 0.7447
  printability: 0.6821
  epochs: 1
  pass_all: 0.6809
  pass_any: 0.6809
  score_public: 0.6599
  score_heldout: 0.7
  parametric: 0.9
- model: gpt-5.1
  harness: openrouter
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.2082
  ci95:
  - 0.1084
  - 0.3285
  spec_pass: 0.234
  requirements: 0.3133
  valid: 0.3617
  printability: 0.3128
  epochs: 1
  pass_all: 0.234
  pass_any: 0.234
  score_public: 0.1834
  score_heldout: 0.2999
  parametric: 0.9314
- model: qwen2.5-coder:7b
  harness: ollama
  language: cadquery
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0213
  ci95:
  - 0.0
  - 0.0638
  spec_pass: 0.0213
  requirements: 0.0384
  valid: 0.0851
  printability: 0.0847
  epochs: 1
  pass_all: 0.0213
  pass_any: 0.0213
  score_public: 0.0
  score_heldout: 0.1
  parametric: 0.9167
- model: gpt-6-astra
  harness: openrouter
  language: cadquery
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9252
  ci95:
  - 0.8504
  - 0.989
  spec_pass: 0.9362
  requirements: 0.9751
  valid: 1.0
  printability: 0.929
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.932
  score_heldout: 0.8999
  parametric: 0.9457
- model: claude-fable-5-1
  harness: claude-cli
  language: cadquery
  attempts: 2
  n: 47
  prompts: 47
  score: 0.914
  ci95:
  - 0.8289
  - 0.9786
  spec_pass: 0.9149
  requirements: 0.958
  valid: 1.0
  printability: 0.9123
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9178
  score_heldout: 0.9
  parametric: 0.8986
- model: claude-opus-5-5
  harness: claude-cli
  language: cadquery
  attempts: 2
  n: 47
  prompts: 47
  score: 0.8584
  ci95:
  - 0.7625
  - 0.9481
  spec_pass: 0.8723
  requirements: 0.9005
  valid: 0.9149
  printability: 0.8443
  epochs: 1
  pass_all: 0.8723
  pass_any: 0.8723
  score_public: 0.8472
  score_heldout: 0.8999
  parametric: 0.874
- model: gpt-6-astra
  harness: openrouter
  language: cadquery@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8589
  ci95:
  - 0.6589
  - 0.983
  spec_pass: 0.9
  requirements: 0.9375
  valid: 1.0
  printability: 0.8433
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: claude-opus-5-5
  harness: claude-cli
  language: cadquery@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.853
  ci95:
  - 0.6518
  - 0.9784
  spec_pass: 0.9
  requirements: 0.9
  valid: 0.9
  printability: 0.7971
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: claude-fable-5-1
  harness: claude-cli
  language: cadquery@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.7158
  ci95:
  - 0.5054
  - 0.9166
  spec_pass: 0.9
  requirements: 0.9875
  valid: 1.0
  printability: 0.8037
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: kimi-k3
  harness: openrouter
  language: cadquery@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.6944
  ci95:
  - 0.3959
  - 0.9851
  spec_pass: 0.7
  requirements: 0.8798
  valid: 0.9
  printability: 0.8552
  epochs: 1
  pass_all: 0.7
  pass_any: 0.7
- model: kimi-k3
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9457
  ci95:
  - 0.8819
  - 0.9992
  spec_pass: 0.9574
  requirements: 0.9657
  valid: 1.0
  printability: 0.892
  epochs: 1
  pass_all: 0.9574
  pass_any: 0.9574
  score_public: 0.9312
  score_heldout: 0.9994
  parametric: 0.8972
- model: claude-opus-5-5
  harness: claude-cli
  language: freecad
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9253
  ci95:
  - 0.8576
  - 0.9815
  spec_pass: 0.9362
  requirements: 0.9591
  valid: 0.9787
  printability: 0.9149
  epochs: 3
  pass_all: 0.9149
  pass_any: 0.9574
  score_public: 0.9323
  score_heldout: 0.8996
  parametric: 0.901
- model: claude-fable-5-1
  harness: claude-cli
  language: freecad
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9074
  ci95:
  - 0.8292
  - 0.9719
  spec_pass: 0.9149
  requirements: 0.9627
  valid: 0.9929
  printability: 0.9329
  epochs: 3
  pass_all: 0.8936
  pass_any: 0.9362
  score_public: 0.9185
  score_heldout: 0.8662
  parametric: 0.9107
- model: gpt-6-sol
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9038
  ci95:
  - 0.8188
  - 0.9785
  spec_pass: 0.9149
  requirements: 0.9278
  valid: 0.9574
  printability: 0.8841
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9049
  score_heldout: 0.8994
  parametric: 0.9185
- model: grok-4.6
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9011
  ci95:
  - 0.8183
  - 0.9759
  spec_pass: 0.9149
  requirements: 0.9518
  valid: 0.9787
  printability: 0.8889
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9014
  score_heldout: 0.9
  parametric: 0.913
- model: gpt-6-astra
  harness: openrouter
  language: freecad
  attempts: 1
  n: 94
  prompts: 47
  score: 0.8986
  ci95:
  - 0.8134
  - 0.9732
  spec_pass: 0.9149
  requirements: 0.9577
  valid: 0.9894
  printability: 0.917
  epochs: 2
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.8983
  score_heldout: 0.8994
  parametric: 0.8961
- model: gpt-6-luna
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8951
  ci95:
  - 0.8104
  - 0.9677
  spec_pass: 0.9362
  requirements: 0.9441
  valid: 0.9574
  printability: 0.847
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.894
  score_heldout: 0.8991
  parametric: 0.9222
- model: claude-opus-5
  harness: claude-cli
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8783
  ci95:
  - 0.7853
  - 0.9605
  spec_pass: 0.8936
  requirements: 0.8986
  valid: 0.8936
  printability: 0.8255
  epochs: 1
  pass_all: 0.8936
  pass_any: 0.8936
  score_public: 0.8726
  score_heldout: 0.8993
  parametric: 0.9286
- model: grok-4.7
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8736
  ci95:
  - 0.7794
  - 0.9481
  spec_pass: 0.9149
  requirements: 0.9344
  valid: 0.9574
  printability: 0.885
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.8936
  score_heldout: 0.7995
  parametric: 0.9185
- model: gpt-6-luna-pro
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8693
  ci95:
  - 0.7838
  - 0.9488
  spec_pass: 0.8936
  requirements: 0.9519
  valid: 0.9787
  printability: 0.8907
  epochs: 1
  pass_all: 0.8936
  pass_any: 0.8936
  score_public: 0.8611
  score_heldout: 0.8994
  parametric: 0.9058
- model: gpt-5.1
  harness: openrouter
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.3384
  ci95:
  - 0.2154
  - 0.4667
  spec_pass: 0.4043
  requirements: 0.5238
  valid: 0.5745
  printability: 0.5319
  epochs: 1
  pass_all: 0.4043
  pass_any: 0.4043
  score_public: 0.3758
  score_heldout: 0.2
  parametric: 0.821
- model: qwen2.5-coder:7b
  harness: ollama
  language: freecad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0213
  ci95:
  - 0.0
  - 0.0638
  spec_pass: 0.0213
  requirements: 0.0623
  valid: 0.1064
  printability: 0.0744
  epochs: 1
  pass_all: 0.0213
  pass_any: 0.0213
  score_public: 0.027
  score_heldout: 0.0
  parametric: 0.9333
- model: claude-fable-5-1
  harness: claude-cli
  language: freecad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9139
  ci95:
  - 0.829
  - 0.9785
  spec_pass: 0.9149
  requirements: 0.9644
  valid: 1.0
  printability: 0.9335
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9178
  score_heldout: 0.8995
  parametric: 0.9149
- model: claude-opus-5-5
  harness: claude-cli
  language: freecad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9039
  ci95:
  - 0.8188
  - 0.9786
  spec_pass: 0.9149
  requirements: 0.9244
  valid: 0.9362
  printability: 0.8691
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9049
  score_heldout: 0.8999
  parametric: 0.9091
- model: gpt-6-astra
  harness: openrouter
  language: freecad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9038
  ci95:
  - 0.8188
  - 0.9785
  spec_pass: 0.9149
  requirements: 0.9639
  valid: 1.0
  printability: 0.9325
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9049
  score_heldout: 0.8994
  parametric: 0.8936
- model: gpt-6-astra
  harness: openrouter
  language: freecad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8435
  ci95:
  - 0.6402
  - 0.9883
  spec_pass: 0.9
  requirements: 0.9625
  valid: 1.0
  printability: 0.9063
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: claude-opus-5-5
  harness: claude-cli
  language: freecad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8412
  ci95:
  - 0.6854
  - 0.9539
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.7998
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
- model: kimi-k3
  harness: openrouter
  language: freecad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.6922
  ci95:
  - 0.4289
  - 0.8991
  spec_pass: 0.8
  requirements: 0.9735
  valid: 1.0
  printability: 0.8701
  epochs: 1
  pass_all: 0.8
  pass_any: 0.8
- model: claude-fable-5-1
  harness: claude-cli
  language: freecad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.6173
  ci95:
  - 0.3466
  - 0.8569
  spec_pass: 0.7
  requirements: 0.8625
  valid: 0.9
  printability: 0.7351
  epochs: 1
  pass_all: 0.7
  pass_any: 0.7
- model: claude-fable-5-1
  harness: claude-cli
  language: fusion
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9478
  ci95:
  - 0.9053
  - 0.9815
  spec_pass: 0.9504
  requirements: 0.9637
  valid: 0.9645
  printability: 0.8884
  epochs: 3
  pass_all: 0.8723
  pass_any: 1.0
  score_public: 0.9518
  score_heldout: 0.9333
  parametric: 0.9485
- model: claude-opus-5-5
  harness: claude-cli
  language: fusion
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9313
  ci95:
  - 0.8754
  - 0.9768
  spec_pass: 0.9362
  requirements: 0.9721
  valid: 0.9787
  printability: 0.8902
  epochs: 3
  pass_all: 0.8511
  pass_any: 0.9787
  score_public: 0.9578
  score_heldout: 0.8333
  parametric: 0.9227
- model: gpt-6-astra
  harness: openrouter
  language: fusion
  attempts: 1
  n: 94
  prompts: 47
  score: 0.8397
  ci95:
  - 0.754
  - 0.9147
  spec_pass: 0.8511
  requirements: 0.8788
  valid: 0.9043
  printability: 0.8276
  epochs: 2
  pass_all: 0.7872
  pass_any: 0.9149
  score_public: 0.891
  score_heldout: 0.65
  parametric: 0.9314
- model: gpt-6-sol
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8184
  ci95:
  - 0.7015
  - 0.9143
  spec_pass: 0.8298
  requirements: 0.8646
  valid: 0.8936
  printability: 0.7837
  epochs: 1
  pass_all: 0.8298
  pass_any: 0.8298
  score_public: 0.7964
  score_heldout: 0.9
  parametric: 0.9286
- model: grok-4.7
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7337
  ci95:
  - 0.6164
  - 0.8511
  spec_pass: 0.7447
  requirements: 0.7745
  valid: 0.7872
  printability: 0.7414
  epochs: 1
  pass_all: 0.7447
  pass_any: 0.7447
  score_public: 0.7699
  score_heldout: 0.6
  parametric: 0.973
- model: grok-4.6
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.6695
  ci95:
  - 0.5416
  - 0.797
  spec_pass: 0.7021
  requirements: 0.7481
  valid: 0.8085
  printability: 0.7352
  epochs: 1
  pass_all: 0.7021
  pass_any: 0.7021
  score_public: 0.6883
  score_heldout: 0.6
  parametric: 0.9561
- model: claude-opus-5
  harness: claude-cli
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.542
  ci95:
  - 0.404
  - 0.6794
  spec_pass: 0.5532
  requirements: 0.6619
  valid: 0.7234
  printability: 0.6187
  epochs: 1
  pass_all: 0.5532
  pass_any: 0.5532
  score_public: 0.5534
  score_heldout: 0.4999
  parametric: 0.8775
- model: gpt-6-luna-pro
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.5206
  ci95:
  - 0.3824
  - 0.6687
  spec_pass: 0.5319
  requirements: 0.6191
  valid: 0.7234
  printability: 0.5757
  epochs: 1
  pass_all: 0.5319
  pass_any: 0.5319
  score_public: 0.5532
  score_heldout: 0.4
  parametric: 0.9265
- model: gpt-6-luna
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.425
  ci95:
  - 0.2868
  - 0.5631
  spec_pass: 0.4468
  requirements: 0.5305
  valid: 0.5957
  printability: 0.4624
  epochs: 1
  pass_all: 0.4468
  pass_any: 0.4468
  score_public: 0.4453
  score_heldout: 0.35
  parametric: 0.9286
- model: kimi-k3
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.3719
  ci95:
  - 0.234
  - 0.5201
  spec_pass: 0.383
  requirements: 0.5075
  valid: 0.5957
  printability: 0.4926
  epochs: 1
  pass_all: 0.383
  pass_any: 0.383
  score_public: 0.3913
  score_heldout: 0.3
  parametric: 0.9524
- model: gpt-5.1
  harness: openrouter
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0426
  ci95:
  - 0.0
  - 0.1064
  spec_pass: 0.0426
  requirements: 0.0767
  valid: 0.1064
  printability: 0.0853
  epochs: 1
  pass_all: 0.0426
  pass_any: 0.0426
  score_public: 0.027
  score_heldout: 0.1
  parametric: 0.6667
- model: qwen2.5-coder:7b
  harness: ollama
  language: fusion
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0
  ci95:
  - 0.0
  - 0.0
  spec_pass: 0.0
  requirements: 0.0
  valid: 0.0
  printability: 0.0
  epochs: 1
  pass_all: 0.0
  pass_any: 0.0
  score_public: 0.0
  score_heldout: 0.0
- model: claude-opus-5-5
  harness: claude-cli+image
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9983
  ci95:
  - 0.9954
  - 1.0
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9025
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9979
  score_heldout: 1.0
  parametric: 0.953
- model: claude-fable-5-1
  harness: claude-cli
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9975
  ci95:
  - 0.9938
  - 0.9998
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9021
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9968
  score_heldout: 1.0
  parametric: 0.9407
- model: claude-fable-5-1
  harness: claude-cli+image
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9975
  ci95:
  - 0.9938
  - 0.9998
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9021
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9968
  score_heldout: 1.0
  parametric: 0.9394
- model: gpt-6-astra
  harness: openrouter+image
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9248
  ci95:
  - 0.8503
  - 0.9989
  spec_pass: 0.9362
  requirements: 0.9743
  valid: 1.0
  printability: 0.9145
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.9316
  score_heldout: 0.8999
  parametric: 0.9615
- model: claude-opus-5-5
  harness: claude-cli
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9132
  ci95:
  - 0.8283
  - 0.9783
  spec_pass: 0.9149
  requirements: 0.984
  valid: 1.0
  printability: 0.8968
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9438
  score_heldout: 0.8
  parametric: 0.9222
- model: gpt-6-astra
  harness: openrouter
  language: fusion
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9036
  ci95:
  - 0.8185
  - 0.9782
  spec_pass: 0.9149
  requirements: 0.9486
  valid: 1.0
  printability: 0.9039
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9316
  score_heldout: 0.8
  parametric: 0.9431
- model: claude-fable-5-1
  harness: claude-cli
  language: fusion@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.9524
  ci95:
  - 0.8728
  - 0.9994
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9287
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
- model: claude-opus-5-5
  harness: claude-cli
  language: fusion@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.8535
  ci95:
  - 0.6395
  - 0.9814
  spec_pass: 0.9
  requirements: 0.925
  valid: 0.9
  printability: 0.792
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: gpt-6-astra
  harness: openrouter
  language: fusion@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.7129
  ci95:
  - 0.4554
  - 0.9349
  spec_pass: 0.8
  requirements: 0.8875
  valid: 0.9
  printability: 0.7058
  epochs: 1
  pass_all: 0.8
  pass_any: 0.8
- model: kimi-k3
  harness: openrouter
  language: fusion@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.1999
  ci95:
  - 0.0
  - 0.4
  spec_pass: 0.2
  requirements: 0.391
  valid: 0.6
  printability: 0.4156
  epochs: 1
  pass_all: 0.2
  pass_any: 0.2
- model: baseline:reference
  harness: baseline
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9972
  ci95:
  - 0.9918
  - 1.0
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.8835
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9965
  score_heldout: 1.0
- model: claude-fable-5-1
  harness: claude-cli
  language: openscad
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9703
  ci95:
  - 0.9419
  - 0.9925
  spec_pass: 0.9716
  requirements: 0.9754
  valid: 1.0
  printability: 0.9271
  epochs: 3
  pass_all: 0.9149
  pass_any: 1.0
  score_public: 0.9712
  score_heldout: 0.9667
  parametric: 0.9184
- model: claude-opus-5-5
  harness: claude-cli
  language: openscad
  attempts: 1
  n: 141
  prompts: 47
  score: 0.9685
  ci95:
  - 0.9291
  - 0.9978
  spec_pass: 0.9716
  requirements: 0.9754
  valid: 0.9716
  printability: 0.9132
  epochs: 3
  pass_all: 0.9362
  pass_any: 1.0
  score_public: 0.969
  score_heldout: 0.9666
  parametric: 0.9501
- model: gpt-6-astra
  harness: openrouter
  language: openscad
  attempts: 1
  n: 94
  prompts: 47
  score: 0.9251
  ci95:
  - 0.8507
  - 0.9888
  spec_pass: 0.9362
  requirements: 0.9434
  valid: 0.9362
  printability: 0.8728
  epochs: 2
  pass_all: 0.9149
  pass_any: 0.9574
  score_public: 0.9184
  score_heldout: 0.95
  parametric: 0.9564
- model: grok-4.6
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.9219
  ci95:
  - 0.8441
  - 0.9863
  spec_pass: 0.9574
  requirements: 0.9629
  valid: 0.9787
  printability: 0.887
  epochs: 1
  pass_all: 0.9574
  pass_any: 0.9574
  score_public: 0.9547
  score_heldout: 0.8002
  parametric: 0.9601
- model: grok-4.7
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.901
  ci95:
  - 0.8156
  - 0.976
  spec_pass: 0.9149
  requirements: 0.9501
  valid: 0.9574
  printability: 0.8762
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9283
  score_heldout: 0.8
  parametric: 0.9593
- model: gpt-6-sol
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8827
  ci95:
  - 0.7861
  - 0.9678
  spec_pass: 0.8936
  requirements: 0.9246
  valid: 0.9362
  printability: 0.8619
  epochs: 1
  pass_all: 0.8936
  pass_any: 0.8936
  score_public: 0.905
  score_heldout: 0.8
  parametric: 0.947
- model: claude-opus-5
  harness: claude-cli
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8206
  ci95:
  - 0.7162
  - 0.9212
  spec_pass: 0.8511
  requirements: 0.9182
  valid: 0.9362
  printability: 0.7697
  epochs: 1
  pass_all: 0.8511
  pass_any: 0.8511
  score_public: 0.7992
  score_heldout: 0.8995
  parametric: 0.9318
- model: kimi-k3
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8189
  ci95:
  - 0.7021
  - 0.9247
  spec_pass: 0.8298
  requirements: 0.8796
  valid: 0.9149
  printability: 0.8345
  epochs: 1
  pass_all: 0.8298
  pass_any: 0.8298
  score_public: 0.851
  score_heldout: 0.7
  parametric: 0.9246
- model: gpt-6-luna-pro
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.8161
  ci95:
  - 0.7016
  - 0.9225
  spec_pass: 0.8298
  requirements: 0.8943
  valid: 0.9149
  printability: 0.8587
  epochs: 1
  pass_all: 0.8298
  pass_any: 0.8298
  score_public: 0.8205
  score_heldout: 0.8
  parametric: 0.9225
- model: gpt-6-luna
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.752
  ci95:
  - 0.6241
  - 0.8691
  spec_pass: 0.766
  requirements: 0.8535
  valid: 0.8936
  printability: 0.8263
  epochs: 1
  pass_all: 0.766
  pass_any: 0.766
  score_public: 0.7121
  score_heldout: 0.9
  parametric: 0.9603
- model: gpt-5.1
  harness: openrouter
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.3344
  ci95:
  - 0.2178
  - 0.4719
  spec_pass: 0.4043
  requirements: 0.6575
  valid: 0.8511
  printability: 0.7027
  epochs: 1
  pass_all: 0.4043
  pass_any: 0.4043
  score_public: 0.3166
  score_heldout: 0.4
  parametric: 0.7833
- model: qwen2.5-coder:7b
  harness: ollama
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0213
  ci95:
  - 0.0
  - 0.0638
  spec_pass: 0.0213
  requirements: 0.0999
  valid: 0.234
  printability: 0.2194
  epochs: 1
  pass_all: 0.0213
  pass_any: 0.0213
  score_public: 0.0
  score_heldout: 0.1
  parametric: 0.5758
- model: baseline:cube
  harness: baseline
  language: openscad
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0
  ci95:
  - 0.0
  - 0.0
  spec_pass: 0.0
  requirements: 0.2597
  valid: 1.0
  printability: 1.0
  epochs: 1
  pass_all: 0.0
  pass_any: 0.0
  score_public: 0.0
  score_heldout: 0.0
- model: claude-fable-5-1
  harness: claude-cli
  language: openscad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9759
  ci95:
  - 0.9306
  - 1.0
  spec_pass: 0.9787
  requirements: 0.9818
  valid: 1.0
  printability: 0.916
  epochs: 1
  pass_all: 0.9787
  pass_any: 0.9787
  score_public: 0.9965
  score_heldout: 0.9
  parametric: 0.9078
- model: claude-opus-5-5
  harness: claude-cli
  language: openscad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9547
  ci95:
  - 0.8908
  - 1.0
  spec_pass: 0.9574
  requirements: 0.9634
  valid: 0.9574
  printability: 0.8936
  epochs: 1
  pass_all: 0.9574
  pass_any: 0.9574
  score_public: 0.9424
  score_heldout: 0.9999
  parametric: 0.9444
- model: gpt-6-astra
  harness: openrouter
  language: openscad
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9252
  ci95:
  - 0.8505
  - 0.9891
  spec_pass: 0.9362
  requirements: 0.944
  valid: 0.9362
  printability: 0.8765
  epochs: 1
  pass_all: 0.9362
  pass_any: 0.9362
  score_public: 0.905
  score_heldout: 1.0
  parametric: 0.9508
- model: claude-fable-5-1
  harness: claude-cli
  language: openscad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.7912
  ci95:
  - 0.5793
  - 0.9939
  spec_pass: 0.8
  requirements: 0.8515
  valid: 0.9
  printability: 0.8733
  epochs: 1
  pass_all: 0.8
  pass_any: 0.8
- model: claude-opus-5-5
  harness: claude-cli
  language: openscad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.7747
  ci95:
  - 0.5537
  - 0.9337
  spec_pass: 0.9
  requirements: 0.925
  valid: 1.0
  printability: 0.821
  epochs: 1
  pass_all: 0.9
  pass_any: 0.9
- model: gpt-6-astra
  harness: openrouter
  language: openscad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.4853
  ci95:
  - 0.1853
  - 0.7706
  spec_pass: 0.5
  requirements: 0.6433
  valid: 0.9
  printability: 0.8242
  epochs: 1
  pass_all: 0.5
  pass_any: 0.5
- model: kimi-k3
  harness: openrouter
  language: openscad@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.4389
  ci95:
  - 0.1539
  - 0.7239
  spec_pass: 0.5
  requirements: 0.6598
  valid: 0.9
  printability: 0.7723
  epochs: 1
  pass_all: 0.5
  pass_any: 0.5
- model: gpt-6-astra
  harness: openrouter
  language: rhino
  attempts: 1
  n: 94
  prompts: 47
  score: 0.8929
  ci95:
  - 0.8286
  - 0.9464
  spec_pass: 0.9043
  requirements: 0.9049
  valid: 0.9043
  printability: 0.8169
  epochs: 2
  pass_all: 0.8298
  pass_any: 0.9787
  score_public: 0.8639
  score_heldout: 1.0
  parametric: 0.9137
- model: claude-opus-5-5
  harness: claude-cli
  language: rhino
  attempts: 1
  n: 141
  prompts: 47
  score: 0.8464
  ci95:
  - 0.7648
  - 0.9234
  spec_pass: 0.8652
  requirements: 0.9248
  valid: 0.9645
  printability: 0.8914
  epochs: 3
  pass_all: 0.7447
  pass_any: 0.9574
  score_public: 0.8409
  score_heldout: 0.8667
  parametric: 0.9277
- model: claude-fable-5-1
  harness: claude-cli
  language: rhino
  attempts: 1
  n: 141
  prompts: 47
  score: 0.7846
  ci95:
  - 0.7015
  - 0.8573
  spec_pass: 0.7872
  requirements: 0.8395
  valid: 0.922
  printability: 0.8281
  epochs: 3
  pass_all: 0.5532
  pass_any: 0.9574
  score_public: 0.7984
  score_heldout: 0.7333
  parametric: 0.9205
- model: grok-4.6
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7546
  ci95:
  - 0.6367
  - 0.8714
  spec_pass: 0.766
  requirements: 0.7918
  valid: 0.8298
  printability: 0.7526
  epochs: 1
  pass_all: 0.766
  pass_any: 0.766
  score_public: 0.7153
  score_heldout: 0.9
  parametric: 0.9402
- model: grok-4.7
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.7048
  ci95:
  - 0.5761
  - 0.8353
  spec_pass: 0.7447
  requirements: 0.7558
  valid: 0.766
  printability: 0.6924
  epochs: 1
  pass_all: 0.7447
  pass_any: 0.7447
  score_public: 0.6791
  score_heldout: 0.8
  parametric: 0.9491
- model: claude-opus-5
  harness: claude-cli
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.6751
  ci95:
  - 0.5476
  - 0.8034
  spec_pass: 0.7021
  requirements: 0.7964
  valid: 0.9149
  printability: 0.7632
  epochs: 1
  pass_all: 0.7021
  pass_any: 0.7021
  score_public: 0.6684
  score_heldout: 0.6998
  parametric: 0.845
- model: kimi-k3
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.6167
  ci95:
  - 0.4784
  - 0.7549
  spec_pass: 0.6383
  requirements: 0.6736
  valid: 0.7234
  printability: 0.636
  epochs: 1
  pass_all: 0.6383
  pass_any: 0.6383
  score_public: 0.5672
  score_heldout: 0.8
  parametric: 0.902
- model: gpt-6-sol
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.5954
  ci95:
  - 0.4465
  - 0.7234
  spec_pass: 0.617
  requirements: 0.6349
  valid: 0.6596
  printability: 0.6087
  epochs: 1
  pass_all: 0.617
  pass_any: 0.617
  score_public: 0.5402
  score_heldout: 0.8
  parametric: 0.8817
- model: gpt-6-luna-pro
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.4465
  ci95:
  - 0.2979
  - 0.5848
  spec_pass: 0.4681
  requirements: 0.483
  valid: 0.5106
  printability: 0.4606
  epochs: 1
  pass_all: 0.4681
  pass_any: 0.4681
  score_public: 0.4861
  score_heldout: 0.3
  parametric: 0.9653
- model: gpt-6-luna
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.3082
  ci95:
  - 0.1805
  - 0.4463
  spec_pass: 0.3191
  requirements: 0.3275
  valid: 0.3404
  printability: 0.3062
  epochs: 1
  pass_all: 0.3191
  pass_any: 0.3191
  score_public: 0.3104
  score_heldout: 0.3
  parametric: 0.9583
- model: gpt-5.1
  harness: openrouter
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.2341
  ci95:
  - 0.1277
  - 0.3617
  spec_pass: 0.2553
  requirements: 0.3431
  valid: 0.4043
  printability: 0.3482
  epochs: 1
  pass_all: 0.2553
  pass_any: 0.2553
  score_public: 0.2434
  score_heldout: 0.2
  parametric: 0.7632
- model: qwen2.5-coder:7b
  harness: ollama
  language: rhino
  attempts: 1
  n: 47
  prompts: 47
  score: 0.0
  ci95:
  - 0.0
  - 0.0
  spec_pass: 0.0
  requirements: 0.0
  valid: 0.0
  printability: 0.0
  epochs: 1
  pass_all: 0.0
  pass_any: 0.0
  score_public: 0.0
  score_heldout: 0.0
- model: gpt-6-astra
  harness: openrouter
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9886
  ci95:
  - 0.9663
  - 1.0
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9181
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9855
  score_heldout: 1.0
  parametric: 0.9147
- model: gpt-6-astra
  harness: openrouter+image
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9886
  ci95:
  - 0.9663
  - 1.0
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9181
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9855
  score_heldout: 1.0
  parametric: 0.9147
- model: claude-opus-5-5
  harness: claude-cli+image
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9672
  ci95:
  - 0.9145
  - 0.9997
  spec_pass: 1.0
  requirements: 1.0
  valid: 1.0
  printability: 0.9384
  epochs: 1
  pass_all: 1.0
  pass_any: 1.0
  score_public: 0.9853
  score_heldout: 0.9002
  parametric: 0.9298
- model: claude-fable-5-1
  harness: claude-cli+image
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9528
  ci95:
  - 0.8874
  - 0.9977
  spec_pass: 0.9574
  requirements: 0.9734
  valid: 0.9787
  printability: 0.9163
  epochs: 1
  pass_all: 0.9574
  pass_any: 0.9574
  score_public: 0.94
  score_heldout: 1.0
  parametric: 0.9677
- model: claude-opus-5-5
  harness: claude-cli
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.9033
  ci95:
  - 0.8182
  - 0.9776
  spec_pass: 0.9149
  requirements: 0.9553
  valid: 0.9787
  printability: 0.9228
  epochs: 1
  pass_all: 0.9149
  pass_any: 0.9149
  score_public: 0.9042
  score_heldout: 0.8999
  parametric: 0.9205
- model: claude-fable-5-1
  harness: claude-cli
  language: rhino
  attempts: 2
  n: 47
  prompts: 47
  score: 0.7613
  ci95:
  - 0.6379
  - 0.8708
  spec_pass: 0.766
  requirements: 0.8412
  valid: 0.9787
  printability: 0.8333
  epochs: 1
  pass_all: 0.766
  pass_any: 0.766
  score_public: 0.7508
  score_heldout: 0.8
  parametric: 0.8953
- model: claude-opus-5-5
  harness: claude-cli
  language: rhino@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.5632
  ci95:
  - 0.2651
  - 0.829
  spec_pass: 0.6
  requirements: 0.7918
  valid: 0.8
  printability: 0.7126
  epochs: 1
  pass_all: 0.6
  pass_any: 0.6
- model: kimi-k3
  harness: openrouter
  language: rhino@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.45
  ci95:
  - 0.1982
  - 0.7488
  spec_pass: 0.6
  requirements: 0.625
  valid: 0.6
  printability: 0.5315
  epochs: 1
  pass_all: 0.6
  pass_any: 0.6
- model: claude-fable-5-1
  harness: claude-cli
  language: rhino@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.3968
  ci95:
  - 0.0992
  - 0.6952
  spec_pass: 0.4
  requirements: 0.5545
  valid: 0.8
  printability: 0.5466
  epochs: 1
  pass_all: 0.4
  pass_any: 0.4
- model: gpt-6-astra
  harness: openrouter
  language: rhino@trackc
  attempts: 1
  n: 10
  prompts: 10
  score: 0.3764
  ci95:
  - 0.0994
  - 0.6757
  spec_pass: 0.4
  requirements: 0.4422
  valid: 0.4
  printability: 0.3841
  epochs: 1
  pass_all: 0.4
  pass_any: 0.4
paired:
- a: claude-opus-5-5
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8994
  ci95:
  - 0.8569
  - 0.9348
  separated: true
- a: gpt-6-astra
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8908
  ci95:
  - 0.8357
  - 0.9376
  separated: true
- a: claude-fable-5-1
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8892
  ci95:
  - 0.8445
  - 0.9282
  separated: true
- a: gpt-6-sol
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8323
  ci95:
  - 0.7777
  - 0.8824
  separated: true
- a: grok-4.6
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8065
  ci95:
  - 0.7432
  - 0.863
  separated: true
- a: grok-4.7
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.8046
  ci95:
  - 0.7459
  - 0.8612
  separated: true
- a: gpt-6-luna-pro
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7468
  ci95:
  - 0.6839
  - 0.8064
  separated: true
- a: claude-opus-5
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7193
  ci95:
  - 0.6685
  - 0.7683
  separated: true
- a: claude-opus-5-5
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7189
  ci95:
  - 0.6578
  - 0.779
  separated: true
- a: kimi-k3
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7168
  ci95:
  - 0.6529
  - 0.7762
  separated: true
- a: gpt-6-astra
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7103
  ci95:
  - 0.6384
  - 0.7754
  separated: true
- a: claude-fable-5-1
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.7086
  ci95:
  - 0.6483
  - 0.7693
  separated: true
- a: gpt-6-luna
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.6773
  ci95:
  - 0.6079
  - 0.7425
  separated: true
- a: gpt-6-sol
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.6518
  ci95:
  - 0.5832
  - 0.7192
  separated: true
- a: grok-4.6
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.626
  ci95:
  - 0.5557
  - 0.6935
  separated: true
- a: grok-4.7
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.624
  ci95:
  - 0.5525
  - 0.6966
  separated: true
- a: gpt-6-luna-pro
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.5663
  ci95:
  - 0.4985
  - 0.6345
  separated: true
- a: claude-opus-5
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.5387
  ci95:
  - 0.4777
  - 0.6013
  separated: true
- a: kimi-k3
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.5362
  ci95:
  - 0.4673
  - 0.6031
  separated: true
- a: gpt-6-luna
  b: gpt-5.1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.4968
  ci95:
  - 0.4252
  - 0.5661
  separated: true
- a: claude-opus-5-5
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.2221
  ci95:
  - 0.1696
  - 0.2745
  separated: true
- a: gpt-6-astra
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.2135
  ci95:
  - 0.1695
  - 0.258
  separated: true
- a: claude-fable-5-1
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.2119
  ci95:
  - 0.1591
  - 0.2684
  separated: true
- a: claude-opus-5-5
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1827
  ci95:
  - 0.1376
  - 0.2274
  separated: true
- a: gpt-5.1
  b: qwen2.5-coder:7b
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1806
  ci95:
  - 0.1247
  - 0.2396
  separated: true
- a: claude-opus-5-5
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1801
  ci95:
  - 0.1395
  - 0.2206
  separated: true
- a: gpt-6-astra
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1741
  ci95:
  - 0.1277
  - 0.2202
  separated: true
- a: claude-fable-5-1
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1724
  ci95:
  - 0.1287
  - 0.2185
  separated: true
- a: gpt-6-astra
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1715
  ci95:
  - 0.126
  - 0.2147
  separated: true
- a: claude-fable-5-1
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1699
  ci95:
  - 0.1258
  - 0.2137
  separated: true
- a: gpt-6-sol
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.155
  ci95:
  - 0.1139
  - 0.1968
  separated: true
- a: claude-opus-5-5
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1526
  ci95:
  - 0.1084
  - 0.1969
  separated: true
- a: gpt-6-astra
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.144
  ci95:
  - 0.103
  - 0.1839
  separated: true
- a: claude-fable-5-1
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1424
  ci95:
  - 0.0977
  - 0.1919
  separated: true
- a: grok-4.6
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1292
  ci95:
  - 0.0784
  - 0.1805
  separated: true
- a: grok-4.7
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1272
  ci95:
  - 0.0698
  - 0.1855
  separated: true
- a: gpt-6-sol
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.1156
  ci95:
  - 0.0639
  - 0.1671
  separated: true
- a: gpt-6-sol
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.113
  ci95:
  - 0.0649
  - 0.1624
  separated: true
- a: claude-opus-5-5
  b: grok-4.7
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0949
  ci95:
  - 0.0536
  - 0.1364
  separated: true
- a: claude-opus-5-5
  b: grok-4.6
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0929
  ci95:
  - 0.0561
  - 0.1336
  separated: true
- a: grok-4.6
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0898
  ci95:
  - 0.0382
  - 0.1411
  separated: true
- a: grok-4.7
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0878
  ci95:
  - 0.0372
  - 0.1389
  separated: true
- a: grok-4.6
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0872
  ci95:
  - 0.0425
  - 0.1312
  separated: true
- a: gpt-6-astra
  b: grok-4.7
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0862
  ci95:
  - 0.0475
  - 0.1279
  separated: true
- a: gpt-6-sol
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0855
  ci95:
  - 0.048
  - 0.1264
  separated: true
- a: grok-4.7
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0853
  ci95:
  - 0.0347
  - 0.138
  separated: true
- a: claude-fable-5-1
  b: grok-4.7
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0846
  ci95:
  - 0.0418
  - 0.1287
  separated: true
- a: gpt-6-astra
  b: grok-4.6
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0843
  ci95:
  - 0.0427
  - 0.1241
  separated: true
- a: claude-fable-5-1
  b: grok-4.6
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0827
  ci95:
  - 0.0423
  - 0.1256
  separated: true
- a: gpt-6-luna-pro
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0695
  ci95:
  - 0.0327
  - 0.1063
  separated: true
- a: claude-opus-5-5
  b: gpt-6-sol
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0671
  ci95:
  - 0.0292
  - 0.1047
  separated: true
- a: grok-4.6
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0597
  ci95:
  - 0.0222
  - 0.0978
  separated: true
- a: gpt-6-astra
  b: gpt-6-sol
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0585
  ci95:
  - 0.0342
  - 0.0828
  separated: true
- a: grok-4.7
  b: gpt-6-luna-pro
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0577
  ci95:
  - 0.011
  - 0.1051
  separated: true
- a: claude-fable-5-1
  b: gpt-6-sol
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0569
  ci95:
  - 0.0164
  - 0.0997
  separated: true
- a: claude-opus-5
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.042
  ci95:
  - -0.0227
  - 0.1085
  separated: false
- a: kimi-k3
  b: gpt-6-luna
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0394
  ci95:
  - -0.0202
  - 0.0988
  separated: false
- a: gpt-6-luna-pro
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0301
  ci95:
  - -0.0216
  - 0.0788
  separated: false
- a: gpt-6-sol
  b: grok-4.7
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0278
  ci95:
  - -0.0141
  - 0.0704
  separated: false
- a: gpt-6-luna-pro
  b: claude-opus-5
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0275
  ci95:
  - -0.0279
  - 0.0839
  separated: false
- a: gpt-6-sol
  b: grok-4.6
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0258
  ci95:
  - -0.0144
  - 0.0666
  separated: false
- a: claude-opus-5-5
  b: claude-fable-5-1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0102
  ci95:
  - -0.0045
  - 0.0257
  separated: false
- a: claude-opus-5-5
  b: gpt-6-astra
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0086
  ci95:
  - -0.0235
  - 0.0423
  separated: false
- a: claude-opus-5
  b: kimi-k3
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0026
  ci95:
  - -0.0542
  - 0.0604
  separated: false
- a: grok-4.6
  b: grok-4.7
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0019
  ci95:
  - -0.043
  - 0.0459
  separated: false
- a: gpt-6-astra
  b: claude-fable-5-1
  scope: all
  languages: 7
  prompts: 47
  delta: 0.0016
  ci95:
  - -0.0401
  - 0.0379
  separated: false
spec_audit:
  c1-001:
    mutation_score: 1.0
    mutants: 8
  c1-002:
    mutation_score: 1.0
    mutants: 10
  c1-003:
    mutation_score: 1.0
    mutants: 10
  c2-004:
    mutation_score: 1.0
    mutants: 11
  c2-005:
    mutation_score: 1.0
    mutants: 9
  c2-006:
    mutation_score: 1.0
    mutants: 9
  c3-007:
    mutation_score: 1.0
    mutants: 11
  c3-008:
    mutation_score: 1.0
    mutants: 12
  c4-009:
    mutation_score: 1.0
    mutants: 10
  c4-010:
    mutation_score: 1.0
    mutants: 11
  t1-001:
    mutation_score: 1.0
    mutants: 8
  t1-002:
    mutation_score: 1.0
    mutants: 8
  t1-003:
    mutation_score: 1.0
    mutants: 7
  t1-004:
    mutation_score: 1.0
    mutants: 9
  t1-005:
    mutation_score: 1.0
    mutants: 7
  t1-006:
    mutation_score: 1.0
    mutants: 10
  t1-007:
    mutation_score: 1.0
    mutants: 9
  t1-008:
    mutation_score: 1.0
    mutants: 9
  t1-009:
    mutation_score: 1.0
    mutants: 10
  t1-010:
    mutation_score: 1.0
    mutants: 8
  t2-001:
    mutation_score: 1.0
    mutants: 10
  t2-002:
    mutation_score: 1.0
    mutants: 9
  t2-003:
    mutation_score: 1.0
    mutants: 10
  t2-004:
    mutation_score: 1.0
    mutants: 8
  t2-005:
    mutation_score: 1.0
    mutants: 10
  t2-006:
    mutation_score: 1.0
    mutants: 9
  t2-007:
    mutation_score: 1.0
    mutants: 8
  t2-008:
    mutation_score: 1.0
    mutants: 10
  t2-009:
    mutation_score: 1.0
    mutants: 10
  t2-010:
    mutation_score: 1.0
    mutants: 10
  t3-001:
    mutation_score: 1.0
    mutants: 13
  t3-002:
    mutation_score: 1.0
    mutants: 11
  t3-003:
    mutation_score: 1.0
    mutants: 12
  t3-004:
    mutation_score: 1.0
    mutants: 11
  t3-005:
    mutation_score: 1.0
    mutants: 11
  t3-006:
    mutation_score: 1.0
    mutants: 9
  t3-007:
    mutation_score: 1.0
    mutants: 11
  t3-008:
    mutation_score: 1.0
    mutants: 13
  t3-009:
    mutation_score: 1.0
    mutants: 10
  t3-010:
    mutation_score: 1.0
    mutants: 12
  t4-001:
    mutation_score: 1.0
    mutants: 11
  t4-002:
    mutation_score: 1.0
    mutants: 10
  t4-003:
    mutation_score: 1.0
    mutants: 15
  t4-004:
    mutation_score: 1.0
    mutants: 17
  t4-005:
    mutation_score: 1.0
    mutants: 10
  t4-006:
    mutation_score: 1.0
    mutants: 11
  t4-007:
    mutation_score: 1.0
    mutants: 16
  t4-008:
    mutation_score: 1.0
    mutants: 12
  t4-009:
    mutation_score: 1.0
    mutants: 12
  t4-010:
    mutation_score: 1.0
    mutants: 14
  t5-001:
    mutation_score: 1.0
    mutants: 11
  t5-002:
    mutation_score: 1.0
    mutants: 12
  t5-003:
    mutation_score: 1.0
    mutants: 10
  t5-004:
    mutation_score: 1.0
    mutants: 9
  t5-005:
    mutation_score: 1.0
    mutants: 10
  t5-006:
    mutation_score: 1.0
    mutants: 8
  t5-007:
    mutation_score: 1.0
    mutants: 7
spec_change:
  unique_parts_pass_to_fail: 79
  unique_parts_fail_to_pass: 1
  by_model:
    claude-fable-5-1:
      pass_to_fail: 6
      fail_to_pass: 0
    claude-opus-5:
      pass_to_fail: 9
      fail_to_pass: 0
    claude-opus-5-5:
      pass_to_fail: 4
      fail_to_pass: 0
    gpt-5.1:
      pass_to_fail: 7
      fail_to_pass: 0
    gpt-6-astra:
      pass_to_fail: 6
      fail_to_pass: 1
    gpt-6-luna:
      pass_to_fail: 13
      fail_to_pass: 0
    gpt-6-luna-pro:
      pass_to_fail: 11
      fail_to_pass: 0
    gpt-6-sol:
      pass_to_fail: 4
      fail_to_pass: 0
    grok-4.6:
      pass_to_fail: 6
      fail_to_pass: 0
    grok-4.7:
      pass_to_fail: 4
      fail_to_pass: 0
    kimi-k3:
      pass_to_fail: 7
      fail_to_pass: 0
    qwen2.5-coder:7b:
      pass_to_fail: 2
      fail_to_pass: 0
  review: every flip reviewed by cross-section; verdicts in prompts/v0.2/spec-audit-0.2.2.yaml and prompts/trackc/spec-audit-0.2.2.yaml