{
  "methodology": "pocketllm-english-v1",
  "weights": {
    "instructions": 25,
    "grounding": 20,
    "conversation": 20,
    "reasoning": 15,
    "writing": 10,
    "honesty": 10
  },
  "evaluation_target": "iPhone 17 Pro Max",
  "build_identity": "a5902f1adbd35ee4f54f259b98b884fd2c2824233ce22c7430159c443954e0f7",
  "fixture_sha256": "5da3d487a9965fb331a21bf5021d852faadfad43a550bbc4c8ce862bdb96fdce",
  "scores": [
    {
      "model": "smollm2-135m",
      "candidate": false,
      "status": "scored",
      "quality_score": 8.0,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 1.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 15.0
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 0.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 0.0
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 30.0
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 16.67
        }
      },
      "repeat_scores": [
        6.75,
        8.5,
        8.75
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 41.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 9,
        "all_examples_passed": 3,
        "failed_examples": 6,
        "invalid_python": 7,
        "unchecked": 14,
        "surrounding_prose": 10,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "qwen2.5-0.5b",
      "candidate": false,
      "status": "scored",
      "quality_score": 45.42,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 26.67
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 58.33
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 51.67
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 35.56
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 54.17
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 60.0
        }
      },
      "repeat_scores": [
        47.25,
        45.25,
        43.75
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 80.83,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 23,
        "all_examples_passed": 17,
        "failed_examples": 6,
        "invalid_python": 0,
        "unchecked": 7,
        "surrounding_prose": 2,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "llama3.2-1b",
      "candidate": false,
      "status": "scored",
      "quality_score": 41.67,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 13.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 57.08
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 40.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 35.56
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 66.67
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 69.17
        }
      },
      "repeat_scores": [
        40.75,
        42.25,
        42.0
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 65.0,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 18,
        "all_examples_passed": 12,
        "failed_examples": 6,
        "invalid_python": 0,
        "unchecked": 12,
        "surrounding_prose": 22,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "deepseek-r1-1.5b",
      "candidate": false,
      "status": "scored",
      "quality_score": 11.0,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 0.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 17.08
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 0.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 0.0
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 33.33
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 42.5
        }
      },
      "repeat_scores": [
        11.5,
        12.25,
        9.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 37.5,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 11,
        "all_examples_passed": 9,
        "failed_examples": 2,
        "invalid_python": 19,
        "unchecked": 0,
        "surrounding_prose": 11,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "lfm-2.5-1.2b",
      "candidate": false,
      "status": "scored",
      "quality_score": 64.33,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 61.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 71.67
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 48.33
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 60.0
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 85.0
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 75.0
        }
      },
      "repeat_scores": [
        66.5,
        62.25,
        64.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 96.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 28,
        "all_examples_passed": 27,
        "failed_examples": 1,
        "invalid_python": 0,
        "unchecked": 2,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "gemma-3-1b-it",
      "candidate": false,
      "status": "scored",
      "quality_score": 58.5,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 40.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 67.5
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 50.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 66.67
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 85.0
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 65.0
        }
      },
      "repeat_scores": [
        58.25,
        57.5,
        59.75
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 86.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 30,
        "all_examples_passed": 20,
        "failed_examples": 10,
        "invalid_python": 0,
        "unchecked": 0,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "minicpm5-1b",
      "candidate": false,
      "status": "scored",
      "quality_score": 55.25,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 48.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 58.75
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 55.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 48.89
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 67.5
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 64.17
        }
      },
      "repeat_scores": [
        53.75,
        59.5,
        52.5
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 95.83,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 30,
        "all_examples_passed": 25,
        "failed_examples": 5,
        "invalid_python": 0,
        "unchecked": 0,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "granite-4.0-h-1b",
      "candidate": false,
      "status": "scored",
      "quality_score": 75.67,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 84.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 76.67
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 65.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 71.11
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 80.0
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 76.67
        }
      },
      "repeat_scores": [
        74.25,
        76.0,
        76.75
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 86.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 29,
        "all_examples_passed": 27,
        "failed_examples": 2,
        "invalid_python": 1,
        "unchecked": 0,
        "surrounding_prose": 8,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "minicpm5-2b",
      "candidate": false,
      "status": "scored",
      "quality_score": 86.67,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 92.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 83.75
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 96.67
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 82.22
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 87.5
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 65.0
        }
      },
      "repeat_scores": [
        86.0,
        85.0,
        89.0
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 96.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 30,
        "all_examples_passed": 27,
        "failed_examples": 3,
        "invalid_python": 0,
        "unchecked": 0,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "ministral-3-3b-instruct",
      "candidate": false,
      "status": "scored",
      "quality_score": 68.83,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 62.67
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 73.75
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 63.33
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 75.56
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 84.17
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 60.0
        }
      },
      "repeat_scores": [
        68.5,
        68.5,
        69.5
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 95.83,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 29,
        "all_examples_passed": 27,
        "failed_examples": 2,
        "invalid_python": 0,
        "unchecked": 1,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "granite-4.2-3b",
      "candidate": false,
      "status": "scored",
      "quality_score": 85.33,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 88.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 84.58
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 96.67
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 80.0
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 80.83
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 70.0
        }
      },
      "repeat_scores": [
        85.25,
        86.0,
        84.75
      ],
      "quality_threshold_met": true,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 95.0,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 30,
        "all_examples_passed": 27,
        "failed_examples": 3,
        "invalid_python": 0,
        "unchecked": 0,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "phi-4-mini-instruct",
      "candidate": false,
      "status": "scored",
      "quality_score": 59.67,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 56.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 66.25
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 45.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 75.56
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 70.83
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 50.0
        }
      },
      "repeat_scores": [
        57.75,
        63.0,
        58.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 98.33,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 30,
        "all_examples_passed": 29,
        "failed_examples": 1,
        "invalid_python": 0,
        "unchecked": 0,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "smollm2-1.7b",
      "candidate": false,
      "status": "scored",
      "quality_score": 56.42,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 52.0
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 73.33
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 48.33
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 37.78
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 70.83
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 63.33
        }
      },
      "repeat_scores": [
        50.0,
        59.0,
        60.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 95.0,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 27,
        "all_examples_passed": 25,
        "failed_examples": 2,
        "invalid_python": 0,
        "unchecked": 3,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "qwen2.5-coder-1.5b",
      "candidate": false,
      "status": "scored",
      "quality_score": 70.17,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 53.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 76.25
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 80.0
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 80.0
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 72.5
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 63.33
        }
      },
      "repeat_scores": [
        72.25,
        65.75,
        72.5
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 84.17,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 26,
        "all_examples_passed": 24,
        "failed_examples": 2,
        "invalid_python": 0,
        "unchecked": 4,
        "surrounding_prose": 15,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "qwen3.5-0.8b",
      "candidate": true,
      "status": "scored",
      "quality_score": 61.17,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 77.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 62.5
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 56.67
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 44.44
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 59.17
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 54.17
        }
      },
      "repeat_scores": [
        62.25,
        60.0,
        61.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 76.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 19,
        "all_examples_passed": 15,
        "failed_examples": 4,
        "invalid_python": 0,
        "unchecked": 11,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    },
    {
      "model": "qwen3.5-2b",
      "candidate": true,
      "status": "scored",
      "quality_score": 81.17,
      "graded_cases": 300,
      "expected_cases": 300,
      "categories": {
        "instructions": {
          "evaluated": 75,
          "expected": 75,
          "score": 81.33
        },
        "grounding": {
          "evaluated": 60,
          "expected": 60,
          "score": 82.92
        },
        "conversation": {
          "evaluated": 60,
          "expected": 60,
          "score": 83.33
        },
        "reasoning": {
          "evaluated": 45,
          "expected": 45,
          "score": 82.22
        },
        "writing": {
          "evaluated": 30,
          "expected": 30,
          "score": 81.67
        },
        "honesty": {
          "evaluated": 30,
          "expected": 30,
          "score": 70.83
        }
      },
      "repeat_scores": [
        79.75,
        81.5,
        82.25
      ],
      "quality_threshold_met": false,
      "coding_appendix": {
        "observed_cases": 30,
        "graded_cases": 30,
        "expected_cases": 30,
        "score": 96.67,
        "included_in_overall_score": false
      },
      "functional_coding": {
        "responses": 30,
        "executed": 27,
        "all_examples_passed": 25,
        "failed_examples": 2,
        "invalid_python": 0,
        "unchecked": 3,
        "surrounding_prose": 0,
        "notice": "Finite synthetic Python examples; separate from rubric partial credit and output-format compliance."
      },
      "release_eligible": false
    }
  ],
  "review_method": "Codex assistant rubric review with model labels and timing hidden. The reviewer has run-order context; this is not independent or double-blind human validation.",
  "notice": "Quality scores describe this device, artifact and app configuration on a fixed synthetic English benchmark. Reliability and broader release validation are separate."
}