{
  "accuracy_note": "All final-stack rows matched eager generated text exactly on the cached correctness set. Ground-truth exact accuracy is below 1.0 because the trained model itself makes errors.",
  "backend": "cpu",
  "best_final_vs_eager_batch_128_throughput_speedup": 9.965288270075453,
  "best_final_vs_eager_batch_1_per_example_latency_speedup": 198.4349807130022,
  "best_final_vs_eager_batch_1_throughput_speedup": 198.4349807130022,
  "best_overall_row": {
    "batch_size": 128,
    "compile_time": 8.492741916998057,
    "correctness_eval_batch_size": 128,
    "correctness_path": "/Users/thibaultcastells/Documents/CODE/transformer_tuto/model_optimization/runs/inference_benchmarks/final_cumulative_stack_cpu/correctness_kv_cache_prefill_unrolled_float32.json",
    "correctness_reused_across_batches": true,
    "device_name": "cpu",
    "dtype": "float32",
    "exact_accuracy": 0.9914555996338114,
    "examples_per_second": 327.6371852750861,
    "generated_tokens_per_second": 6225.106520226636,
    "git_commit": "unavailable: not a git repository",
    "ground_truth_all_correct": false,
    "hardware_backend": "cpu",
    "implementation_name": "kv_cache_prefill_unrolled",
    "mean_latency_per_batch": 0.3906760457990458,
    "measured_total_time": 3.9067604579904582,
    "median_latency_per_batch": 0.388554249999288,
    "notes": "KV-cache prefill/decode with the fixed 19-token decode loop unrolled at JIT trace time. This trades larger compile graphs for less loop/control overhead in steady-state inference. Ground truth exact accuracy is below 1.0 on the configured correctness set.",
    "number_correct": 3249,
    "number_of_generated_answer_tokens_per_prompt": 19,
    "number_of_measured_iterations": 10,
    "number_of_prompts": 10000,
    "number_of_warmup_iterations": 3,
    "number_total": 3277,
    "p90_latency_per_batch": 0.3974122911007726,
    "p95_latency_per_batch": 0.3998113540503255,
    "p99_latency_per_batch": 0.4017306044099678,
    "paper_ideas": [
      {
        "implementation": "Use Python-level unrolling inside JIT for the known answer length.",
        "paper": "Static graph specialization for fixed-shape autoregressive decoding.",
        "url": "https://openxla.org/xla"
      }
    ],
    "params_path": "/Users/thibaultcastells/Documents/CODE/transformer_tuto/model_optimization/models/arithmetic_transformer_params.msgpack",
    "peak_memory_mb": 3236.298752,
    "per_example_latency": 0.0030521566078050453,
    "reference_match_accuracy": 1.0,
    "reference_matches": 3277,
    "reference_total": 3277,
    "run_name": "final_cumulative_stack_cpu",
    "timestamp": "2026-05-26T10:13:49.957453+00:00",
    "total_generated_tokens": 24320,
    "valid_for_speed_comparison": false,
    "warmup_time": 1.1722084590001032
  },
  "correctness_examples": 3277,
  "dtype": "float32",
  "excluded_from_final_stack": [
    "bfloat16: not used for final no-loss stack because earlier runs showed dtype-dependent output changes",
    "dot_product_attention rewrite: float32 was effectively tied and bfloat16 changed generated text",
    "static closed-over params: mixed result",
    "int quantization: not combined with KV-cache and did not beat the cumulative stack"
  ],
  "final_stack_methods": [
    "JAX JIT compiled fixed-shape generation graph",
    "manual KV cache for autoregressive decode",
    "parallel prompt prefill",
    "fixed 19-token decode loop unrolled inside JIT",
    "default XLA fusion enabled",
    "batching up to 128 prompts"
  ],
  "final_stack_variant": "kv_cache_prefill_unrolled",
  "max_same_batch_speedup": {
    "batch_latency_ratio_eager_over_final": 16.23340934230335,
    "batch_size": 1,
    "eager_examples_per_second": 1.6511059899713443,
    "eager_mean_latency_per_batch": 0.6056546376028564,
    "eager_per_example_latency": 0.6056546376028564,
    "eager_tokens_per_second": 31.371013809455544,
    "exact_accuracy": 0.9914555996338114,
    "examples_per_second_speedup": 16.23340934230335,
    "final_compile_time": 7.960653791000368,
    "final_examples_per_second": 26.80307940273384,
    "final_mean_latency_per_batch": 0.03730914589978056,
    "final_per_example_latency": 0.03730914589978056,
    "final_tokens_per_second": 509.2585086519429,
    "per_example_latency_ratio_eager_over_final": 16.23340934230335,
    "reference_match_accuracy": 1.0,
    "reference_matches": 3277,
    "reference_total": 3277,
    "tokens_per_second_speedup": 16.233409342303347
  },
  "measure_iters": 10,
  "prompt_count": 10000,
  "run_dir": "/Users/thibaultcastells/Documents/CODE/transformer_tuto/model_optimization/runs/inference_benchmarks/final_cumulative_stack_cpu",
  "run_name": "final_cumulative_stack_cpu",
  "same_batch_comparisons": [
    {
      "batch_latency_ratio_eager_over_final": 16.23340934230335,
      "batch_size": 1,
      "eager_examples_per_second": 1.6511059899713443,
      "eager_mean_latency_per_batch": 0.6056546376028564,
      "eager_per_example_latency": 0.6056546376028564,
      "eager_tokens_per_second": 31.371013809455544,
      "exact_accuracy": 0.9914555996338114,
      "examples_per_second_speedup": 16.23340934230335,
      "final_compile_time": 7.960653791000368,
      "final_examples_per_second": 26.80307940273384,
      "final_mean_latency_per_batch": 0.03730914589978056,
      "final_per_example_latency": 0.03730914589978056,
      "final_tokens_per_second": 509.2585086519429,
      "per_example_latency_ratio_eager_over_final": 16.23340934230335,
      "reference_match_accuracy": 1.0,
      "reference_matches": 3277,
      "reference_total": 3277,
      "tokens_per_second_speedup": 16.233409342303347
    },
    {
      "batch_latency_ratio_eager_over_final": 10.135498747458733,
      "batch_size": 8,
      "eager_examples_per_second": 11.695782891777773,
      "eager_mean_latency_per_batch": 0.6840072249993682,
      "eager_per_example_latency": 0.08550090312492102,
      "eager_tokens_per_second": 222.21987494377768,
      "exact_accuracy": 0.9914555996338114,
      "examples_per_second_speedup": 10.135498747458733,
      "final_compile_time": 8.095088583999313,
      "final_examples_per_second": 118.5425928501629,
      "final_mean_latency_per_batch": 0.06748629170033382,
      "final_per_example_latency": 0.008435786462541728,
      "final_tokens_per_second": 2252.3092641530952,
      "per_example_latency_ratio_eager_over_final": 10.135498747458733,
      "reference_match_accuracy": 1.0,
      "reference_matches": 3277,
      "reference_total": 3277,
      "tokens_per_second_speedup": 10.135498747458733
    },
    {
      "batch_latency_ratio_eager_over_final": 6.853917382842257,
      "batch_size": 32,
      "eager_examples_per_second": 31.30682118948165,
      "eager_mean_latency_per_batch": 1.0221414626008483,
      "eager_per_example_latency": 0.03194192070627651,
      "eager_tokens_per_second": 594.8296026001514,
      "exact_accuracy": 0.9914555996338114,
      "examples_per_second_speedup": 6.853917382842257,
      "final_compile_time": 8.072002625005553,
      "final_examples_per_second": 214.57436595212258,
      "final_mean_latency_per_batch": 0.14913244579802268,
      "final_per_example_latency": 0.004660388931188209,
      "final_tokens_per_second": 4076.9129530903288,
      "per_example_latency_ratio_eager_over_final": 6.853917382842257,
      "reference_match_accuracy": 1.0,
      "reference_matches": 3277,
      "reference_total": 3277,
      "tokens_per_second_speedup": 6.853917382842257
    },
    {
      "batch_latency_ratio_eager_over_final": 9.965288270075453,
      "batch_size": 128,
      "eager_examples_per_second": 32.877843208905524,
      "eager_mean_latency_per_batch": 3.893199416600692,
      "eager_per_example_latency": 0.030415620442192905,
      "eager_tokens_per_second": 624.679020969205,
      "exact_accuracy": 0.9914555996338114,
      "examples_per_second_speedup": 9.965288270075453,
      "final_compile_time": 8.492741916998057,
      "final_examples_per_second": 327.6371852750861,
      "final_mean_latency_per_batch": 0.3906760457990458,
      "final_per_example_latency": 0.0030521566078050453,
      "final_tokens_per_second": 6225.106520226636,
      "per_example_latency_ratio_eager_over_final": 9.965288270075453,
      "reference_match_accuracy": 1.0,
      "reference_matches": 3277,
      "reference_total": 3277,
      "tokens_per_second_speedup": 9.965288270075453
    }
  ],
  "warmup_iters": 3
}