evaluation "metrics"
title "The numbers every design is measured by"
about "Speed, heat, hardware, size and logical error, each a term over the device, the programme and the replay of one on the other."

# ------------------------------------------------------------------ the device's own laws

# The two-ion gate's error law, read off the device's ms_gate primitive: its error at
# n-bar = 0 in a two-ion trap, its growth per quantum, and Murali's chain-length factor
# N / ln N, clamped so that no chain is better than the reference one.
define eps0 = 1 - primitive("ms_gate", "fidelity_at_n0")
define kappa = slope("ms_gate")
define Nref = chain_reference("ms_gate")
define chain_factor(N) = max(max(N, 2) / ln(max(N, 2)), Nref / ln(Nref))
define chain_ratio(N) = chain_factor(N) / chain_factor(Nref)
define ms_error(n, N) =
  if Nref = none or chain_ratio(N) = 1 then eps0 + kappa * max(n, 0)
  else (eps0 - kappa / 2) + kappa / 2 * chain_ratio(N) * (2 * max(n, 0) + 1)

# the n-bar a two-ion gate meets: the hotter of its two ions, as the gate starts
define gate_nbar(g) = max(nbar(i, start(g)) for i in ions(g))
# how many ions share the gate's trap as it starts
define gate_chain(g) = occupancy(site(g), start(g))

# ------------------------------------------------------------------ speed

metric T "Execution time" unit "ms" better low group "speed"
  about "How long the programme runs on the device's clock: its steps one after another, each as long as its slowest participant."
  = time(final()) / 1000

metric T_jones "Execution time, on QCCDSim's clock" unit "ms" better low group "speed" clock "qccdsim_jones"
  about "The same execution time, priced at the operating points of the qccdsim_jones table: QCCDSim's durations with Jones and Murali's heating. The round-time boards rank by it."
  = time(final()) / 1000

metric T_transport "Execution time, on the transport-excitation clock" unit "ms" better low group "speed" clock "transport_excitation"
  about "The execution time at the transport_excitation table's operating points (Munoz and colleagues): slower, colder transport."
  = time(final()) / 1000

metric T_quantinuum "Execution time, on Goal 1's clock" unit "ms" better low group "speed" clock "quantinuum"
  about "The execution time at the quantinuum table's operating points, the cited Helios-class physics every Goal 1 release fixes."
  = time(final()) / 1000

metric time_transport "Transport time" unit "ms" better low group "speed"
  about "The part of the execution time spent carrying ions."
  = sum(duration(k) for k in steps where kind(k) = "simd") / 1000

metric time_gates "Gate time" unit "ms" better low group "speed"
  about "The part spent on laser gates, the frame changes VZ included."
  = sum(duration(k) for k in steps where kind(k) = "gate") / 1000

metric time_cooling "Cooling time" unit "ms" better low group "speed"
  about "The part spent cooling ions back toward their motional ground state."
  = sum(duration(k) for k in steps where kind(k) = "cool") / 1000

metric time_spam "Readout and reset time" unit "ms" better low group "speed"
  about "The part spent measuring ions and preparing them in |0>."
  = sum(duration(k) for k in steps where kind(k) in {"measure", "reset"}) / 1000

metric instructions "Instructions" unit "count" better low group "speed"
  about "How many instructions the programme has: one machine step each."
  = count(k in steps)

metric two_ion_gates "Two-ion gates" unit "count" better low group "speed"
  about "How many times two ions are brought into contact for an entangling gate: every pair of every MS or two-qubit gate."
  = count(g in two_qubit_gates)

# a transport instruction, as the leaderboard counts them: a rotation of a loop, or a step
# that names the ions it moves
define is_transport(k) = kind(k) = "simd" and (rotation(k) or loop_riders(k) > 0)

metric transport_instructions "Transport instructions" unit "count" better low group "speed"
  about "How many instructions carry ions."
  = count(k in steps where is_transport(k))

metric ions_per_instruction "Ions moved per transport instruction" unit "ions" better high group "speed"
  about "How much of the machine one waveform moves at once: the broadcast a design gets out of its wiring."
  = if transport_instructions = 0 then 0
    else sum(loop_riders(k) for k in steps where is_transport(k)) / transport_instructions

metric cost "Transport cost" unit "model cost units" better low group "speed"
  about "Every rail crossed, every junction entered and every split or merge, counted once each."
  = count(h in hops) + count(h in hops where degree(to(h)) >= 3) + sum(size(entails(o)) for o in moves)

metric depth "Machine steps" unit "steps" better low group "speed"
  about "The steps' depths added up: a transport cycle is as deep as its longest route, one for each rail it crosses and each split or merge it is charged; any other step that takes time is one deep."
  = sum(depth(k) for k in steps)

# ------------------------------------------------------------------ heat

metric peak_nbar "Hottest any ion gets" unit "quanta" better low group "heat"
  about "The most motional energy any ion carries at any moment of the execution, before a cooling takes it away: the hottest ion at the end of each step, and the hottest of those."
  = max(max(nbar(i, end(k)) for i in ions) for k in steps)

metric gate_nbar_max "Hottest a two-ion gate starts" unit "quanta" better low group "heat"
  about "The hottest pair any two-ion gate meets; rule R7 caps it at the device's heat budget."
  = if count(g in two_qubit_gates) = 0 then 0 else max(gate_nbar(g) for g in two_qubit_gates)

metric gate_nbar_mean "Mean heat at a two-ion gate" unit "quanta" better low group "heat"
  about "The hotter ion's n-bar, averaged over every two-ion gate."
  = if count(g in two_qubit_gates) = 0 then 0
    else sum(gate_nbar(g) for g in two_qubit_gates) / count(g in two_qubit_gates)

metric junction_crossings "Junction crossings" unit "count" better low group "heat"
  about "How many times an ion is carried into a node where three or more rails meet."
  = count(h in hops where degree(to(h)) >= 3)

metric transport_quanta "Heat put in by transport" unit "quanta" better low group "heat"
  about "Every quantum the rails, junctions, splits and merges deposited, on every ion, before any cooling took it away."
  = deposited("shuttle") + deposited("junction") + deposited("split_merge")

metric total_quanta "All the heat put in" unit "quanta" better low group "heat"
  about "The same, with the heat the gates deposit and the anomalous heating every ion accrues with the clock."
  = deposited("shuttle") + deposited("junction") + deposited("split_merge") + deposited("gate")
    + deposited("anomalous")

metric n_coolings "Coolings" unit "count" better low group "heat"
  about "How many cooling operations the programme needs."
  = count(o in coolings)

metric gate_error_sum "Summed two-ion gate error" unit "expected errors" better low group "heat"
  about "The device's gate error law summed over every two-ion gate, at the heat and chain length each meets: the heat's whole price."
  = sum(ms_error(gate_nbar(g), gate_chain(g)) for g in two_qubit_gates)

# ------------------------------------------------------------------ hardware

# The least wiring the physics allows, whatever a device declares (the deck's array, p.19-20;
# WISE, arXiv:2305.12773): 24 electrodes a trap, 8 of them compensation shims driven through a
# 1:100 demultiplexer, and 32 channels to move one site (16 electrode roles, a motion bank and
# a hold bank).
param least_electrodes_per_trap = 24
param least_compensation_per_trap = 8
param most_shims_per_dac = 100
param least_channels_per_site = 32

metric n_sites "Sites" unit "count" better low group "hardware"
  about "The places an ion can stand."
  = count(s in sites)

metric n_junctions "Junctions" unit "count" better low group "hardware"
  about "The nodes where three or more rails meet."
  = count(n in junctions)

metric n_rails "Rails" unit "count" better low group "hardware"
  about "The segments ions are carried along."
  = count(r in rails)

metric places "Places for ions" unit "ions" better high group "hardware"
  about "How many ions the device can hold at once: every node's capacity, added up."
  = sum(capacity(n) for n in nodes)

define scheme = wiring("scheme", "direct")
define broadcast_wiring = scheme in {"wise", "broadcast_groups"}
define per_trap = if channel_map() or broadcast_wiring then wiring("electrodes_per_trap", 24)
                  else max(wiring("electrodes_per_trap", 24), least_electrodes_per_trap)
define per_junction = if channel_map() or broadcast_wiring then wiring("electrodes_per_junction", 48)
                      else max(wiring("electrodes_per_junction", 48),
                               budget("junction_electrode_multiplier", 2) * per_trap)
define least_compensation = ceil(n_sites * least_compensation_per_trap / most_shims_per_dac)
define least_driven = min(driven(s) for s in sites where driven(s) > 0)

metric electrodes "Electrodes" unit "count" better low group "hardware"
  about "The DC electrodes that shape the wells: so many a trap, more at a junction, or as the channel map lists them."
  = if channel_map() then map_electrodes() else n_sites * per_trap + n_junctions * per_junction

metric switches "Switches" unit "count" better low group "hardware"
  about "Every DC electrode is wired through a two-way switch: it follows the motion bank or the hold bank."
  = if channel_map() then map_switches() else 2 * (n_sites * per_trap + n_junctions * per_junction)

define broadcast_dacs =
  if channel_map() then
    (if count(s in sites where driven(s) > 0) > 0 and least_driven < least_channels_per_site
     then ceil(shared_channels() * least_channels_per_site / least_driven)
     else shared_channels())
  else if broadcast_wiring then max(wiring("dacs_dynamic", 0), least_channels_per_site)
  else 0

define compensation_dacs =
  if channel_map() then max(compensation_channels(), least_compensation)
  else if broadcast_wiring then
    max(ceil(n_sites * wiring("compensation_electrodes_per_trap", 8) / wiring("shim_per_dac", 1)),
        least_compensation)
  else electrodes

metric dacs "DACs" unit "count" better low group "hardware"
  about "The digital-to-analogue converters that drive the electrodes: one an electrode when wired directly; under broadcast wiring a constant bank for the shared waveforms plus the compensation shims."
  = broadcast_dacs + compensation_dacs

# ------------------------------------------------------------------ size

metric n_ions "Ions" unit "ions" better low group "size"
  about "How many ions the programme places."
  = count(i in ions)

metric n_logical "Logical qubits" unit "logical qubits" better high group "size"
  about "How many logical qubits the experiment protects."
  = logical_qubits()

metric ions_per_logical_qubit "Ions per logical qubit" unit "ions" better low group "size"
  about "The ions each logical qubit costs."
  = n_ions / n_logical

# ------------------------------------------------------------------ logical error

# Wilson's 95 % score interval for k failures in n shots; it stays honest at k = 0.
param z95 = 1.959963984540054
define wilson_centre(k, n) = (k / n + z95 * z95 / (2 * n)) / (1 + z95 * z95 / n)
define wilson_half(k, n) = z95 * sqrt(k / n * (1 - k / n) / n + z95 * z95 / (4 * n * n)) / (1 + z95 * z95 / n)
define wilson_lo(k, n) = if n = 0 then 0 else if k = 0 then 0 else max(0, wilson_centre(k, n) - wilson_half(k, n))
define wilson_hi(k, n) = if n = 0 then 1 else if k = n then 1 else min(1, wilson_centre(k, n) + wilson_half(k, n))
# the per-round rate that compounds to p over the experiment's rounds
define per_round(p) = if rounds() <= 1 then p else if p >= 1 then 1 else 1 - pow(1 - p, 1 / rounds())

metric ler "Logical error rate" unit "per experiment" better low group "logical"
  about "The fraction of shots in which the decoder's correction left some logical observable wrong."
  = failures() / shots()

metric ler_per_round "Logical error rate per round" unit "per round" better low group "logical"
  about "The rate per round of syndrome extraction that compounds to the experiment's."
  = per_round(ler)

metric ler_per_round_per_logical "Logical error rate per round per logical qubit" unit "per round per logical qubit" better low group "logical"
  about "Goal 1's ranking: the per-round rate shared out over the logical qubits the experiment protects."
  = ler_per_round / n_logical

metric ler_per_round_lo "Lower end of its 95 % interval" unit "per round" better low group "logical"
  = per_round(wilson_lo(failures(), shots()))

metric ler_per_round_hi "Upper end of its 95 % interval" unit "per round" better low group "logical"
  = per_round(wilson_hi(failures(), shots()))