Kaden

0.1.3-beta

Payload/scripts/harness/compare-model-routing-eval.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Compare two model-routing eval-run files against catalog thresholds.

.DESCRIPTION
    Windows-first entry for scripts/harness/compare-model-routing-eval.mjs.
    Does not call an LLM. Fill baseline and candidate runs from host sessions,
    then compare. Use -Init to emit a blank eval-run with every catalog case.

.EXAMPLE
    .\scripts\harness\compare-model-routing-eval.ps1 -Init -Out tests\harness\evals\model-routing\eval-run.mine.json

.EXAMPLE
    .\scripts\harness\compare-model-routing-eval.ps1 `
        -Baseline tests\harness\evals\model-routing\eval-run.synthetic-baseline.json `
        -Candidate tests\harness\evals\model-routing\eval-run.synthetic-candidate.json
#>
[CmdletBinding()]
param(
    [switch]$Init,

    [string]$Out = '',

    [string]$Baseline = '',

    [string]$Candidate = '',

    [string]$Catalog = '',

    [string]$RepoRoot = ''
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'

$scriptDir = if ($PSScriptRoot) { $PSScriptRoot } else { Split-Path -Parent $MyInvocation.MyCommand.Path }
if ([string]::IsNullOrWhiteSpace($RepoRoot)) {
    $RepoRoot = (Resolve-Path (Join-Path $scriptDir '..\..')).Path
}

$modulePath = Join-Path $PSScriptRoot 'lib\RepoRuntime.psm1'
Import-Module $modulePath -Force
Initialize-RepoShellEnvironment -LogPrefix 'model-routing-eval' | Out-Null

$runner = Join-Path $PSScriptRoot 'compare-model-routing-eval.mjs'
if (-not (Test-Path -LiteralPath $runner)) {
    throw "Runner not found: $runner"
}

$nodeArgs = [System.Collections.Generic.List[string]]::new()
if ($Init.IsPresent) {
    [void]$nodeArgs.Add('--init')
}
if (-not [string]::IsNullOrWhiteSpace($Out)) {
    [void]$nodeArgs.Add('--out')
    [void]$nodeArgs.Add($Out)
}
if (-not [string]::IsNullOrWhiteSpace($Baseline)) {
    [void]$nodeArgs.Add('--baseline')
    [void]$nodeArgs.Add($Baseline)
}
if (-not [string]::IsNullOrWhiteSpace($Candidate)) {
    [void]$nodeArgs.Add('--candidate')
    [void]$nodeArgs.Add($Candidate)
}
if (-not [string]::IsNullOrWhiteSpace($Catalog)) {
    [void]$nodeArgs.Add('--catalog')
    [void]$nodeArgs.Add($Catalog)
}

Push-Location $RepoRoot
try {
    & node $runner @nodeArgs
    exit $LASTEXITCODE
}
finally {
    Pop-Location
}