feat(eval): compare a run against a saved baseline and fail on regression

cmd/root/eval.go

@@ -1,6 +1,7 @@
 package root
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -25,6 +26,13 @@ type evalFlags struct {
 
 	runConfig config.RuntimeConfig
 	outputDir string
+
+	// baseline is a previously saved run (an -eval.json written by a prior
+	// invocation) to compare this run against; empty disables the check.
+	baseline string
+	// regressionTolerance is how far an aggregate quality rate may fall before
+	// the comparison fails. See evaluation.Compare for the exact semantics.
+	regressionTolerance float64
 }
 
 func newEvalCmd() *cobra.Command {
@@ -48,6 +56,8 @@ func newEvalCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&flags.KeepContainers, "keep-containers", false, "Keep containers after evaluation (don't use --rm)")
 	cmd.Flags().StringSliceVarP(&flags.EnvVars, "env", "e", nil, "Environment variables to pass to container (KEY or KEY=VALUE)")
 	cmd.Flags().IntVar(&flags.Repeat, "repeat", 1, "Number of times to repeat each evaluation (useful for computing baselines)")
+	cmd.Flags().StringVar(&flags.baseline, "baseline", "", "Compare against a previously saved run JSON and exit non-zero on regression")
+	cmd.Flags().Float64Var(&flags.regressionTolerance, "regression-tolerance", 0, "How far an aggregate quality rate may fall before --baseline reports a regression (0-1)")
 
 	return cmd
 }
@@ -149,5 +159,33 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr
 
 	fmt.Fprintf(teeOut, "Log: %s\n", logPath)
 
+	if regressionErr := f.checkBaseline(teeOut, run); regressionErr != nil && evalErr == nil {
+		// When the run itself also errored, that is the more fundamental
+		// problem and keeps the exit code.
+		return regressionErr
+	}
+
 	return evalErr
 }
+
+// checkBaseline compares run against the configured baseline and returns a
+// non-nil error when it regressed, so CI fails on the exit code. A no-op when
+// --baseline was not supplied.
+func (f *evalFlags) checkBaseline(out io.Writer, run *evaluation.EvalRun) error {
+	if f.baseline == "" {
+		return nil
+	}
+
+	baseline, err := evaluation.LoadBaseline(f.baseline)
+	if err != nil {
+		return err
+	}
+
+	comparison := evaluation.Compare(baseline, run, f.regressionTolerance)
+	evaluation.PrintComparison(out, comparison)
+
+	if comparison.Regressed {
+		return errors.New("evaluation regressed against baseline")
+	}
+	return nil
+}

cmd/root/eval_baseline_test.go

@@ -0,0 +1,106 @@
+package root
+
+import (
+	"bytes"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/docker/docker-agent/pkg/evaluation"
+)
+
+func sizeRun(pass bool) *evaluation.EvalRun {
+	r := evaluation.Result{InputPath: "a", SizeExpected: "medium", Size: "medium"}
+	if !pass {
+		r.Size = "short"
+	}
+	return &evaluation.EvalRun{Name: "run", Results: []evaluation.Result{r}}
+}
+
+func TestCheckBaseline_NoBaselineIsANoOp(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{}
+	var buf bytes.Buffer
+	require.NoError(t, f.checkBaseline(&buf, sizeRun(false)))
+	assert.Empty(t, buf.String(), "without --baseline nothing is compared or printed")
+}
+
+func TestCheckBaseline_RegressionReturnsAnError(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	path, err := evaluation.SaveRunJSON(sizeRun(true), dir)
+	require.NoError(t, err)
+
+	f := &evalFlags{baseline: path}
+	var buf bytes.Buffer
+	err = f.checkBaseline(&buf, sizeRun(false))
+
+	require.Error(t, err, "a regression must surface as a non-zero exit")
+	assert.Contains(t, err.Error(), "regressed against baseline")
+	assert.Contains(t, buf.String(), "Regression against baseline")
+}
+
+func TestCheckBaseline_NoRegressionSucceeds(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	path, err := evaluation.SaveRunJSON(sizeRun(true), dir)
+	require.NoError(t, err)
+
+	f := &evalFlags{baseline: path}
+	var buf bytes.Buffer
+	require.NoError(t, f.checkBaseline(&buf, sizeRun(true)))
+	assert.Contains(t, buf.String(), "No regression against baseline")
+}
+
+func TestCheckBaseline_ToleranceIsPlumbedThrough(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	base := &evaluation.EvalRun{Name: "b", Results: []evaluation.Result{
+		{InputPath: "a", ToolCallsExpected: 0.8, ToolCallsScore: 1.0},
+		{InputPath: "b", ToolCallsExpected: 0.8, ToolCallsScore: 1.0},
+	}}
+	cur := &evaluation.EvalRun{Name: "c", Results: []evaluation.Result{
+		{InputPath: "a", ToolCallsExpected: 0.8, ToolCallsScore: 1.0},
+		{InputPath: "b", ToolCallsExpected: 0.8, ToolCallsScore: 0.9},
+	}}
+
+	path, err := evaluation.SaveRunJSON(base, dir)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	strict := &evalFlags{baseline: path, regressionTolerance: 0.01}
+	require.Error(t, strict.checkBaseline(&buf, cur), "a tight tolerance gates the 0.05 drop")
+
+	buf.Reset()
+	lenient := &evalFlags{baseline: path, regressionTolerance: 0.10}
+	require.NoError(t, lenient.checkBaseline(&buf, cur), "a wider tolerance absorbs it")
+}
+
+func TestCheckBaseline_MissingBaselineFileIsAnError(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{baseline: filepath.Join(t.TempDir(), "nope.json")}
+	var buf bytes.Buffer
+	err := f.checkBaseline(&buf, sizeRun(true))
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "reading baseline",
+		"a bad --baseline path must fail loudly rather than silently skipping the gate")
+}
+
+func TestEvalCmd_BaselineFlagsAreRegistered(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NotNil(t, cmd.Flags().Lookup("baseline"))
+
+	tolerance := cmd.Flags().Lookup("regression-tolerance")
+	require.NotNil(t, tolerance)
+	assert.Equal(t, "0", tolerance.DefValue, "the default gates any drop")
+}