evaluation

type AttrValue struct {
	Type  string      `json:"Type"`
	Value interface{} `json:"Value"`
}

type CheckAgentVersionBuilder struct {
	ES *EvalState
}

type CollectMetricsBuilder struct {
	ES *EvalState
}

type CollectTraceTokensBuilder struct {
	ES *EvalState
}

type Convergence string

type CreatePointDirBuilder struct {
	ES *EvalState
}

type DiscoverSuiteSamplesBuilder struct {
	ES *EvalSessionState
}

type DumpConfigBuilder struct {
	ES *EvalState
}

type EvalFactoryDeps struct {
	Ctx              context.Context
	Registry         *core.Registry
	Stderr           io.Writer
	SuitePath        string
	OutputDir        string
	OllamaURL        string
	ChildAgentBinary string
	CoreRoot         string
	Directory        string
}

type EvalMeta struct {
	Harness      string                 `json:"harness"`
	Model        string                 `json:"model"`
	Sample       string                 `json:"sample"`
	GridParams   map[string]interface{} `json:"grid_params"`
	Repetition   int                    `json:"repetition"`
	ExitCode     int                    `json:"exit_code"`
	Duration     time.Duration          `json:"duration_ns"`
	TestsPassed  bool                   `json:"tests_passed"`
	TestOutput   string                 `json:"test_output"`
	TimedOut     bool                   `json:"timed_out"`
	FailureStage string                 `json:"failure_stage,omitempty"`
	FailureCause string                 `json:"failure_cause,omitempty"`
}

type EvalRunResult struct {
	Sample      string
	Model       string
	Repetition  int
	TestsPassed bool
	ExitCode    int
	TimedOut    bool
	Iterations  int
	TokensIn    int
	TokensOut   int
	Duration    time.Duration
	Progression *RunProgression
}

type EvalSessionState struct {
	EvalState

	// Configured from CLI flags / tool YAML config
	SuitePath string
	OutputDir string
	Reps      int
	Timeout   time.Duration
	OllamaURL string
	// ChildAgentBinary overrides the harness binary the evaluator launches for
	// each suite profile. Empty means the default "agent" (resolved from PATH).
	ChildAgentBinary string
	CoreRoot         string

	Suite        SuiteConfig
	SessionDir   string
	PointMachine string
	Result       SessionResult
	Stderr       io.Writer
	// contains filtered or unexported fields
}

type EvalState struct {
	PC  *PointContext
	Ctx context.Context
}

type EvaluationArtifactBuilder struct {
	Name      string
	Operation string
	DataDir   string
}

type EvaluationModelStat struct {
	Model         string  `json:"model"`
	Runs          int     `json:"runs"`
	Successes     int     `json:"successes"`
	SuccessRate   float64 `json:"successRate"`
	CleanRate     float64 `json:"cleanRate"`
	RecoveryRate  float64 `json:"recoveryRate"`
	StuckRate     float64 `json:"stuckRate"`
	MeanIter      float64 `json:"meanIter"`
	MeanTokensIn  float64 `json:"meanTokensIn"`
	MeanTokensOut float64 `json:"meanTokensOut"`
	MeanDurationS float64 `json:"meanDurationS"`
}

type EvaluationPoint struct {
	PointID     string  `json:"pointId"`
	Sample      string  `json:"sample"`
	Model       string  `json:"model"`
	TestsPassed bool    `json:"testsPassed"`
	TimedOut    bool    `json:"timedOut"`
	ExitCode    int     `json:"exitCode"`
	DurationS   float64 `json:"durationS"`
	Iterations  int     `json:"iterations"`
	TokensIn    int     `json:"tokensIn"`
	TokensOut   int     `json:"tokensOut"`
	Convergence string  `json:"convergence"`
}

type EvaluationSampleStat struct {
	Sample        string  `json:"sample"`
	Model         string  `json:"model"`
	Runs          int     `json:"runs"`
	SuccessRate   float64 `json:"successRate"`
	MeanIter      float64 `json:"meanIter"`
	MeanTokens    float64 `json:"meanTokens"`
	MeanDurationS float64 `json:"meanDurationS"`
}

type EvaluationSessionDetail struct {
	ID            string                 `json:"id"`
	ModelStats    []EvaluationModelStat  `json:"modelStats"`
	SampleStats   []EvaluationSampleStat `json:"sampleStats"`
	TotalPoints   int                    `json:"totalPoints"`
	TotalPassed   int                    `json:"totalPassed"`
	TotalFailed   int                    `json:"totalFailed"`
	TotalTimedOut int                    `json:"totalTimedOut"`
}

type EvaluationSessionSummary struct {
	ID           string `json:"id"`
	Name         string `json:"name"`
	Timestamp    string `json:"timestamp"`
	PointCount   int    `json:"pointCount"`
	PassCount    int    `json:"passCount"`
	FailCount    int    `json:"failCount"`
	TimeoutCount int    `json:"timeoutCount"`
}

type EvaluationToolSnapshot struct {
	Tool      string `json:"tool"`
	Signal    string `json:"signal"`
	Iteration int    `json:"iteration"`
	Output    string `json:"output,omitempty"`
}

type EvaluationTrace struct {
	PointID   string                   `json:"pointId"`
	Spans     []EvaluationTraceSpan    `json:"spans"`
	Snapshots []EvaluationToolSnapshot `json:"snapshots"`
}

type EvaluationTraceSpan struct {
	Name       string  `json:"name"`
	StartTime  string  `json:"startTime"`
	EndTime    string  `json:"endTime"`
	DurationMs float64 `json:"durationMs"`
	ToolName   string  `json:"toolName,omitempty"`
	Signal     string  `json:"signal,omitempty"`
	TokensIn   int     `json:"tokensIn,omitempty"`
	TokensOut  int     `json:"tokensOut,omitempty"`
}

type Event struct {
	Name       string     `json:"Name"`
	Attributes []KeyValue `json:"Attributes"`
	Time       time.Time  `json:"Time"`
}

type ExpandEvalGridBuilder struct {
	ES *EvalSessionState
}

type GridPoint map[string]interface{}

type GroupKey struct {
	Sample string
	Model  string
}

type Harness struct {
	Name    string
	Binary  string
	Version string
}

type InitEvalSessionBuilder struct {
	ES *EvalSessionState
}

type KeyValue struct {
	Key   string    `json:"Key"`
	Value AttrValue `json:"Value"`
}

type MaterializeEvalPointsBuilder struct {
	ES *EvalSessionState
}

type ModelStats struct {
	Model         string
	Runs          int
	Successes     int
	SuccessRate   float64
	CleanRate     float64
	RecoveryRate  float64
	StuckRate     float64
	MeanIter      float64
	MeanTokensIn  float64
	MeanTokensOut float64
	MeanDuration  time.Duration
}

type ParseSuiteConfigBuilder struct {
	ES *EvalSessionState
}

type PointContext struct {
	SessionDir  string
	PointID     string
	Sample      Sample
	Harness     Harness
	Model       string
	ProfilePath string
	CoreRoot    string
	GridPoint   GridPoint
	Rep         int
	Timeout     time.Duration
	LLMTimeout  time.Duration
	OllamaURL   string
	Stderr      io.Writer

	// Populated during execution
	PointDir        string
	TracePath       string
	ResultPath      string
	AgentCommit     string
	Tokens          int
	TestsPassed     bool
	TestOutput      string
	TraceVersion    string
	VersionMismatch bool
	TimedOut        bool
	ExitCode        int
	Duration        time.Duration
	FailureStage    string
	FailureCause    string
}

type PointResult struct {
	PointID     string
	Sample      string
	Harness     string
	Model       string
	TestsPassed bool
	TimedOut    bool
	ExitCode    int
	Tokens      int
	Duration    time.Duration
}

type RecordAgentCommitBuilder struct {
	ES *EvalState
}

type RecordOracleResultBuilder struct {
	ES *EvalState
}

type RecordPointFailureBuilder struct {
	ES *EvalState
}

type ReportSessionBuilder struct {
	ES *EvalSessionState
}

type ReportSuiteSummaryBuilder struct {
	ES *EvalSessionState
}

type RunAgentBuilder struct {
	ES *EvalState
}

type RunPointBuilder struct {
	ES            *EvalSessionState
	PointRegistry *core.Registry
	Config        catalog.RunPointConfig
}

type RunProgression struct {
	Tools   []ToolProgression `json:"tools"`
	Overall Convergence       `json:"overall"`
	Summary string            `json:"summary"`
}

type Sample struct {
	Name         string
	PromptPath   string
	DocDir       string
	WorkspaceDir string
}

type SampleDocsBuilder struct {
	ES *EvalState
}

type SampleModelRow struct {
	Sample       string
	Model        string
	Runs         int
	SuccessRate  float64
	MeanIter     float64
	MeanTokens   float64
	MeanDuration time.Duration
	Convergences map[Convergence]int
}

type SessionResult struct {
	TotalPoints int
	Passed      int
	Failed      int
	TimedOut    int
	Duration    time.Duration
	Points      []PointResult
}

type Span struct {
	Name        string     `json:"Name"`
	SpanContext SpanCtx    `json:"SpanContext"`
	Parent      SpanCtx    `json:"Parent"`
	StartTime   time.Time  `json:"StartTime"`
	EndTime     time.Time  `json:"EndTime"`
	Attributes  []KeyValue `json:"Attributes"`
	Events      []Event    `json:"Events"`
}

type SpanCtx struct {
	SpanID string `json:"SpanID"`
}

type SuiteConfig struct {
	Name       string           `yaml:"name"`
	Profiles   []SuiteProfile   `yaml:"-"`
	Grid       map[string][]any `yaml:"grid,omitempty"`
	SamplesDir string           `yaml:"-"`
	Samples    []Sample         `yaml:"-"`
	Timeout    time.Duration    `yaml:"-"`
	OllamaURL  string           `yaml:"-"`
	Reps       int              `yaml:"-"`
}

type SuiteProfile struct {
	Path    string               `yaml:"path"`
	Name    string               `yaml:"-"`
	Model   string               `yaml:"-"`
	Binary  string               `yaml:"-"`
	Profile catalog.AgentProfile `yaml:"-"`
}

type SummarizePointResultsBuilder struct {
	ES *EvalState
}

type ToolProgression struct {
	Tool        string         `json:"tool"`
	Snapshots   []ToolSnapshot `json:"snapshots"`
	Convergence Convergence    `json:"convergence"`
	Timeline    string         `json:"timeline"`
}

type ToolSnapshot struct {
	Tool   string `json:"tool"`
	Signal string `json:"signal"`
	Total  int    `json:"total"`
	Passed int    `json:"passed"`
	Failed int    `json:"failed"`
}