Add DriftJudge AI role with verdict parsing

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-05-31 16:45:59 -04:00
parent 7e9d00b80b
commit 83bae00e45
3 changed files with 144 additions and 0 deletions
+30
View File
@@ -42,3 +42,33 @@ Rules:
User intent: ` + intent
}
// DriftJudge decides whether the current window is on-task for a commitment.
// It takes primitives, not domain/evidence types, so ai stays a leaf package.
type DriftJudge interface {
JudgeDrift(ctx context.Context, commitment, windowClass, windowTitle string) (Verdict, error)
}
// JudgeDrift makes Service satisfy DriftJudge over the same backend as Coach.
func (s *Service) JudgeDrift(ctx context.Context, commitment, windowClass, windowTitle string) (Verdict, error) {
out, err := s.backend.Run(ctx, buildDriftPrompt(commitment, windowClass, windowTitle))
if err != nil {
return Verdict{}, err
}
return parseVerdict(out)
}
func buildDriftPrompt(commitment, class, title string) string {
return `You are a focus monitor. The user committed to a task. Decide whether their CURRENT window is on-task or a distraction.
Respond with ONLY a JSON object, no prose and no code fences, exactly this shape:
{"on_task": <true or false>, "reason": "<short explanation, one sentence>"}
Rules:
- on_task: true if the window plausibly serves the commitment, false if it is a distraction.
- reason: one short sentence. REQUIRED when on_task is false.
Commitment: ` + commitment + `
Current window class: ` + class + `
Current window title: ` + title
}
+47
View File
@@ -0,0 +1,47 @@
package ai
import (
"encoding/json"
"errors"
"fmt"
"strings"
)
// Verdict is the drift judge's call on a single window.
type Verdict struct {
OnTask bool
Reason string
}
// ErrInvalidVerdict marks a parseable-but-unusable judge response.
var ErrInvalidVerdict = errors.New("ai: invalid verdict")
type rawVerdict struct {
OnTask bool `json:"on_task"`
Reason string `json:"reason"`
}
// parseVerdict extracts and validates a Verdict from raw CLI output. It reuses
// extractJSON and the shared empty/no-JSON sentinels. A drifting verdict with no
// reason is unusable for the UI and is rejected as ErrInvalidVerdict.
func parseVerdict(s string) (Verdict, error) {
if strings.TrimSpace(s) == "" {
return Verdict{}, ErrEmptyResponse
}
if strings.IndexByte(s, '{') < 0 {
return Verdict{}, ErrNoJSON
}
jsonStr, err := extractJSON(s)
if err != nil {
return Verdict{}, fmt.Errorf("%w: %v", ErrInvalidVerdict, err)
}
var raw rawVerdict
if err := json.Unmarshal([]byte(jsonStr), &raw); err != nil {
return Verdict{}, fmt.Errorf("%w: %v", ErrInvalidVerdict, err)
}
v := Verdict{OnTask: raw.OnTask, Reason: strings.TrimSpace(raw.Reason)}
if !v.OnTask && v.Reason == "" {
return Verdict{}, ErrInvalidVerdict
}
return v, nil
}
+67
View File
@@ -0,0 +1,67 @@
package ai
import (
"context"
"errors"
"strings"
"testing"
)
func TestParseVerdictOnTask(t *testing.T) {
v, err := parseVerdict(`{"on_task": true, "reason": ""}`)
if err != nil {
t.Fatalf("parse: %v", err)
}
if !v.OnTask {
t.Fatalf("expected on-task, got %+v", v)
}
}
func TestParseVerdictDriftingChatty(t *testing.T) {
v, err := parseVerdict("Sure! here is the call:\n{\"on_task\": false, \"reason\": \"Reddit is unrelated\"}\nhope that helps")
if err != nil {
t.Fatalf("parse: %v", err)
}
if v.OnTask || v.Reason != "Reddit is unrelated" {
t.Fatalf("bad verdict: %+v", v)
}
}
func TestParseVerdictDriftingNeedsReason(t *testing.T) {
if _, err := parseVerdict(`{"on_task": false}`); !errors.Is(err, ErrInvalidVerdict) {
t.Fatalf("want ErrInvalidVerdict for reasonless drift, got %v", err)
}
}
func TestParseVerdictEmpty(t *testing.T) {
if _, err := parseVerdict(" "); !errors.Is(err, ErrEmptyResponse) {
t.Fatalf("want ErrEmptyResponse, got %v", err)
}
}
func TestParseVerdictNoJSON(t *testing.T) {
if _, err := parseVerdict("I cannot help"); !errors.Is(err, ErrNoJSON) {
t.Fatalf("want ErrNoJSON, got %v", err)
}
}
func TestServiceJudgeDrift(t *testing.T) {
fb := &fakeBackend{out: `{"on_task": false, "reason": "YouTube is off-task"}`}
v, err := NewService(fb).JudgeDrift(context.Background(), "write the report", "firefox", "YouTube")
if err != nil {
t.Fatalf("judge: %v", err)
}
if v.OnTask || v.Reason == "" {
t.Fatalf("bad verdict: %+v", v)
}
if !strings.Contains(fb.gotPrompt, "write the report") || !strings.Contains(fb.gotPrompt, "YouTube") {
t.Fatalf("prompt should embed commitment and window: %s", fb.gotPrompt)
}
}
func TestServiceJudgeDriftBackendError(t *testing.T) {
fb := &fakeBackend{err: errors.New("boom")}
if _, err := NewService(fb).JudgeDrift(context.Background(), "x", "c", "t"); err == nil {
t.Fatal("want backend error")
}
}