Production Agent Evals: Catch Score Drift, Ship Confidently

npm install @chanl/sdk

import Chanl from '@chanl/sdk';
 
const chanl = new Chanl({
  apiKey: process.env.CHANL_API_KEY,
  baseUrl: 'https://platform.chanl.ai',
});

// create-scorecard.ts
const scorecard = await chanl.scorecards.create({
  name: 'Production Quality - Customer Support',
  description: 'Grades live customer support conversations across five dimensions',
  criteria: [
    {
      name: 'Accuracy',
      description: 'Information provided is factually correct and reflects current policies',
      weight: 25,
      scoringGuide: `
        5: All facts correct, cites current policy, no outdated information
        4: All facts correct, minor omission of relevant detail
        3: Mostly correct, one factual imprecision that doesn't affect outcome
        2: Contains one incorrect fact that could mislead the customer
        1: Multiple incorrect facts or references deprecated policy
      `,
    },
    {
      name: 'Task Resolution',
      description: 'The customer issue was fully addressed without requiring follow-up',
      weight: 25,
      scoringGuide: `
        5: Issue fully resolved, customer confirmed satisfaction
        4: Issue resolved, minor loose end acknowledged
        3: Partial resolution, customer needs one follow-up action
        2: Issue identified but not resolved, escalation needed
        1: Issue misidentified or not addressed
      `,
    },
    {
      name: 'Policy Adherence',
      description: 'Agent follows current company policies and does not make unauthorized commitments',
      weight: 20,
      scoringGuide: `
        5: All statements align with current policy, no unauthorized promises
        4: Policy adherent, slightly imprecise language on one policy point
        3: Mostly adherent, one statement could be misinterpreted as a commitment
        2: Makes one commitment outside policy scope
        1: Contradicts active policy or quotes deprecated terms
      `,
    },
    {
      name: 'Tone Appropriateness',
      description: 'Communication style matches the emotional context of the conversation',
      weight: 15,
      scoringGuide: `
        5: Tone perfectly matches context, empathetic when needed, professional throughout
        4: Appropriate tone with one minor mismatch in emotional context
        3: Generally appropriate, slightly mechanical in emotional moments
        2: Noticeably tone-deaf in one exchange
        1: Inappropriate tone that could upset the customer
      `,
    },
    {
      name: 'Efficiency',
      description: 'Conversation reaches resolution without unnecessary back-and-forth',
      weight: 15,
      scoringGuide: `
        5: Resolves in minimum turns, asks all needed questions upfront
        4: Efficient with one unnecessary clarifying question
        3: Adequate pacing, one redundant exchange
        2: Notable inefficiency, customer repeats information
        1: Excessive back-and-forth, customer visibly frustrated by repetition
      `,
    },
  ],
});
 
console.log(`Scorecard created: ${scorecard.id}`);

// evaluate-conversation.ts
async function evaluateConversation(callId: string, scorecardId: string) {
  const result = await chanl.scorecards.evaluate({
    callId,
    scorecardId,
  });
 
  return {
    callId,
    overallScore: result.overallScore,
    criteriaScores: result.criteria.map((c) => ({
      name: c.name,
      score: c.score,
      reasoning: c.reasoning,
    })),
    evaluatedAt: new Date().toISOString(),
  };
}

// sample-and-evaluate.ts
async function runProductionEvalBatch(
  scorecardId: string,
  sampleRate: number = 0.05,
) {
  // Pull recent conversations
  const recentCalls = await chanl.calls.list({
    startDate: oneDayAgo(),
    endDate: now(),
    limit: 1000,
  });
 
  // Random sample
  const sampled = recentCalls.items
    .filter(() => Math.random() < sampleRate)
    .slice(0, 50); // Cap at 50 per batch to manage cost
 
  const results = [];
 
  for (const call of sampled) {
    const result = await evaluateConversation(call.id, scorecardId);
    results.push(result);
 
    // Rate limit: avoid hammering the eval endpoint
    await sleep(500);
  }
 
  return {
    batchSize: results.length,
    averageScore: mean(results.map((r) => r.overallScore)),
    criteriaAverages: aggregateByCriteria(results),
    lowScorers: results.filter((r) => r.overallScore < 3.0),
    evaluatedAt: new Date().toISOString(),
  };
}

// analyze-results.ts
const results = await chanl.scorecards.listResults({
  agentId: 'agent_abc123',
  dateRange: {
    start: thirtyDaysAgo(),
    end: now(),
  },
});
 
// Group by week for trend analysis
const weeklyScores = groupByWeek(results.items);
 
for (const [week, scores] of Object.entries(weeklyScores)) {
  const avg = mean(scores.map((s) => s.overallScore));
  const policyAvg = mean(
    scores.map((s) => s.criteria.find((c) => c.name === 'Policy Adherence')?.score ?? 0),
  );
  console.log(`Week ${week}: Overall ${avg.toFixed(2)}, Policy ${policyAvg.toFixed(2)}`);
}

// create-personas.ts
 
// Persona 1: Tests policy boundary awareness
const policyProber = await chanl.personas.create({
  name: 'Policy Boundary Prober',
  traits: {
    personality: 'Polite but persistent. Asks follow-up questions that push toward policy edges.',
    goal: 'Get a refund on a final-sale item by finding exceptions or loopholes.',
    behavior: 'Starts reasonable, then escalates requests incrementally. Cites things friends told them. Asks "but what if..." questions.',
    style: 'Conversational, uses casual language, occasionally misspells words.',
  },
});
 
// Persona 2: Tests context retention over long conversations
const contextStresser = await chanl.personas.create({
  name: 'Context Window Stresser',
  traits: {
    personality: 'Friendly but disorganized. Provides information across many turns.',
    goal: 'Resolve a billing issue that requires remembering details from early in the conversation.',
    behavior: 'Mentions account number on turn 2, problem on turn 5, relevant detail on turn 8. Expects agent to connect all three without repeating.',
    style: 'Verbose, includes personal anecdotes between relevant details.',
  },
});
 
// Persona 3: Tests topic-switch handling
const topicSwitcher = await chanl.personas.create({
  name: 'Mid-Conversation Switcher',
  traits: {
    personality: 'Busy, slightly impatient. Has multiple issues and jumps between them.',
    goal: 'Resolve both a shipping question and a billing dispute in one conversation.',
    behavior: 'Starts with shipping, switches to billing mid-conversation, then asks a follow-up about the original shipping question.',
    style: 'Short messages, expects quick answers, occasionally references "what you said earlier."',
  },
});
 
console.log('Personas created:', [policyProber.id, contextStresser.id, topicSwitcher.id]);

// run-scenarios.ts
async function runScenarioSuite(agentId: string) {
  // Run all scenarios for this agent
  const execution = await chanl.scenarios.runAll({ agentId });
 
  console.log(`Suite started: ${execution.id}`);
  console.log(`Scenarios: ${execution.totalScenarios}`);
 
  // Poll for completion
  let status = execution;
  while (status.status === 'running') {
    await sleep(5000);
    status = await chanl.scenarios.getExecution(execution.id);
    console.log(`Progress: ${status.completed}/${status.totalScenarios}`);
  }
 
  return status;
}

// check-scenario.ts
async function checkScenarioResult(executionId: string) {
  const execution = await chanl.scenarios.getExecution(executionId);
 
  for (const result of execution.results) {
    const scenario = await chanl.scenarios.get({ scenarioId: result.scenarioId });
 
    console.log(`\n--- ${scenario.name} ---`);
    console.log(`Status: ${result.status}`);
    console.log(`Score: ${result.score}`);
 
    if (result.score < 3.5) {
      console.log(`⚠ Below threshold`);
      console.log(`Turns: ${result.turnCount}`);
      console.log(`Failed criteria:`);
      for (const criterion of result.criteria.filter((c) => c.score < 3)) {
        console.log(`  - ${criterion.name}: ${criterion.score} - ${criterion.reasoning}`);
      }
    }
  }
 
  return {
    passed: execution.results.every((r) => r.score >= 3.5),
    failures: execution.results.filter((r) => r.score < 3.5),
    averageScore: mean(execution.results.map((r) => r.score)),
  };
}

// incident-to-scenario.ts
async function createScenarioFromIncident(
  lowScoredCallId: string,
  agentId: string,
) {
  // Pull the conversation that scored poorly
  const call = await chanl.calls.get({ callId: lowScoredCallId });
 
  // Create a persona that mimics the customer's behavior
  const persona = await chanl.personas.create({
    name: `Regression - ${call.summary?.substring(0, 50)}`,
    traits: {
      personality: 'Mirrors the customer behavior from the flagged conversation',
      goal: call.summary || 'Reproduce the failure scenario',
      behavior: `Follows a conversation pattern similar to call ${lowScoredCallId}`,
      style: 'Natural conversational style',
    },
  });
 
  // Create a scenario that tests this specific failure mode
  const scenario = await chanl.scenarios.create({
    name: `Regression: ${call.failureCategory || 'Quality Drop'} - ${new Date().toISOString().split('T')[0]}`,
    agentId,
    personaId: persona.id,
    expectedOutcome: 'Agent handles this correctly after the fix',
  });
 
  return { persona, scenario };
}

// baseline.ts
async function establishBaseline(agentId: string): Promise<Baseline> {
  // Pull the first two weeks of production scores
  const results = await chanl.calls.getMetrics({
    agentId,
    dateRange: {
      start: fourteenDaysAgo(),
      end: now(),
    },
  });
 
  const scores = results.qualityScores;
 
  return {
    overall: {
      mean: mean(scores.map((s) => s.overall)),
      stdDev: standardDeviation(scores.map((s) => s.overall)),
    },
    byCriterion: Object.fromEntries(
      ['accuracy', 'taskResolution', 'policyAdherence', 'tone', 'efficiency'].map(
        (criterion) => [
          criterion,
          {
            mean: mean(scores.map((s) => s[criterion] ?? 0)),
            stdDev: standardDeviation(scores.map((s) => s[criterion] ?? 0)),
          },
        ],
      ),
    ),
    sampleSize: scores.length,
    period: {
      start: fourteenDaysAgo().toISOString(),
      end: now().toISOString(),
    },
  };
}

// drift-check.ts
async function checkForDrift(
  agentId: string,
  baseline: Baseline,
): Promise<DriftReport> {
  // Pull the last 7 days of scores
  const recent = await chanl.calls.getMetrics({
    agentId,
    dateRange: {
      start: sevenDaysAgo(),
      end: now(),
    },
  });
 
  const recentScores = recent.qualityScores;
  const recentMean = mean(recentScores.map((s) => s.overall));
 
  // Calculate drift magnitude
  const drift = baseline.overall.mean - recentMean;
  const driftInStdDevs = drift / baseline.overall.stdDev;
 
  // Per-criterion analysis
  const criterionDrift = Object.entries(baseline.byCriterion).map(
    ([criterion, baselineStats]) => {
      const recentCriterionMean = mean(
        recentScores.map((s) => s[criterion] ?? 0),
      );
      const criterionDriftVal = baselineStats.mean - recentCriterionMean;
 
      return {
        criterion,
        baselineMean: baselineStats.mean,
        recentMean: recentCriterionMean,
        drift: criterionDriftVal,
        severity: classifyDrift(criterionDriftVal, baselineStats.stdDev),
      };
    },
  );
 
  return {
    agentId,
    baselineMean: baseline.overall.mean,
    recentMean,
    overallDrift: drift,
    driftInStdDevs,
    severity: classifyDrift(drift, baseline.overall.stdDev),
    criterionDrift,
    sampleSize: recentScores.length,
    period: {
      start: sevenDaysAgo().toISOString(),
      end: now().toISOString(),
    },
  };
}
 
function classifyDrift(
  drift: number,
  stdDev: number,
): 'none' | 'warning' | 'critical' {
  const deviations = drift / stdDev;
  if (deviations >= 2) return 'critical';
  if (deviations >= 1) return 'warning';
  return 'none';
}

// drift-monitor.ts
async function runDriftMonitor(agents: string[], baseline: Map<string, Baseline>) {
  const reports: DriftReport[] = [];
 
  for (const agentId of agents) {
    const agentBaseline = baseline.get(agentId);
    if (!agentBaseline) {
      console.warn(`No baseline for agent ${agentId}, skipping`);
      continue;
    }
 
    const report = await checkForDrift(agentId, agentBaseline);
    reports.push(report);
 
    if (report.severity === 'critical') {
      await sendAlert({
        channel: 'pagerduty',
        title: `Critical quality drift: agent ${agentId}`,
        body: formatDriftReport(report),
      });
    } else if (report.severity === 'warning') {
      await sendAlert({
        channel: 'slack',
        title: `Quality drift warning: agent ${agentId}`,
        body: formatDriftReport(report),
      });
    }
 
    // Log per-criterion breakdown for dashboarding
    for (const criterion of report.criterionDrift) {
      if (criterion.severity !== 'none') {
        console.log(
          `  ${criterion.criterion}: ${criterion.baselineMean.toFixed(2)} → ${criterion.recentMean.toFixed(2)} (${criterion.severity})`,
        );
      }
    }
  }
 
  return reports;
}

# .github/workflows/agent-eval.yml
name: Agent Quality Gate
 
on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'agents/**'
      - 'tools/**'
      - 'config/agent-*.yaml'
 
jobs:
  scenario-regression:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
 
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
 
      - name: Install dependencies
        run: npm ci
 
      - name: Run scenario regression suite
        env:
          CHANL_API_KEY: ${{ secrets.CHANL_API_KEY }}
        run: npx ts-node scripts/run-regression.ts
 
      - name: Post results to PR
        if: always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const results = JSON.parse(fs.readFileSync('eval-results.json', 'utf-8'));
 
            const body = `## Agent Eval Results
 
            **Overall**: ${results.passed ? '✅ Passed' : '❌ Failed'}
            **Average Score**: ${results.averageScore.toFixed(2)} / 5.0
            **Scenarios**: ${results.passedCount}/${results.totalCount} passed
 
            ${results.failures.length > 0 ? `### Failures\n${results.failures.map(f =>
              `- **${f.name}**: ${f.score.toFixed(2)} - ${f.failureReason}`
            ).join('\n')}` : ''}
 
            ${results.warnings.length > 0 ? `### Warnings\n${results.warnings.map(w =>
              `- **${w.name}**: ${w.score.toFixed(2)} - ${w.note}`
            ).join('\n')}` : ''}`;
 
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });

// scripts/run-regression.ts
import Chanl from '@chanl/sdk';
import { writeFileSync } from 'fs';
 
const chanl = new Chanl({
  apiKey: process.env.CHANL_API_KEY!,
  baseUrl: 'https://platform.chanl.ai',
});
 
const QUALITY_THRESHOLD = 3.5;
const CRITICAL_THRESHOLD = 3.0;
 
async function main() {
  const agentId = process.env.AGENT_ID || 'agent_production_support';
 
  // Run the full scenario suite
  const execution = await chanl.scenarios.runAll({ agentId });
 
  // Wait for completion
  let status = await chanl.scenarios.getExecution(execution.id);
  while (status.status === 'running') {
    await new Promise((r) => setTimeout(r, 5000));
    status = await chanl.scenarios.getExecution(execution.id);
  }
 
  // Analyze results
  const results = {
    passed: status.results.every((r) => r.score >= QUALITY_THRESHOLD),
    averageScore: mean(status.results.map((r) => r.score)),
    totalCount: status.results.length,
    passedCount: status.results.filter((r) => r.score >= QUALITY_THRESHOLD).length,
    failures: status.results
      .filter((r) => r.score < CRITICAL_THRESHOLD)
      .map((r) => ({
        name: r.scenarioName,
        score: r.score,
        failureReason: r.criteria
          .filter((c) => c.score < 3)
          .map((c) => `${c.name}: ${c.reasoning}`)
          .join('; '),
      })),
    warnings: status.results
      .filter((r) => r.score >= CRITICAL_THRESHOLD && r.score < QUALITY_THRESHOLD)
      .map((r) => ({
        name: r.scenarioName,
        score: r.score,
        note: r.criteria
          .filter((c) => c.score < 4)
          .map((c) => `${c.name}: ${c.score}`)
          .join(', '),
      })),
  };
 
  // Write results for the CI job to read
  writeFileSync('eval-results.json', JSON.stringify(results, null, 2));
 
  // Exit with appropriate code
  if (!results.passed) {
    console.error(`\n❌ Quality gate failed. ${results.failures.length} scenarios below threshold.`);
    process.exit(1);
  }
 
  console.log(`\n✅ All ${results.totalCount} scenarios passed. Average: ${results.averageScore.toFixed(2)}`);
}
 
function mean(values: number[]): number {
  return values.reduce((sum, v) => sum + v, 0) / values.length;
}
 
main().catch((err) => {
  console.error('Regression suite failed:', err);
  process.exit(1);
});