SKILL.md- the skill, written out in plain English

AGENTS.md- what the AI loads when this skill comes up

# CLI usage
npx tsx state/lib/prompt-improver.ts <slug>

# Read recent failures for a slug
grep '"skill":"<slug>"' state/log/evals.ndjson | python3 -c "import sys,json; rows=[json.loads(l) for l in sys.stdin]; bad=[r for r in rows if r.get('score',1)<0.7]; print(len(bad), 'failures')"

# Check if prompt-fragment exists
ls state/skills/<slug>/prompt-fragment.md 2>/dev/null || echo "no fragment yet"

echo "[$(date -u +%FT%TZ)] prompt-improver: <what was missing or fixed> [FIXED|LOGGED] action_kind=<kind>" >> state/log/loader-feedback.log

api.ts- the code it can call

#!/usr/bin/env npx tsx
/**
 * snappy-prompt-improver/api.ts -- Generate a prompt patch for a skill based on eval failures.
 *
 * The system extends itself: given a skill slug, read its SKILL.md + AGENTS.md + recent
 * eval failures, then use the configured snappy-os dispatch runtime to compose
 * a small prompt fragment patch.
 * Returns {old, proposed, reason} for the caller to emit a before/after card.
 *
 * Per-skill prompt fragments live at: state/skills/<slug>/prompt-fragment.md
 * (new convention). Server does not auto-load these yet — writing the file is
 * the artifact; Phase 2 will wire server injection.
 *
 * Usage (import):
 *   import { generatePromptPatch } from "./prompt-improver.ts";
 *   const patch = await generatePromptPatch("dogfood-loop");
 *
 * Usage (CLI):
 *   npx tsx state/lib/prompt-improver.ts <slug>
 */

import { readFileSync, existsSync, readdirSync } from "fs";
import { join, resolve, dirname } from "path";
import { fileURLToPath } from "url";
import { dispatchFor, readDefaultModel, readDispatchConfig } from "./dispatch.ts";

const HERE = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = resolve(HERE, "..", "..");

export interface PromptPatch {
  slug: string;
  old: string;
  proposed: string;
  reason: string;
}

interface EvalRow {
  ts: string;
  skill?: string;
  verb?: string;
  score: number;
  primary_issue?: string | null;
  note?: string | null;
}

// --- Read helpers ---

function readSkillMd(slug: string): string {
  const path = join(REPO_ROOT, "state/skills", slug, "SKILL.md");
  if (!existsSync(path)) throw new Error(`SKILL.md not found for slug: ${slug}`);
  return readFileSync(path, "utf8");
}

function readAgentsMd(slug: string): string {
  const path = join(REPO_ROOT, "state/skills", slug, "AGENTS.md");
  if (!existsSync(path)) return "";
  return readFileSync(path, "utf8");
}

function readPromptFragment(slug: string): string {
  const path = join(REPO_ROOT, "state/skills", slug, "prompt-fragment.md");
  if (!existsSync(path)) return "";
  return readFileSync(path, "utf8");
}

function readRecentFailures(slug: string, limit = 20): EvalRow[] {
  const evalsPath = join(REPO_ROOT, "state/log/evals.ndjson");
  if (!existsSync(evalsPath)) return [];

  const lines = readFileSync(evalsPath, "utf8").trim().split("\n").filter(Boolean);
  // Take tail-200 to avoid full-file scan
  const tail = lines.slice(-200);

  const failures: EvalRow[] = [];
  for (const line of tail) {
    try {
      const row = JSON.parse(line) as EvalRow;
      const rowSlug = row.skill || row.verb || "";
      if (rowSlug !== slug) continue;
      if (typeof row.score !== "number") continue;
      if (row.score < 0.5) failures.push(row);
    } catch {
      // skip malformed lines
    }
  }

  // Return most recent failures first, up to limit
  return failures.slice(-limit).reverse();
}

function summarizeFailureMode(failures: EvalRow[]): string {
  if (failures.length === 0) return "No recent failures found (fewer than 20 eval rows with score < 0.5).";

  // Count primary_issue occurrences
  const issueCounts: Record<string, number> = {};
  for (const f of failures) {
    const issue = (f.primary_issue || "").trim();
    if (issue) {
      issueCounts[issue] = (issueCounts[issue] || 0) + 1;
    }
  }

  const topIssues = Object.entries(issueCounts)
    .sort((a, b) => b[1] - a[1])
    .slice(0, 3)
    .map(([issue, count]) => `"${issue}" (${count}x)`);

  const noteSnippets = failures
    .slice(0, 3)
    .map(f => f.note)
    .filter(Boolean)
    .map(n => `- ${(n || "").slice(0, 120)}`);

  let summary = `${failures.length} failures in last 200 evals (score < 0.5).`;
  if (topIssues.length) summary += ` Top issues: ${topIssues.join(", ")}.`;
  if (noteSnippets.length) summary += `\nRecent notes:\n${noteSnippets.join("\n")}`;

  return summary;
}

// --- LLM calls ---

const PROMPT_IMPROVER_SYSTEM = `You write concise, surgical prompt fragments for snappy-os skills.

A "prompt fragment" is a short block of text (typically 3-15 lines) that gets prepended to a skill agent's system prompt to address observed failure modes. It is NOT a full system prompt rewrite — it is a targeted patch.

WHAT YOU ARE GIVEN:
- The skill's SKILL.md (its purpose and behavior spec)
- Its current prompt-fragment.md (empty string if none exists yet)
- Its recent eval failures (score < 0.5): primary_issue + notes

YOUR JOB:
Write a NEW prompt-fragment.md that either:
a) Creates a new fragment (if none exists) that directly addresses the top failure modes.
b) Patches the existing fragment to fix the gaps the failures reveal.

FORMAT RULES:
- Start with a single-line header: # <slug> — prompt fragment
- Then 2-10 bullet points or short rules. No paragraphs.
- Each bullet addresses one specific failure mode, stated as a DO or DO NOT rule.
- Be direct and prescriptive, not abstract.
- No greetings, no "This fragment...", no meta-commentary.
- Under 200 words total.
- Output ONLY the fragment text. No explanation, no fences.`;

function buildUserPrompt(slug: string, skillBody: string, agentsBody: string, oldFragment: string, failures: EvalRow[]): string {
  const failureLines = failures.slice(0, 10).map(f => {
    const parts = [`score=${f.score}`];
    if (f.primary_issue) parts.push(`issue="${f.primary_issue}"`);
    if (f.note) parts.push(`note="${(f.note || "").slice(0, 100)}"`);
    return `- ${parts.join(" ")}`;
  }).join("\n") || "- (no failures found in recent evals)";

  return `SKILL SLUG: ${slug}

SKILL.md (truncated to 2000 chars):
${skillBody.slice(0, 2000)}

AGENTS.md (first 800 chars, if any):
${agentsBody.slice(0, 800) || "(no AGENTS.md)"}

CURRENT PROMPT FRAGMENT (empty = none exists):
${oldFragment || "(none)"}

RECENT FAILURES (last ${failures.length}, score < 0.5):
${failureLines}

Write the new prompt-fragment.md that addresses these failures. Output ONLY the fragment text.`;
}

function normalizeFragment(output: string): string {
  const trimmed = output.trim();
  const fenceMatch = trimmed.match(/```(?:markdown|md)?\s*([\s\S]*?)```/i);
  return (fenceMatch?.[1] || trimmed).trim();
}

async function callConfiguredRuntime(slug: string, systemPrompt: string, userPrompt: string): Promise<string> {
  const axis = readDispatchConfig().subagent;
  const modelLabel = axis.model === "auto" ? readDefaultModel().slug : axis.model;
  process.stderr.write(`[prompt-improver] composing ${slug} via ${axis.backend}/${modelLabel}\n`);

  const result = await dispatchFor("subagent", {
    prompt: userPrompt,
    systemPrompt,
    cwd: REPO_ROOT,
    tools: ["read", "grep", "ls"],
    timeoutMs: 120_000,
    interviewMode: false,
  });

  if (!result.ok || !result.output.trim()) {
    const detail = result.error || result.stderr || `exit ${result.exitCode}`;
    throw new Error(`${result.provider}/${result.model} failed: ${detail}`);
  }

  return normalizeFragment(result.output);
}

// --- Main export ---

export async function generatePromptPatch(slug: string): Promise<PromptPatch> {
  const skillMd = readSkillMd(slug); // throws if not found
  const agentsMd = readAgentsMd(slug);
  const oldFragment = readPromptFragment(slug);
  const failures = readRecentFailures(slug);
  const reason = summarizeFailureMode(failures);

  // Strip frontmatter from SKILL.md to get body prose
  const skillBody = skillMd.replace(/^---\n[\s\S]+?\n---\n?/, "").trim();

  const userPrompt = buildUserPrompt(slug, skillBody, agentsMd, oldFragment, failures);

  let proposed: string;
  try {
    proposed = await callConfiguredRuntime(slug, PROMPT_IMPROVER_SYSTEM, userPrompt);
  } catch (e) {
    throw new Error(`prompt runtime failed for ${slug}: ${(e as Error).message}`);
  }

  // Basic sanity: must start with # or bullet
  if (!proposed || (!proposed.startsWith("#") && !proposed.startsWith("-") && !proposed.startsWith("*"))) {
    process.stderr.write(`[prompt-improver] LLM output for ${slug} looks off, using as-is\n`);
  }

  return { slug, old: oldFragment, proposed, reason };
}

// --- CLI ---

const isMain = (() => {
  try {
    return import.meta.url === `file://${require("fs").realpathSync(process.argv[1])}`;
  } catch {
    try {
      return import.meta.url === `file://${process.argv[1]}`;
    } catch {
      return false;
    }
  }
})();

if (isMain) {
  (async () => {
    const slug = process.argv[2];
    if (!slug) {
      console.error("Usage: npx tsx state/lib/prompt-improver.ts <slug>");
      process.exit(1);
    }
    try {
      const patch = await generatePromptPatch(slug);
      console.log("=== OLD FRAGMENT ===");
      console.log(patch.old || "(none)");
      console.log("\n=== PROPOSED ===");
      console.log(patch.proposed);
      console.log("\n=== REASON ===");
      console.log(patch.reason);
    } catch (e) {
      console.error("Error:", (e as Error).message);
      process.exit(1);
    }
  })();
}

scripts- helper scripts it can run

prose-only skill - no sidecar under state/bin/ yet. Steps, if any, are described in SKILL.md.

how we check it- the checks, plus the last 3 runs