feat: mount skill into Docker container for skill activation eval

This commit is contained in:
Matt Rossman
2026-03-03 14:31:27 -05:00
parent dce4600c08
commit debf7a66f4

View File

@@ -1,4 +1,5 @@
import { execSync, spawnSync } from "node:child_process";
import path from "node:path";
const apiKey = process.env.ANTHROPIC_API_KEY;
if (!apiKey) throw new Error("ANTHROPIC_API_KEY required");
@@ -6,10 +7,25 @@ if (!apiKey) throw new Error("ANTHROPIC_API_KEY required");
try {
execSync("docker image inspect evals-claude", { stdio: "ignore" });
} catch {
console.error("Docker image 'evals-claude' not found. Build it first with:\n npm run evals:build");
console.error(
"Docker image 'evals-claude' not found. Build it first with:\n npm run evals:build",
);
process.exit(1);
}
const repoRoot = path.resolve(__dirname, "..");
const skillPath = path.join(
repoRoot,
"skills",
"supabase-postgres-best-practices",
);
const prompt = `Review this SQL query for a Supabase project and suggest optimizations:
SELECT * FROM orders WHERE user_id = 123 AND status = 'pending';
What indexes should I add and why?`;
const result = spawnSync(
"docker",
[
@@ -17,10 +33,12 @@ const result = spawnSync(
"--rm",
"-e",
`ANTHROPIC_API_KEY=${apiKey}`,
"-v",
`${skillPath}:/root/.claude/skills/supabase-postgres-best-practices:ro`, // :ro = read-only snapshot
"evals-claude",
"claude",
"-p",
"Hello",
prompt,
],
{ encoding: "utf-8" },
);