mirror of
https://github.com/supabase/agent-skills.git
synced 2026-01-26 19:09:51 +08:00
fix format
This commit is contained in:
@@ -7,7 +7,11 @@ import {
|
|||||||
validateSkillExists,
|
validateSkillExists,
|
||||||
} from "./config.js";
|
} from "./config.js";
|
||||||
import { parseRuleFile } from "./parser.js";
|
import { parseRuleFile } from "./parser.js";
|
||||||
import { filterRulesForProfile, listProfiles, loadProfile } from "./profiles.js";
|
import {
|
||||||
|
filterRulesForProfile,
|
||||||
|
listProfiles,
|
||||||
|
loadProfile,
|
||||||
|
} from "./profiles.js";
|
||||||
import type { Metadata, Profile, Rule, Section } from "./types.js";
|
import type { Metadata, Profile, Rule, Section } from "./types.js";
|
||||||
import { validateRuleFile } from "./validate.js";
|
import { validateRuleFile } from "./validate.js";
|
||||||
|
|
||||||
@@ -118,10 +122,7 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
|
|||||||
// Check if rules directory exists
|
// Check if rules directory exists
|
||||||
if (!existsSync(paths.rulesDir)) {
|
if (!existsSync(paths.rulesDir)) {
|
||||||
console.log(` No rules directory found. Generating empty AGENTS.md.`);
|
console.log(` No rules directory found. Generating empty AGENTS.md.`);
|
||||||
writeFileSync(
|
writeFileSync(outputFile, `# ${skillTitle}\n\nNo rules defined yet.\n`);
|
||||||
outputFile,
|
|
||||||
`# ${skillTitle}\n\nNo rules defined yet.\n`,
|
|
||||||
);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,7 +158,9 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
|
|||||||
let filteredRules = rules;
|
let filteredRules = rules;
|
||||||
if (profile) {
|
if (profile) {
|
||||||
filteredRules = filterRulesForProfile(rules, profile);
|
filteredRules = filterRulesForProfile(rules, profile);
|
||||||
console.log(` Filtered to ${filteredRules.length} rules for profile "${profile.name}"`);
|
console.log(
|
||||||
|
` Filtered to ${filteredRules.length} rules for profile "${profile.name}"`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Group rules by section and assign IDs
|
// Group rules by section and assign IDs
|
||||||
@@ -244,7 +247,9 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
|
|||||||
prerequisites.push(`PostgreSQL ${rule.minVersion}+`);
|
prerequisites.push(`PostgreSQL ${rule.minVersion}+`);
|
||||||
}
|
}
|
||||||
if (rule.extensions && rule.extensions.length > 0) {
|
if (rule.extensions && rule.extensions.length > 0) {
|
||||||
prerequisites.push(`Extension${rule.extensions.length > 1 ? "s" : ""}: ${rule.extensions.join(", ")}`);
|
prerequisites.push(
|
||||||
|
`Extension${rule.extensions.length > 1 ? "s" : ""}: ${rule.extensions.join(", ")}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if (prerequisites.length > 0) {
|
if (prerequisites.length > 0) {
|
||||||
output.push(`**Prerequisites:** ${prerequisites.join(" | ")}\n`);
|
output.push(`**Prerequisites:** ${prerequisites.join(" | ")}\n`);
|
||||||
@@ -302,7 +307,11 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
|
|||||||
/**
|
/**
|
||||||
* Parse CLI arguments
|
* Parse CLI arguments
|
||||||
*/
|
*/
|
||||||
function parseArgs(): { skill?: string; profile?: string; allProfiles: boolean } {
|
function parseArgs(): {
|
||||||
|
skill?: string;
|
||||||
|
profile?: string;
|
||||||
|
allProfiles: boolean;
|
||||||
|
} {
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
let skill: string | undefined;
|
let skill: string | undefined;
|
||||||
let profile: string | undefined;
|
let profile: string | undefined;
|
||||||
|
|||||||
@@ -251,7 +251,8 @@ export function parseRuleFile(
|
|||||||
const examples = extractExamples(body);
|
const examples = extractExamples(body);
|
||||||
|
|
||||||
const tags = frontmatter.tags?.split(",").map((t) => t.trim()) || [];
|
const tags = frontmatter.tags?.split(",").map((t) => t.trim()) || [];
|
||||||
const extensions = frontmatter.extensions?.split(",").map((e) => e.trim()) || [];
|
const extensions =
|
||||||
|
frontmatter.extensions?.split(",").map((e) => e.trim()) || [];
|
||||||
|
|
||||||
// Validation warnings
|
// Validation warnings
|
||||||
if (!explanation || explanation.length < 20) {
|
if (!explanation || explanation.length < 20) {
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ import type { Profile, Rule } from "./types.js";
|
|||||||
/**
|
/**
|
||||||
* Load a profile from the profiles directory
|
* Load a profile from the profiles directory
|
||||||
*/
|
*/
|
||||||
export function loadProfile(profilesDir: string, profileName: string): Profile | null {
|
export function loadProfile(
|
||||||
|
profilesDir: string,
|
||||||
|
profileName: string,
|
||||||
|
): Profile | null {
|
||||||
const profileFile = join(profilesDir, `${profileName}.json`);
|
const profileFile = join(profilesDir, `${profileName}.json`);
|
||||||
if (!existsSync(profileFile)) {
|
if (!existsSync(profileFile)) {
|
||||||
return null;
|
return null;
|
||||||
@@ -54,14 +57,20 @@ function compareVersions(a: string, b: string): number {
|
|||||||
/**
|
/**
|
||||||
* Check if a rule is compatible with a profile
|
* Check if a rule is compatible with a profile
|
||||||
*/
|
*/
|
||||||
export function isRuleCompatibleWithProfile(rule: Rule, profile: Profile): boolean {
|
export function isRuleCompatibleWithProfile(
|
||||||
|
rule: Rule,
|
||||||
|
profile: Profile,
|
||||||
|
): boolean {
|
||||||
// Check version requirement
|
// Check version requirement
|
||||||
if (rule.minVersion) {
|
if (rule.minVersion) {
|
||||||
if (compareVersions(rule.minVersion, profile.minVersion) > 0) {
|
if (compareVersions(rule.minVersion, profile.minVersion) > 0) {
|
||||||
// Rule requires a higher version than profile supports
|
// Rule requires a higher version than profile supports
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (profile.maxVersion && compareVersions(rule.minVersion, profile.maxVersion) > 0) {
|
if (
|
||||||
|
profile.maxVersion &&
|
||||||
|
compareVersions(rule.minVersion, profile.maxVersion) > 0
|
||||||
|
) {
|
||||||
// Rule requires a version higher than profile's max
|
// Rule requires a version higher than profile's max
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
import { generateText } from "ai";
|
|
||||||
import { anthropic } from "@ai-sdk/anthropic";
|
|
||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import type { CriterionResult, EvalConfig, EvalResult, EvalScenario } from "./types.js";
|
import { anthropic } from "@ai-sdk/anthropic";
|
||||||
|
import { generateText } from "ai";
|
||||||
|
import type {
|
||||||
|
CriterionResult,
|
||||||
|
EvalConfig,
|
||||||
|
EvalResult,
|
||||||
|
EvalScenario,
|
||||||
|
} from "./types.js";
|
||||||
|
|
||||||
const DEFAULT_CONFIG: EvalConfig = {
|
const DEFAULT_CONFIG: EvalConfig = {
|
||||||
agentsPath: join(import.meta.dirname, "..", "AGENTS.md"),
|
agentsPath: join(import.meta.dirname, "..", "AGENTS.md"),
|
||||||
@@ -27,7 +32,9 @@ function buildUserPrompt(scenario: EvalScenario): string {
|
|||||||
if (scenario.input.availableExtensions.length === 0) {
|
if (scenario.input.availableExtensions.length === 0) {
|
||||||
parts.push("Available Extensions: None installed");
|
parts.push("Available Extensions: None installed");
|
||||||
} else {
|
} else {
|
||||||
parts.push(`Available Extensions: ${scenario.input.availableExtensions.join(", ")}`);
|
parts.push(
|
||||||
|
`Available Extensions: ${scenario.input.availableExtensions.join(", ")}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,7 +65,10 @@ function extractRuleIds(response: string): string[] {
|
|||||||
/**
|
/**
|
||||||
* Evaluate the response against expected criteria
|
* Evaluate the response against expected criteria
|
||||||
*/
|
*/
|
||||||
function evaluateCriteria(scenario: EvalScenario, response: string): CriterionResult[] {
|
function evaluateCriteria(
|
||||||
|
scenario: EvalScenario,
|
||||||
|
response: string,
|
||||||
|
): CriterionResult[] {
|
||||||
const results: CriterionResult[] = [];
|
const results: CriterionResult[] = [];
|
||||||
const responseLower = response.toLowerCase();
|
const responseLower = response.toLowerCase();
|
||||||
|
|
||||||
@@ -79,7 +89,9 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
|
|||||||
results.push({
|
results.push({
|
||||||
criterion: `Response should NOT contain "${term}"`,
|
criterion: `Response should NOT contain "${term}"`,
|
||||||
passed: !found,
|
passed: !found,
|
||||||
evidence: found ? "Found in response (should not be present)" : "Not found (correct)",
|
evidence: found
|
||||||
|
? "Found in response (should not be present)"
|
||||||
|
: "Not found (correct)",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -102,7 +114,9 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
|
|||||||
results.push({
|
results.push({
|
||||||
criterion: `Should NOT recommend rule ${ruleId}`,
|
criterion: `Should NOT recommend rule ${ruleId}`,
|
||||||
passed: !found,
|
passed: !found,
|
||||||
evidence: found ? "Rule referenced (should not be)" : "Rule not referenced (correct)",
|
evidence: found
|
||||||
|
? "Rule referenced (should not be)"
|
||||||
|
: "Rule not referenced (correct)",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -115,7 +129,7 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
|
|||||||
*/
|
*/
|
||||||
export async function runEval(
|
export async function runEval(
|
||||||
scenario: EvalScenario,
|
scenario: EvalScenario,
|
||||||
config: Partial<EvalConfig> = {}
|
config: Partial<EvalConfig> = {},
|
||||||
): Promise<EvalResult> {
|
): Promise<EvalResult> {
|
||||||
const finalConfig = { ...DEFAULT_CONFIG, ...config };
|
const finalConfig = { ...DEFAULT_CONFIG, ...config };
|
||||||
|
|
||||||
@@ -138,7 +152,7 @@ When making recommendations, reference specific rule IDs (e.g., "1.1", "2.3") fr
|
|||||||
|
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
const { text } = await generateText({
|
const { text } = await generateText({
|
||||||
model: anthropic(finalConfig.model!),
|
model: anthropic(finalConfig.model ?? DEFAULT_CONFIG.model),
|
||||||
system: systemPrompt,
|
system: systemPrompt,
|
||||||
prompt: userPrompt,
|
prompt: userPrompt,
|
||||||
maxTokens: finalConfig.maxTokens,
|
maxTokens: finalConfig.maxTokens,
|
||||||
@@ -177,7 +191,7 @@ When making recommendations, reference specific rule IDs (e.g., "1.1", "2.3") fr
|
|||||||
*/
|
*/
|
||||||
export async function runEvals(
|
export async function runEvals(
|
||||||
scenarios: EvalScenario[],
|
scenarios: EvalScenario[],
|
||||||
config: Partial<EvalConfig> = {}
|
config: Partial<EvalConfig> = {},
|
||||||
): Promise<EvalResult[]> {
|
): Promise<EvalResult[]> {
|
||||||
const results: EvalResult[] = [];
|
const results: EvalResult[] = [];
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -54,9 +54,8 @@ describe("Covering Index Suggestion", () => {
|
|||||||
// Response should mention covering index concept
|
// Response should mention covering index concept
|
||||||
const responseLower = result.response.toLowerCase();
|
const responseLower = result.response.toLowerCase();
|
||||||
expect(
|
expect(
|
||||||
responseLower.includes("covering") || responseLower.includes("index-only")
|
responseLower.includes("covering") ||
|
||||||
|
responseLower.includes("index-only"),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenario };
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -48,9 +48,7 @@ describe("Extension Available - pg_stat_statements", () => {
|
|||||||
expect(
|
expect(
|
||||||
responseLower.includes("create extension") ||
|
responseLower.includes("create extension") ||
|
||||||
responseLower.includes("enable") ||
|
responseLower.includes("enable") ||
|
||||||
responseLower.includes("query")
|
responseLower.includes("query"),
|
||||||
).toBe(true);
|
).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenario };
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -49,8 +49,8 @@ describe("Extension Unavailable - No pg_stat_statements", () => {
|
|||||||
const responseLower = result.response.toLowerCase();
|
const responseLower = result.response.toLowerCase();
|
||||||
|
|
||||||
// Should suggest EXPLAIN ANALYZE as an alternative
|
// Should suggest EXPLAIN ANALYZE as an alternative
|
||||||
expect(responseLower.includes("explain") && responseLower.includes("analyze")).toBe(true);
|
expect(
|
||||||
|
responseLower.includes("explain") && responseLower.includes("analyze"),
|
||||||
|
).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenario };
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -43,14 +43,14 @@ describe("Missing Index Detection", () => {
|
|||||||
console.log("Criteria results:", result.criteriaResults);
|
console.log("Criteria results:", result.criteriaResults);
|
||||||
|
|
||||||
// Check that key criteria passed
|
// Check that key criteria passed
|
||||||
expect(result.criteriaResults.some((c) => c.criterion.includes("index") && c.passed)).toBe(
|
expect(
|
||||||
true
|
result.criteriaResults.some(
|
||||||
);
|
(c) => c.criterion.includes("index") && c.passed,
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
|
||||||
// Response should mention creating an index
|
// Response should mention creating an index
|
||||||
expect(result.response.toLowerCase()).toContain("index");
|
expect(result.response.toLowerCase()).toContain("index");
|
||||||
expect(result.response.toLowerCase()).toContain("customer_id");
|
expect(result.response.toLowerCase()).toContain("customer_id");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenario };
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -64,8 +64,8 @@ describe("N+1 Query Detection", () => {
|
|||||||
|
|
||||||
// Response should explain the N+1 problem
|
// Response should explain the N+1 problem
|
||||||
const responseLower = result.response.toLowerCase();
|
const responseLower = result.response.toLowerCase();
|
||||||
expect(responseLower.includes("n+1") || responseLower.includes("n + 1")).toBe(true);
|
expect(
|
||||||
|
responseLower.includes("n+1") || responseLower.includes("n + 1"),
|
||||||
|
).toBe(true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenario };
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { runEval } from "../runner.js";
|
import { runEval } from "../runner.js";
|
||||||
import type { EvalScenario } from "../types.js";
|
import type { EvalScenario } from "../types.js";
|
||||||
|
|
||||||
@@ -104,5 +104,3 @@ describe("Version Constraint Tests", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
export { scenarioPg10NoCoveringIndex, scenarioPg93NoUpsert };
|
|
||||||
|
|||||||
@@ -12,7 +12,10 @@ export interface EvalScenario {
|
|||||||
description: string;
|
description: string;
|
||||||
|
|
||||||
/** Category of the scenario */
|
/** Category of the scenario */
|
||||||
category: "query-performance" | "version-constraints" | "extension-requirements";
|
category:
|
||||||
|
| "query-performance"
|
||||||
|
| "version-constraints"
|
||||||
|
| "extension-requirements";
|
||||||
|
|
||||||
/** Difficulty level */
|
/** Difficulty level */
|
||||||
difficulty: "basic" | "intermediate" | "advanced";
|
difficulty: "basic" | "intermediate" | "advanced";
|
||||||
|
|||||||
@@ -36,7 +36,9 @@ export function formatDetailedResult(result: EvalResult): string {
|
|||||||
lines.push(`## ${result.scenarioId}\n`);
|
lines.push(`## ${result.scenarioId}\n`);
|
||||||
lines.push(`**Status:** ${result.passed ? "PASS" : "FAIL"}`);
|
lines.push(`**Status:** ${result.passed ? "PASS" : "FAIL"}`);
|
||||||
lines.push(`**Latency:** ${result.latencyMs}ms`);
|
lines.push(`**Latency:** ${result.latencyMs}ms`);
|
||||||
lines.push(`**Rules Referenced:** ${result.rulesReferenced.join(", ") || "none"}\n`);
|
lines.push(
|
||||||
|
`**Rules Referenced:** ${result.rulesReferenced.join(", ") || "none"}\n`,
|
||||||
|
);
|
||||||
|
|
||||||
if (result.error) {
|
if (result.error) {
|
||||||
lines.push(`**Error:** ${result.error}\n`);
|
lines.push(`**Error:** ${result.error}\n`);
|
||||||
@@ -63,7 +65,7 @@ export function formatDetailedResult(result: EvalResult): string {
|
|||||||
* Create a scenario builder for cleaner test definitions
|
* Create a scenario builder for cleaner test definitions
|
||||||
*/
|
*/
|
||||||
export function createScenario(
|
export function createScenario(
|
||||||
partial: Omit<EvalScenario, "id"> & { id?: string }
|
partial: Omit<EvalScenario, "id"> & { id?: string },
|
||||||
): EvalScenario {
|
): EvalScenario {
|
||||||
return {
|
return {
|
||||||
id: partial.id || partial.name.toLowerCase().replace(/\s+/g, "-"),
|
id: partial.id || partial.name.toLowerCase().replace(/\s+/g, "-"),
|
||||||
|
|||||||
@@ -3,21 +3,9 @@
|
|||||||
"minVersion": "13",
|
"minVersion": "13",
|
||||||
"maxVersion": "16",
|
"maxVersion": "16",
|
||||||
"extensions": {
|
"extensions": {
|
||||||
"available": [
|
"available": ["pg_stat_statements", "pgcrypto", "uuid-ossp"],
|
||||||
"pg_stat_statements",
|
"installable": ["postgis", "pg_hint_plan", "pg_similarity"],
|
||||||
"pgcrypto",
|
"unavailable": ["pg_cron", "pg_partman", "timescaledb"]
|
||||||
"uuid-ossp"
|
|
||||||
],
|
|
||||||
"installable": [
|
|
||||||
"postgis",
|
|
||||||
"pg_hint_plan",
|
|
||||||
"pg_similarity"
|
|
||||||
],
|
|
||||||
"unavailable": [
|
|
||||||
"pg_cron",
|
|
||||||
"pg_partman",
|
|
||||||
"timescaledb"
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"notes": "AWS Aurora PostgreSQL. Some extensions are not available due to managed service restrictions. Aurora has its own connection pooling (RDS Proxy) and automatic failover."
|
"notes": "AWS Aurora PostgreSQL. Some extensions are not available due to managed service restrictions. Aurora has its own connection pooling (RDS Proxy) and automatic failover."
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user