fix format

This commit is contained in:
Pedro Rodrigues
2026-01-23 17:33:44 +00:00
parent 1d9f4ea441
commit b5289ff6ee
19 changed files with 616 additions and 595 deletions

View File

@@ -7,7 +7,11 @@ import {
validateSkillExists, validateSkillExists,
} from "./config.js"; } from "./config.js";
import { parseRuleFile } from "./parser.js"; import { parseRuleFile } from "./parser.js";
import { filterRulesForProfile, listProfiles, loadProfile } from "./profiles.js"; import {
filterRulesForProfile,
listProfiles,
loadProfile,
} from "./profiles.js";
import type { Metadata, Profile, Rule, Section } from "./types.js"; import type { Metadata, Profile, Rule, Section } from "./types.js";
import { validateRuleFile } from "./validate.js"; import { validateRuleFile } from "./validate.js";
@@ -118,10 +122,7 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
// Check if rules directory exists // Check if rules directory exists
if (!existsSync(paths.rulesDir)) { if (!existsSync(paths.rulesDir)) {
console.log(` No rules directory found. Generating empty AGENTS.md.`); console.log(` No rules directory found. Generating empty AGENTS.md.`);
writeFileSync( writeFileSync(outputFile, `# ${skillTitle}\n\nNo rules defined yet.\n`);
outputFile,
`# ${skillTitle}\n\nNo rules defined yet.\n`,
);
return; return;
} }
@@ -157,7 +158,9 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
let filteredRules = rules; let filteredRules = rules;
if (profile) { if (profile) {
filteredRules = filterRulesForProfile(rules, profile); filteredRules = filterRulesForProfile(rules, profile);
console.log(` Filtered to ${filteredRules.length} rules for profile "${profile.name}"`); console.log(
` Filtered to ${filteredRules.length} rules for profile "${profile.name}"`,
);
} }
// Group rules by section and assign IDs // Group rules by section and assign IDs
@@ -244,7 +247,9 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
prerequisites.push(`PostgreSQL ${rule.minVersion}+`); prerequisites.push(`PostgreSQL ${rule.minVersion}+`);
} }
if (rule.extensions && rule.extensions.length > 0) { if (rule.extensions && rule.extensions.length > 0) {
prerequisites.push(`Extension${rule.extensions.length > 1 ? "s" : ""}: ${rule.extensions.join(", ")}`); prerequisites.push(
`Extension${rule.extensions.length > 1 ? "s" : ""}: ${rule.extensions.join(", ")}`,
);
} }
if (prerequisites.length > 0) { if (prerequisites.length > 0) {
output.push(`**Prerequisites:** ${prerequisites.join(" | ")}\n`); output.push(`**Prerequisites:** ${prerequisites.join(" | ")}\n`);
@@ -302,7 +307,11 @@ function buildSkill(paths: SkillPaths, profile?: Profile): void {
/** /**
* Parse CLI arguments * Parse CLI arguments
*/ */
function parseArgs(): { skill?: string; profile?: string; allProfiles: boolean } { function parseArgs(): {
skill?: string;
profile?: string;
allProfiles: boolean;
} {
const args = process.argv.slice(2); const args = process.argv.slice(2);
let skill: string | undefined; let skill: string | undefined;
let profile: string | undefined; let profile: string | undefined;

View File

@@ -251,7 +251,8 @@ export function parseRuleFile(
const examples = extractExamples(body); const examples = extractExamples(body);
const tags = frontmatter.tags?.split(",").map((t) => t.trim()) || []; const tags = frontmatter.tags?.split(",").map((t) => t.trim()) || [];
const extensions = frontmatter.extensions?.split(",").map((e) => e.trim()) || []; const extensions =
frontmatter.extensions?.split(",").map((e) => e.trim()) || [];
// Validation warnings // Validation warnings
if (!explanation || explanation.length < 20) { if (!explanation || explanation.length < 20) {

View File

@@ -5,7 +5,10 @@ import type { Profile, Rule } from "./types.js";
/** /**
* Load a profile from the profiles directory * Load a profile from the profiles directory
*/ */
export function loadProfile(profilesDir: string, profileName: string): Profile | null { export function loadProfile(
profilesDir: string,
profileName: string,
): Profile | null {
const profileFile = join(profilesDir, `${profileName}.json`); const profileFile = join(profilesDir, `${profileName}.json`);
if (!existsSync(profileFile)) { if (!existsSync(profileFile)) {
return null; return null;
@@ -54,14 +57,20 @@ function compareVersions(a: string, b: string): number {
/** /**
* Check if a rule is compatible with a profile * Check if a rule is compatible with a profile
*/ */
export function isRuleCompatibleWithProfile(rule: Rule, profile: Profile): boolean { export function isRuleCompatibleWithProfile(
rule: Rule,
profile: Profile,
): boolean {
// Check version requirement // Check version requirement
if (rule.minVersion) { if (rule.minVersion) {
if (compareVersions(rule.minVersion, profile.minVersion) > 0) { if (compareVersions(rule.minVersion, profile.minVersion) > 0) {
// Rule requires a higher version than profile supports // Rule requires a higher version than profile supports
return false; return false;
} }
if (profile.maxVersion && compareVersions(rule.minVersion, profile.maxVersion) > 0) { if (
profile.maxVersion &&
compareVersions(rule.minVersion, profile.maxVersion) > 0
) {
// Rule requires a version higher than profile's max // Rule requires a version higher than profile's max
return false; return false;
} }

View File

@@ -1,8 +1,13 @@
import { generateText } from "ai";
import { anthropic } from "@ai-sdk/anthropic";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import type { CriterionResult, EvalConfig, EvalResult, EvalScenario } from "./types.js"; import { anthropic } from "@ai-sdk/anthropic";
import { generateText } from "ai";
import type {
CriterionResult,
EvalConfig,
EvalResult,
EvalScenario,
} from "./types.js";
const DEFAULT_CONFIG: EvalConfig = { const DEFAULT_CONFIG: EvalConfig = {
agentsPath: join(import.meta.dirname, "..", "AGENTS.md"), agentsPath: join(import.meta.dirname, "..", "AGENTS.md"),
@@ -27,7 +32,9 @@ function buildUserPrompt(scenario: EvalScenario): string {
if (scenario.input.availableExtensions.length === 0) { if (scenario.input.availableExtensions.length === 0) {
parts.push("Available Extensions: None installed"); parts.push("Available Extensions: None installed");
} else { } else {
parts.push(`Available Extensions: ${scenario.input.availableExtensions.join(", ")}`); parts.push(
`Available Extensions: ${scenario.input.availableExtensions.join(", ")}`,
);
} }
} }
@@ -58,7 +65,10 @@ function extractRuleIds(response: string): string[] {
/** /**
* Evaluate the response against expected criteria * Evaluate the response against expected criteria
*/ */
function evaluateCriteria(scenario: EvalScenario, response: string): CriterionResult[] { function evaluateCriteria(
scenario: EvalScenario,
response: string,
): CriterionResult[] {
const results: CriterionResult[] = []; const results: CriterionResult[] = [];
const responseLower = response.toLowerCase(); const responseLower = response.toLowerCase();
@@ -79,7 +89,9 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
results.push({ results.push({
criterion: `Response should NOT contain "${term}"`, criterion: `Response should NOT contain "${term}"`,
passed: !found, passed: !found,
evidence: found ? "Found in response (should not be present)" : "Not found (correct)", evidence: found
? "Found in response (should not be present)"
: "Not found (correct)",
}); });
} }
} }
@@ -102,7 +114,9 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
results.push({ results.push({
criterion: `Should NOT recommend rule ${ruleId}`, criterion: `Should NOT recommend rule ${ruleId}`,
passed: !found, passed: !found,
evidence: found ? "Rule referenced (should not be)" : "Rule not referenced (correct)", evidence: found
? "Rule referenced (should not be)"
: "Rule not referenced (correct)",
}); });
} }
} }
@@ -115,7 +129,7 @@ function evaluateCriteria(scenario: EvalScenario, response: string): CriterionRe
*/ */
export async function runEval( export async function runEval(
scenario: EvalScenario, scenario: EvalScenario,
config: Partial<EvalConfig> = {} config: Partial<EvalConfig> = {},
): Promise<EvalResult> { ): Promise<EvalResult> {
const finalConfig = { ...DEFAULT_CONFIG, ...config }; const finalConfig = { ...DEFAULT_CONFIG, ...config };
@@ -138,7 +152,7 @@ When making recommendations, reference specific rule IDs (e.g., "1.1", "2.3") fr
const start = Date.now(); const start = Date.now();
const { text } = await generateText({ const { text } = await generateText({
model: anthropic(finalConfig.model!), model: anthropic(finalConfig.model ?? DEFAULT_CONFIG.model),
system: systemPrompt, system: systemPrompt,
prompt: userPrompt, prompt: userPrompt,
maxTokens: finalConfig.maxTokens, maxTokens: finalConfig.maxTokens,
@@ -177,7 +191,7 @@ When making recommendations, reference specific rule IDs (e.g., "1.1", "2.3") fr
*/ */
export async function runEvals( export async function runEvals(
scenarios: EvalScenario[], scenarios: EvalScenario[],
config: Partial<EvalConfig> = {} config: Partial<EvalConfig> = {},
): Promise<EvalResult[]> { ): Promise<EvalResult[]> {
const results: EvalResult[] = []; const results: EvalResult[] = [];

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -54,9 +54,8 @@ describe("Covering Index Suggestion", () => {
// Response should mention covering index concept // Response should mention covering index concept
const responseLower = result.response.toLowerCase(); const responseLower = result.response.toLowerCase();
expect( expect(
responseLower.includes("covering") || responseLower.includes("index-only") responseLower.includes("covering") ||
responseLower.includes("index-only"),
).toBe(true); ).toBe(true);
}); });
}); });
export { scenario };

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -48,9 +48,7 @@ describe("Extension Available - pg_stat_statements", () => {
expect( expect(
responseLower.includes("create extension") || responseLower.includes("create extension") ||
responseLower.includes("enable") || responseLower.includes("enable") ||
responseLower.includes("query") responseLower.includes("query"),
).toBe(true); ).toBe(true);
}); });
}); });
export { scenario };

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -49,8 +49,8 @@ describe("Extension Unavailable - No pg_stat_statements", () => {
const responseLower = result.response.toLowerCase(); const responseLower = result.response.toLowerCase();
// Should suggest EXPLAIN ANALYZE as an alternative // Should suggest EXPLAIN ANALYZE as an alternative
expect(responseLower.includes("explain") && responseLower.includes("analyze")).toBe(true); expect(
responseLower.includes("explain") && responseLower.includes("analyze"),
).toBe(true);
}); });
}); });
export { scenario };

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -43,14 +43,14 @@ describe("Missing Index Detection", () => {
console.log("Criteria results:", result.criteriaResults); console.log("Criteria results:", result.criteriaResults);
// Check that key criteria passed // Check that key criteria passed
expect(result.criteriaResults.some((c) => c.criterion.includes("index") && c.passed)).toBe( expect(
true result.criteriaResults.some(
); (c) => c.criterion.includes("index") && c.passed,
),
).toBe(true);
// Response should mention creating an index // Response should mention creating an index
expect(result.response.toLowerCase()).toContain("index"); expect(result.response.toLowerCase()).toContain("index");
expect(result.response.toLowerCase()).toContain("customer_id"); expect(result.response.toLowerCase()).toContain("customer_id");
}); });
}); });
export { scenario };

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -64,8 +64,8 @@ describe("N+1 Query Detection", () => {
// Response should explain the N+1 problem // Response should explain the N+1 problem
const responseLower = result.response.toLowerCase(); const responseLower = result.response.toLowerCase();
expect(responseLower.includes("n+1") || responseLower.includes("n + 1")).toBe(true); expect(
responseLower.includes("n+1") || responseLower.includes("n + 1"),
).toBe(true);
}); });
}); });
export { scenario };

View File

@@ -1,4 +1,4 @@
import { describe, it, expect } from "vitest"; import { describe, expect, it } from "vitest";
import { runEval } from "../runner.js"; import { runEval } from "../runner.js";
import type { EvalScenario } from "../types.js"; import type { EvalScenario } from "../types.js";
@@ -104,5 +104,3 @@ describe("Version Constraint Tests", () => {
}); });
}); });
}); });
export { scenarioPg10NoCoveringIndex, scenarioPg93NoUpsert };

View File

@@ -12,7 +12,10 @@ export interface EvalScenario {
description: string; description: string;
/** Category of the scenario */ /** Category of the scenario */
category: "query-performance" | "version-constraints" | "extension-requirements"; category:
| "query-performance"
| "version-constraints"
| "extension-requirements";
/** Difficulty level */ /** Difficulty level */
difficulty: "basic" | "intermediate" | "advanced"; difficulty: "basic" | "intermediate" | "advanced";

View File

@@ -36,7 +36,9 @@ export function formatDetailedResult(result: EvalResult): string {
lines.push(`## ${result.scenarioId}\n`); lines.push(`## ${result.scenarioId}\n`);
lines.push(`**Status:** ${result.passed ? "PASS" : "FAIL"}`); lines.push(`**Status:** ${result.passed ? "PASS" : "FAIL"}`);
lines.push(`**Latency:** ${result.latencyMs}ms`); lines.push(`**Latency:** ${result.latencyMs}ms`);
lines.push(`**Rules Referenced:** ${result.rulesReferenced.join(", ") || "none"}\n`); lines.push(
`**Rules Referenced:** ${result.rulesReferenced.join(", ") || "none"}\n`,
);
if (result.error) { if (result.error) {
lines.push(`**Error:** ${result.error}\n`); lines.push(`**Error:** ${result.error}\n`);
@@ -63,7 +65,7 @@ export function formatDetailedResult(result: EvalResult): string {
* Create a scenario builder for cleaner test definitions * Create a scenario builder for cleaner test definitions
*/ */
export function createScenario( export function createScenario(
partial: Omit<EvalScenario, "id"> & { id?: string } partial: Omit<EvalScenario, "id"> & { id?: string },
): EvalScenario { ): EvalScenario {
return { return {
id: partial.id || partial.name.toLowerCase().replace(/\s+/g, "-"), id: partial.id || partial.name.toLowerCase().replace(/\s+/g, "-"),

View File

@@ -3,21 +3,9 @@
"minVersion": "13", "minVersion": "13",
"maxVersion": "16", "maxVersion": "16",
"extensions": { "extensions": {
"available": [ "available": ["pg_stat_statements", "pgcrypto", "uuid-ossp"],
"pg_stat_statements", "installable": ["postgis", "pg_hint_plan", "pg_similarity"],
"pgcrypto", "unavailable": ["pg_cron", "pg_partman", "timescaledb"]
"uuid-ossp"
],
"installable": [
"postgis",
"pg_hint_plan",
"pg_similarity"
],
"unavailable": [
"pg_cron",
"pg_partman",
"timescaledb"
]
}, },
"notes": "AWS Aurora PostgreSQL. Some extensions are not available due to managed service restrictions. Aurora has its own connection pooling (RDS Proxy) and automatic failover." "notes": "AWS Aurora PostgreSQL. Some extensions are not available due to managed service restrictions. Aurora has its own connection pooling (RDS Proxy) and automatic failover."
} }