realtime scenario

packages/evals/evals/collaborative-rooms-realtime/EVAL.ts

@@ -0,0 +1,333 @@
+import { expect, test } from "vitest";
+
+import { findMigrationFiles, getMigrationSQL } from "../eval-utils.ts";
+
+test("migration file exists", () => {
+	expect(findMigrationFiles().length).toBeGreaterThan(0);
+});
+
+test("creates rooms table", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	expect(sql).toMatch(/create\s+table[\s\S]*?rooms/);
+});
+
+test("creates room_members table", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Accept room_members, members, memberships, room_users, etc.
+	const hasMembership =
+		/create\s+table[\s\S]*?room_members/.test(sql) ||
+		/create\s+table[\s\S]*?room_users/.test(sql) ||
+		/create\s+table[\s\S]*?memberships/.test(sql);
+	expect(hasMembership).toBe(true);
+});
+
+test("creates content table", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Accept content, contents, items, room_content, room_items, documents, etc.
+	const hasContent =
+		/create\s+table[\s\S]*?content/.test(sql) ||
+		/create\s+table[\s\S]*?items/.test(sql) ||
+		/create\s+table[\s\S]*?documents/.test(sql) ||
+		/create\s+table[\s\S]*?posts/.test(sql) ||
+		/create\s+table[\s\S]*?messages/.test(sql);
+	expect(hasContent).toBe(true);
+});
+
+test("room_members has role column with owner/editor/viewer", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	expect(sql).toMatch(/role/);
+	// Should define the three roles somewhere (enum, check constraint, or comment)
+	expect(sql).toMatch(/owner/);
+	expect(sql).toMatch(/editor/);
+	expect(sql).toMatch(/viewer/);
+});
+
+test("enables RLS on all application tables", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Must enable RLS on rooms
+	expect(sql).toMatch(
+		/alter\s+table[\s\S]*?rooms[\s\S]*?enable\s+row\s+level\s+security/,
+	);
+	// Must enable RLS on membership table
+	const hasMembershipRls =
+		/alter\s+table[\s\S]*?room_members[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?room_users[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?memberships[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		);
+	expect(hasMembershipRls).toBe(true);
+	// Must enable RLS on content table (accept various names)
+	const hasContentRls =
+		/alter\s+table[\s\S]*?content[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?items[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?documents[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?posts[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		) ||
+		/alter\s+table[\s\S]*?messages[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		);
+	expect(hasContentRls).toBe(true);
+});
+
+test("FK to auth.users with ON DELETE CASCADE", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	expect(sql).toMatch(/references\s+auth\.users/);
+	expect(sql).toMatch(/on\s+delete\s+cascade/);
+});
+
+test("content has room_id FK referencing rooms", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Content table should have a foreign key to rooms
+	expect(sql).toMatch(/room_id[\s\S]*?references[\s\S]*?rooms/);
+});
+
+test("policies use (select auth.uid())", () => {
+	const sql = getMigrationSQL();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	expect(policyBlocks.length).toBeGreaterThan(0);
+	for (const policy of policyBlocks) {
+		if (policy.includes("auth.uid()")) {
+			expect(policy).toMatch(/\(\s*select\s+auth\.uid\(\)\s*\)/i);
+		}
+	}
+});
+
+test("policies use TO authenticated", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	// Filter to only application table policies (not realtime.messages which may use different roles)
+	const appPolicies = policyBlocks.filter(
+		(p) => !p.includes("realtime.messages"),
+	);
+	expect(appPolicies.length).toBeGreaterThan(0);
+	for (const policy of appPolicies) {
+		expect(policy).toMatch(/to\s+authenticated/);
+	}
+});
+
+test("private schema with security_definer helper function", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Private schema should be created
+	expect(sql).toMatch(/create\s+schema[\s\S]*?private/);
+	// A function in the private schema with SECURITY DEFINER
+	expect(sql).toMatch(/private\./);
+	expect(sql).toMatch(/security\s+definer/);
+	expect(sql).toMatch(/set\s+search_path\s*=\s*''/);
+});
+
+test("role-based write policies: content INSERT/UPDATE restricted to owner or editor", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	// Find INSERT or UPDATE policies on the content table
+	const writePolicies = policyBlocks.filter(
+		(p) =>
+			(/for\s+(insert|update|all)/.test(p) || /insert|update/.test(p)) &&
+			(p.includes("content") ||
+				p.includes("items") ||
+				p.includes("documents") ||
+				p.includes("posts") ||
+				p.includes("messages")),
+	);
+	// At least one write policy should check for owner or editor role
+	const checksRole = writePolicies.some(
+		(p) => p.includes("owner") || p.includes("editor"),
+	);
+	expect(checksRole).toBe(true);
+});
+
+test("viewer role is read-only (no write access to content)", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	// Find content write policies (INSERT, UPDATE, DELETE)
+	const contentWritePolicies = policyBlocks.filter(
+		(p) =>
+			/for\s+(insert|update|delete)/.test(p) &&
+			(p.includes("content") ||
+				p.includes("items") ||
+				p.includes("documents") ||
+				p.includes("posts") ||
+				p.includes("messages")),
+	);
+	// None of the write policies should grant access to viewer role
+	// They should either explicitly check for owner/editor OR exclude viewer
+	if (contentWritePolicies.length > 0) {
+		const anyGrantsViewer = contentWritePolicies.some((p) => {
+			// If the policy doesn't mention any role, it's too permissive
+			const mentionsRole =
+				p.includes("owner") || p.includes("editor") || p.includes("viewer");
+			if (!mentionsRole) return true; // no role check = viewer could write
+			// If it specifically includes viewer in a write context, that's wrong
+			return (
+				p.includes("viewer") && !p.includes("owner") && !p.includes("editor")
+			);
+		});
+		expect(anyGrantsViewer).toBe(false);
+	}
+});
+
+test("indexes on membership lookup columns", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	expect(sql).toMatch(/create\s+index/);
+	const indexBlocks = sql.match(/create\s+index[\s\S]*?;/gi) ?? [];
+	// Should index user_id and/or room_id on the membership table
+	const membershipIndexes = indexBlocks.filter(
+		(idx) =>
+			idx.toLowerCase().includes("user_id") ||
+			idx.toLowerCase().includes("room_id"),
+	);
+	expect(membershipIndexes.length).toBeGreaterThanOrEqual(1);
+});
+
+test("uses timestamptz not plain timestamp", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Match "timestamp" that is NOT followed by "tz" or "with time zone"
+	const hasPlainTimestamp =
+		/(?:created_at|updated_at|invited_at|joined_at)\s+timestamp(?!\s*tz)(?!\s+with\s+time\s+zone)/;
+	// Only fail if the migration defines time columns with plain timestamp
+	if (
+		sql.includes("created_at") ||
+		sql.includes("updated_at") ||
+		sql.includes("_at ")
+	) {
+		expect(sql).not.toMatch(hasPlainTimestamp);
+	}
+});
+
+test("idempotent DDL", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	expect(sql).toMatch(/if\s+not\s+exists/);
+});
+
+test("realtime publication enabled for content table", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Should add the content table to supabase_realtime publication
+	expect(sql).toMatch(/alter\s+publication\s+supabase_realtime\s+add\s+table/);
+});
+
+test("broadcast trigger for content changes", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Should use realtime.broadcast_changes() or realtime.send() in a trigger
+	const usesBroadcastChanges = /realtime\.broadcast_changes/.test(sql);
+	const usesRealtimeSend = /realtime\.send/.test(sql);
+	expect(usesBroadcastChanges || usesRealtimeSend).toBe(true);
+	// Should create a trigger on the content table
+	expect(sql).toMatch(/create\s+trigger/);
+});
+
+test("broadcast trigger function uses security definer", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	// Find function definitions that reference realtime.broadcast_changes or realtime.send
+	const functionBlocks =
+		sql.match(/create[\s\S]*?function[\s\S]*?\$\$[\s\S]*?\$\$/gi) ?? [];
+	const realtimeFunctions = functionBlocks.filter(
+		(f) =>
+			f.toLowerCase().includes("realtime.broadcast_changes") ||
+			f.toLowerCase().includes("realtime.send"),
+	);
+	expect(realtimeFunctions.length).toBeGreaterThan(0);
+	// The trigger function should have security definer and search_path
+	const hasSecurityDefiner = realtimeFunctions.some(
+		(f) =>
+			/security\s+definer/.test(f.toLowerCase()) &&
+			/set\s+search_path\s*=\s*''/.test(f.toLowerCase()),
+	);
+	expect(hasSecurityDefiner).toBe(true);
+});
+
+test("RLS policies on realtime.messages", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	const realtimePolicies = policyBlocks.filter((p) =>
+		p.includes("realtime.messages"),
+	);
+	expect(realtimePolicies.length).toBeGreaterThan(0);
+	// At least one policy should target authenticated users
+	const hasAuthPolicy = realtimePolicies.some(
+		(p) => /to\s+authenticated/.test(p) || /auth\.uid\(\)/.test(p),
+	);
+	expect(hasAuthPolicy).toBe(true);
+});
+
+test("realtime policy checks extension column", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+	const realtimePolicies = policyBlocks.filter((p) =>
+		p.includes("realtime.messages"),
+	);
+	// At least one realtime policy should reference the extension column
+	const checksExtension = realtimePolicies.some(
+		(p) =>
+			p.includes("extension") &&
+			(p.includes("broadcast") || p.includes("presence")),
+	);
+	expect(checksExtension).toBe(true);
+});
+
+test("overall quality score", () => {
+	const sql = getMigrationSQL().toLowerCase();
+	const policyBlocks = sql.match(/create\s+policy[\s\S]*?;/gi) ?? [];
+
+	const signals = [
+		// 1. RLS enabled on rooms
+		/alter\s+table[\s\S]*?rooms[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		),
+		// 2. RLS enabled on membership table
+		/alter\s+table[\s\S]*?(room_members|room_users|memberships)[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		),
+		// 3. RLS enabled on content table
+		/alter\s+table[\s\S]*?(content|items|documents|posts|messages)[\s\S]*?enable\s+row\s+level\s+security/.test(
+			sql,
+		),
+		// 4. FK to auth.users with cascade
+		/references\s+auth\.users/.test(sql) && /on\s+delete\s+cascade/.test(sql),
+		// 5. Private schema created
+		/create\s+schema[\s\S]*?private/.test(sql),
+		// 6. security_definer with search_path
+		/security\s+definer/.test(sql) && /set\s+search_path\s*=\s*''/.test(sql),
+		// 7. Subselect auth.uid()
+		/\(\s*select\s+auth\.uid\(\)\s*\)/.test(sql),
+		// 8. TO authenticated on policies
+		policyBlocks.length > 0 &&
+			policyBlocks.filter((p) => !p.includes("realtime.messages")).length > 0 &&
+			policyBlocks
+				.filter((p) => !p.includes("realtime.messages"))
+				.every((p) => /to\s+authenticated/.test(p)),
+		// 9. Indexes on lookup columns
+		/create\s+index/.test(sql),
+		// 10. timestamptz usage (accepts both timestamptz and timestamp with time zone)
+		/timestamptz/.test(sql) || /timestamp\s+with\s+time\s+zone/.test(sql),
+		// 11. IF NOT EXISTS for idempotency
+		/if\s+not\s+exists/.test(sql),
+		// 12. Role-based policies (owner/editor/viewer)
+		sql.includes("owner") && sql.includes("editor") && sql.includes("viewer"),
+		// 13. Realtime publication
+		/alter\s+publication\s+supabase_realtime\s+add\s+table/.test(sql),
+		// 14. Broadcast trigger (broadcast_changes or realtime.send)
+		/realtime\.broadcast_changes/.test(sql) || /realtime\.send/.test(sql),
+		// 15. Trigger creation
+		/create\s+trigger/.test(sql),
+		// 16. RLS on realtime.messages
+		policyBlocks.some((p) => p.includes("realtime.messages")),
+		// 17. Extension check in realtime policy
+		policyBlocks
+			.filter((p) => p.includes("realtime.messages"))
+			.some((p) => p.includes("extension")),
+		// 18. room_id FK on content
+		/room_id[\s\S]*?references[\s\S]*?rooms/.test(sql),
+	];
+	const passed = signals.filter(Boolean).length;
+	expect(passed).toBeGreaterThanOrEqual(13);
+});

packages/evals/evals/collaborative-rooms-realtime/PROMPT.md

@@ -0,0 +1 @@
+Build a collaborative app where users can create rooms (shared spaces for group work), invite other users to join their rooms, and share content within them. Users should only see rooms they've been invited to or created. Room owners can manage members, editors can create and modify content, and viewers can only read. All changes should appear in real-time.

packages/evals/evals/collaborative-rooms-realtime/package.json

@@ -0,0 +1,5 @@
+{
+	"name": "collaborative-rooms-realtime",
+	"private": true,
+	"type": "module"
+}

packages/evals/evals/collaborative-rooms-realtime/supabase/config.toml

@@ -0,0 +1,111 @@
+# For detailed configuration reference documentation, visit:
+# https://supabase.com/docs/guides/local-development/cli/config
+# A string used to distinguish different Supabase projects on the same host. Defaults to the
+# working directory name when running `supabase init`.
+project_id = "collaborative-rooms-realtime"
+
+[api]
+enabled = true
+# Port to use for the API URL.
+port = 54321
+# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
+# endpoints. `public` and `graphql_public` schemas are included by default.
+schemas = ["public", "graphql_public"]
+# Extra schemas to add to the search_path of every request.
+extra_search_path = ["public", "extensions"]
+# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
+# for accidental or malicious requests.
+max_rows = 1000
+
+[db]
+# Port to use for the local database URL.
+port = 54322
+# Port used by db diff command to initialize the shadow database.
+shadow_port = 54320
+# The database major version to use. This has to be the same as your remote database's. Run `SHOW
+# server_version;` on the remote database to check.
+major_version = 17
+
+[db.pooler]
+enabled = false
+# Port to use for the local connection pooler.
+port = 54329
+# Specifies when a server connection can be reused by other clients.
+# Configure one of the supported pooler modes: `transaction`, `session`.
+pool_mode = "transaction"
+# How many server connections to allow per user/database pair.
+default_pool_size = 20
+# Maximum number of client connections allowed.
+max_client_conn = 100
+
+[db.migrations]
+# If disabled, migrations will be skipped during a db push or reset.
+enabled = true
+schema_paths = []
+
+[db.seed]
+# If enabled, seeds the database after migrations during a db reset.
+enabled = true
+# Specifies an ordered list of seed files to load during db reset.
+sql_paths = ["./seed.sql"]
+
+[realtime]
+enabled = true
+
+[studio]
+enabled = true
+# Port to use for Supabase Studio.
+port = 54323
+# External URL of the API server that frontend connects to.
+api_url = "http://127.0.0.1"
+
+[inbucket]
+enabled = true
+# Port to use for the email testing server web interface.
+port = 54324
+
+[storage]
+enabled = true
+# The maximum file size allowed (e.g. "5MB", "500KB").
+file_size_limit = "50MiB"
+
+[auth]
+enabled = true
+# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
+# in emails.
+site_url = "http://127.0.0.1:3000"
+# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
+additional_redirect_urls = ["https://127.0.0.1:3000"]
+# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week).
+jwt_expiry = 3600
+# If disabled, the refresh token will never expire.
+enable_refresh_token_rotation = true
+# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds.
+# Requires enable_refresh_token_rotation = true.
+refresh_token_reuse_interval = 10
+# Allow/disallow new user signups to your project.
+enable_signup = true
+# Allow/disallow anonymous sign-ins to your project.
+enable_anonymous_sign_ins = false
+
+[auth.email]
+# Allow/disallow new user signups via email to your project.
+enable_signup = true
+# If enabled, a user will be required to confirm any email change on both the old, and new email
+# addresses. If disabled, only the new email is required to confirm.
+double_confirm_changes = true
+# If enabled, users need to confirm their email address before signing in.
+enable_confirmations = false
+
+[edge_runtime]
+enabled = true
+# Configure one of the supported request policies: `oneshot`, `per_worker`.
+policy = "per_worker"
+# Port to attach the Chrome inspector for debugging edge functions.
+inspector_port = 8083
+
+[analytics]
+enabled = true
+port = 54327
+# Configure one of the supported backends: `postgres`, `bigquery`.
+backend = "postgres"

packages/evals/scenarios/SCENARIOS.md

@@ -6,4 +6,5 @@
 | 2 | [team-rls-security-definer](team-rls-security-definer.md) | Team-based RLS with security definer helper in a private schema |
 | 3 | [storage-rls-user-folders](storage-rls-user-folders.md) | Storage buckets with RLS policies for user-isolated folders |
 | 4 | [edge-function-hello-world](edge-function-hello-world.md) | Hello-world Edge Function with CORS and shared utilities |
-| 5 | edge-function-stripe-webhook | Stripe webhook Edge Function with signature verification and orders migration |
+| 5 | edge-function-stripe-webhook | Stripe webhook Edge Function with signature verification and orders migration |
+| 6 | [collaborative-rooms-realtime](collaborative-rooms-realtime.md) | Collaborative rooms with role-based RLS, broadcast triggers, and Realtime authorization |

packages/evals/scenarios/collaborative-rooms-realtime.md

@@ -0,0 +1,156 @@
+# Scenario: collaborative-rooms-realtime
+
+## Summary
+
+The agent must create SQL migrations for a collaborative app with rooms (shared
+spaces), membership with roles (owner, editor, viewer), and content sharing.
+The migration must implement role-based RLS policies so users only see rooms
+they belong to, enforce role-specific write permissions, configure Realtime
+(publication + broadcast triggers), and set up RLS on `realtime.messages` so
+only room members receive live updates.
+
+## Real-World Justification
+
+Why this is a common and important workflow:
+
+1. **Room-based collaboration with membership is the canonical Realtime use
+   case** -- The Supabase Realtime documentation uses rooms and room_users
+   as its primary example for authorization policies, and the official
+   `realtime.topic()` function is designed specifically for this pattern.
+   - Source: https://supabase.com/docs/guides/realtime/authorization
+   - Source: https://github.com/orgs/supabase/discussions/22484
+
+2. **Realtime + RLS is the most common source of broken subscriptions** --
+   GitHub issues show dozens of developers hitting CHANNEL_ERROR or silent
+   failures when combining private channels with RLS policies. Getting the
+   `realtime.messages` policies correct (checking `extension`, using
+   `realtime.topic()`, matching private flag) is non-obvious and
+   well-documented as a pain point.
+   - Source: https://github.com/supabase/realtime/issues/1111
+   - Source: https://github.com/supabase/supabase-js/issues/1733
+   - Source: https://github.com/orgs/supabase/discussions/7630
+
+3. **Broadcast from database triggers is the recommended pattern over Postgres
+   Changes** -- The Supabase docs explicitly recommend
+   `realtime.broadcast_changes()` for scalability, since Postgres Changes
+   processes on a single thread and triggers per-subscriber RLS checks. Many
+   developers still use the older pattern and hit scaling walls.
+   - Source: https://supabase.com/docs/guides/realtime/broadcast
+   - Source: https://supabase.com/blog/realtime-broadcast-from-database
+
+4. **Role-based content access with multiple permission levels is a top RBAC
+   question** -- The Supabase RBAC discussion and community guides show
+   developers building exactly this pattern (owner/editor/viewer roles on
+   shared resources) and struggling with correct policy structure.
+   - Source: https://github.com/orgs/supabase/discussions/346
+   - Source: https://supabase.com/docs/guides/database/postgres/custom-claims-and-role-based-access-control-rbac
+   - Source: https://makerkit.dev/blog/tutorials/supabase-rls-best-practices
+
+## Skill References Exercised
+
+Which reference files the agent should consult and what each teaches:
+
+| Reference File | What It Teaches | What the Agent Should Apply |
+|---|---|---|
+| `references/db-rls-mandatory.md` | RLS must be enabled on all public tables | Enable RLS on rooms, room_members, and content tables |
+| `references/db-rls-common-mistakes.md` | Missing TO clause, missing SELECT policy for UPDATE | Always use `TO authenticated`, provide SELECT policies alongside UPDATE |
+| `references/db-rls-performance.md` | Wrap auth.uid() in SELECT, use security_definer for joins | Use `(select auth.uid())` form, helper function for membership lookups |
+| `references/db-security-functions.md` | security_definer in private schema with search_path = '' | Create membership helper in private schema |
+| `references/db-schema-auth-fk.md` | FK to auth.users with ON DELETE CASCADE | Reference auth.users with cascade on room_members |
+| `references/db-schema-timestamps.md` | Use timestamptz not timestamp | All time columns use timestamptz |
+| `references/db-schema-realtime.md` | Tables need PKs for Realtime, enable publication, replica identity | Add table to supabase_realtime publication, set replica identity full |
+| `references/db-perf-indexes.md` | Index columns used in RLS policy lookups | Index user_id and room_id on membership table |
+| `references/db-migrations-idempotent.md` | IF NOT EXISTS for safe reruns | Idempotent DDL throughout |
+| `references/realtime-setup-auth.md` | Private channels require RLS on realtime.messages | Create RLS policies on realtime.messages with extension checks |
+| `references/realtime-broadcast-database.md` | Use realtime.broadcast_changes() from triggers | Create trigger using realtime.broadcast_changes for content changes |
+| `references/realtime-setup-channels.md` | Topic naming convention scope:id:entity | Trigger topic should follow room:{id} pattern |
+
+## Workspace Setup
+
+What the workspace starts with before the agent runs:
+
+- Pre-initialized Supabase project (`supabase/config.toml` exists)
+- Empty `supabase/migrations/` directory
+- The agent creates migration files within this structure
+
+## Agent Task (PROMPT.md draft)
+
+The prompt to give the agent:
+
+> Build a collaborative app where users can create rooms (shared spaces for
+> group work), invite other users to join their rooms, and share content within
+> them. Users should only see rooms they've been invited to or created. Room
+> owners can manage members, editors can create and modify content, and viewers
+> can only read. All changes should appear in real-time.
+
+## Evaluation Criteria
+
+What vitest should assert on the agent's output. Each assertion tests a
+specific quality signal:
+
+| # | Test Name | What It Checks | Quality Dimension |
+|---|-----------|----------------|-------------------|
+| 1 | migration file exists | A `.sql` file exists in `supabase/migrations/` | structure |
+| 2 | creates rooms table | SQL contains `CREATE TABLE` for rooms | correctness |
+| 3 | creates room_members table | SQL contains `CREATE TABLE` for room_members/memberships | correctness |
+| 4 | creates content table | SQL contains `CREATE TABLE` for content/items | correctness |
+| 5 | room_members has role column | The membership table includes a role column (owner/editor/viewer) | correctness |
+| 6 | enables RLS on all tables | `ALTER TABLE ... ENABLE ROW LEVEL SECURITY` for all application tables | security |
+| 7 | FK to auth.users with ON DELETE CASCADE | room_members references `auth.users` with cascade | correctness |
+| 8 | room_id FK on content | content references rooms | correctness |
+| 9 | policies use (select auth.uid()) | Subselect form in all policies referencing auth.uid() | performance |
+| 10 | policies use TO authenticated | All policies scoped to authenticated role | security |
+| 11 | private schema for helper function | A `CREATE SCHEMA ... private` and security_definer helper with `SET search_path = ''` | security |
+| 12 | role-based write policies | Content INSERT/UPDATE restricted to owner/editor roles | security |
+| 13 | viewer read-only enforcement | Viewer role can SELECT but not INSERT/UPDATE/DELETE content | security |
+| 14 | indexes on membership lookup columns | `CREATE INDEX` on user_id and/or room_id in room_members | performance |
+| 15 | uses timestamptz | No plain `timestamp` for time columns | correctness |
+| 16 | idempotent DDL | Uses `IF NOT EXISTS` or `DROP ... IF EXISTS` patterns | idempotency |
+| 17 | realtime publication enabled | `ALTER PUBLICATION supabase_realtime ADD TABLE` for content table | realtime |
+| 18 | broadcast trigger for content changes | A trigger using `realtime.broadcast_changes()` or `realtime.send()` on content | realtime |
+| 19 | trigger function is security definer | The broadcast trigger function uses `SECURITY DEFINER` and `SET search_path = ''` | security |
+| 20 | RLS on realtime.messages | At least one policy on `realtime.messages` for authenticated users | realtime |
+| 21 | realtime policy checks extension | The realtime.messages policy references the `extension` column (broadcast/presence) | realtime |
+| 22 | overall quality score | At least 14 of 18 best-practice signals present | overall |
+
+## Reasoning
+
+Step-by-step reasoning for why this scenario is well-designed:
+
+1. **Baseline differentiator:** An agent without the skill would likely: (a) use
+   Postgres Changes instead of broadcast triggers for realtime, (b) omit RLS on
+   `realtime.messages` entirely, (c) not know about `realtime.topic()` or the
+   `extension` column check, (d) use bare `auth.uid()` instead of the subselect
+   form, (e) put helper functions in the public schema, and (f) forget that
+   broadcast trigger functions need `SECURITY DEFINER` with `SET search_path =
+   ''`. These are all patterns that require reading multiple specific reference
+   files.
+
+2. **Skill value:** This scenario exercises 12+ reference files spanning
+   database, security, and realtime sections. The skill teaches: (a) broadcast
+   triggers over Postgres Changes for scalability, (b) `realtime.messages` RLS
+   with extension checks, (c) private schema for security_definer functions,
+   (d) `(select auth.uid())` caching, (e) role-based policy patterns, and (f)
+   publication configuration. No single reference file is sufficient -- the
+   agent must synthesize knowledge across sections.
+
+3. **Testability:** Every assertion checks for specific SQL patterns via regex.
+   The realtime-specific patterns (publication, broadcast_changes/send, trigger
+   creation, realtime.messages policies, extension column) are all highly
+   distinctive strings that reliably differentiate skill-guided output. The
+   role-based policies (owner/editor/viewer) are checkable via string matching
+   in policy definitions.
+
+4. **Realism:** This is exactly what developers build with Supabase Realtime --
+   collaborative rooms with membership. The Supabase authorization docs use
+   `room_users` as the primary example. GitHub has dozens of issues from
+   developers trying to combine private channels with RLS policies for room-
+   based apps (chat apps, Figma clones, shared whiteboards, collaborative
+   editors).
+
+## Difficulty
+
+**Rating:** HARD
+
+- Without skill: ~25-40% of assertions expected to pass
+- With skill: ~80-90% of assertions expected to pass