Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,8 @@ Skillcheck treats a skill like a drug trial treats a drug:
`HELPS` (CI fully above zero), `HARMS` (fully below), or `PLACEBO` (overlaps zero).

Every run is fresh: tasks and outputs are generated anew each time and `check`
stores nothing locally, so a repeated check is an independent measurement. Full
stores nothing locally (aside from checkpoints retained after an interrupted
run for `--resume`), so a repeated check is an independent measurement. Full
methodology in [`METHODOLOGY.md`](METHODOLOGY.md).

## Architecture
Expand Down Expand Up @@ -271,8 +272,7 @@ skillcheck rot [--results dir] [--output report.json] # detect
skillcheck --version
```

Accepted inputs: any Markdown (`.md`) file — `SKILL.md`, `AGENTS.md`, `CLAUDE.md`,
or any other `.md` — or a folder containing one. Headings are used as the declared domain
Accepted inputs: any Markdown (`.md`, `.mdc`) or `.cursorrules` file (`SKILL.md`, `AGENTS.md`, `CLAUDE.md`, `.cursorrules`, `.cursor/rules/*.mdc`, or any other `.md`), or a folder containing one. Headings are used as the declared domain
when front matter is absent; if a heading is itself an instruction, pass `--domain "<neutral topic>"`
for a blind run. `--tasks` is capped at 50 and `--trials` at 10; mistyped options are
rejected rather than silently ignored.
Expand Down
10 changes: 8 additions & 2 deletions packages/cli/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -418,15 +418,21 @@ export function parseCheckOptions(argv: string[], startIndex = 3): CheckOptions
inputIndex
);

const json = hasFlag(argv, '--json');
const markdown = hasFlag(argv, '--markdown');
if (json && markdown) {
throw new Error('Cannot combine --markdown and --json. Choose one output format.');
}

return {
evalOptions: {
inputPath,
...evalOptions,
resume: hasFlag(argv, '--resume'),
saveArtifacts: Boolean(evalOptions.output)
},
json: hasFlag(argv, '--json'),
markdown: hasFlag(argv, '--markdown'),
json,
markdown,
output: evalOptions.output,
clipboard: hasFlag(argv, '--clipboard'),
inspect: hasFlag(argv, '--inspect'),
Expand Down
25 changes: 17 additions & 8 deletions packages/cli/src/normalize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@ function formatForFile(filePath: string): SkillFormat | undefined {
if (known) {
return known.format;
}
const ext = path.extname(basename).toLowerCase();
if (ext === '.mdc') {
return 'mdc';
}
// Any Markdown file is accepted as a generic skill document.
if (path.extname(basename).toLowerCase() === '.md') {
if (ext === '.md') {
return 'markdown';
}
return undefined;
Expand All @@ -40,11 +44,16 @@ async function firstMarkdownInDir(dirPath: string): Promise<{ filePath: string;

const entries = await readdir(dirPath, { withFileTypes: true });
const markdown = entries
.filter((entry) => entry.isFile() && path.extname(entry.name).toLowerCase() === '.md')
.filter((entry) => {
if (!entry.isFile()) return false;
const ext = path.extname(entry.name).toLowerCase();
return ext === '.md' || ext === '.mdc';
})
.map((entry) => entry.name)
.sort();
if (markdown[0]) {
return { filePath: path.join(dirPath, markdown[0]), format: 'markdown' };
const filePath = path.join(dirPath, markdown[0]);
return { filePath, format: formatForFile(filePath) ?? 'markdown' };
}
return undefined;
}
Expand All @@ -54,14 +63,14 @@ async function resolveSkillFile(inputPath: string): Promise<{ filePath: string;
if (stats.isDirectory()) {
const found = await firstMarkdownInDir(inputPath);
if (!found) {
throw new Error(`No .md file found in ${inputPath}. Skillcheck only analyzes Markdown (.md) skill files.`);
throw new Error(`No .md file found in ${inputPath}. Skillcheck only analyzes Markdown (.md, .mdc) and .cursorrules skill files.`);
}
return found;
}

const format = formatForFile(inputPath);
if (!format) {
throw new Error(`Skillcheck only analyzes Markdown (.md) files. "${path.basename(inputPath)}" is not a .md file.`);
throw new Error(`Skillcheck only analyzes Markdown (.md, .mdc) and .cursorrules files. "${path.basename(inputPath)}" is not supported.`);
}
return { filePath: inputPath, format };
}
Expand Down Expand Up @@ -142,7 +151,7 @@ function isGenericHeading(heading: string, fileName?: string): boolean {
}
if (fileName) {
const fileNorm = fileName.trim().toLowerCase();
const fileBase = fileNorm.replace(/\.md$/, '');
const fileBase = fileNorm.replace(/\.(md|mdc)$/, '');
if (norm === fileNorm || norm === fileBase) {
return true;
}
Expand Down Expand Up @@ -210,10 +219,10 @@ async function listAssets(skillFilePath: string): Promise<string[]> {

function nameFromPath(filePath: string, format: SkillFormat): string {
const base = path.basename(filePath);
if (format === 'markdown') {
if (format === 'markdown' || format === 'mdc') {
// A user-named file (e.g. frontend-design.md) is more meaningful than its
// parent folder — turn "frontend-design.md" into "frontend design".
const stem = base.replace(/\.md$/i, '').replace(/[-_]+/g, ' ').trim();
const stem = base.replace(/\.(md|mdc)$/i, '').replace(/[-_]+/g, ' ').trim();
return stem || path.basename(path.dirname(filePath)) || base;
}
// Conventional files (SKILL.md, AGENTS.md, …) take their identity from the folder.
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export type SkillFormat = 'SKILL.md' | 'AGENTS.md' | '.cursorrules' | 'CLAUDE.md' | 'markdown';
export type SkillFormat = 'SKILL.md' | 'AGENTS.md' | '.cursorrules' | 'CLAUDE.md' | 'markdown' | 'mdc';
export type CriterionType = 'rubric' | 'deterministic';

// Task-generation difficulty: harder levels ask the generator for edge
Expand Down
20 changes: 15 additions & 5 deletions packages/cli/src/ui/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,19 @@ function excerpt(prompt: string, max = 80): string {
return single.length > max ? `${single.slice(0, max - 1).trimEnd()}…` : single;
}

// A literal `|` would split a GFM table cell, so escape it.
function escapeCell(text: string): string {
return text.replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
// Escapes untrusted text so it cannot ping users, break out of headings,
// or inject links, HTML, code spans, or table columns in GitHub-Flavored Markdown.
export function escapeMarkdown(text: string): string {
return text
.replace(/\r?\n/g, ' ')
.replace(/\\/g, '\\\\')
.replace(/@/g, '@​')
.replace(/\[/g, '\\[')
.replace(/\]/g, '\\]')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/`/g, '\\`')
.replace(/\|/g, '\\|');
}

/**
Expand All @@ -33,7 +43,7 @@ export function formatMarkdownReport(result: EvalResult): string {
const [ciLow, ciHigh] = outcome.ci_pp;

const lines: string[] = [
`# Skillcheck Evaluation: ${skill.name}`,
`# Skillcheck Evaluation: ${escapeMarkdown(skill.name)}`,
'',
`## ${VERDICT_BADGE[outcome.verdict]}`,
''
Expand Down Expand Up @@ -71,7 +81,7 @@ export function formatMarkdownReport(result: EvalResult): string {
for (const task of result.tasks) {
const delta = (task.with_skill_pass_rate - task.no_skill_pass_rate) * 100;
lines.push(
`| ${escapeCell(task.id)} | ${escapeCell(excerpt(task.prompt))} | ${pct(task.with_skill_pass_rate)} | ${pct(task.no_skill_pass_rate)} | ${pp(delta)} |`
`| ${escapeMarkdown(task.id)} | ${escapeMarkdown(excerpt(task.prompt))} | ${pct(task.with_skill_pass_rate)} | ${pct(task.no_skill_pass_rate)} | ${pp(delta)} |`
);
}
lines.push('');
Expand Down
19 changes: 13 additions & 6 deletions packages/cli/src/ui/picker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,15 @@ import readline from 'node:readline';
import { SYM, padDisplay, paint, truncateDisplay } from './theme.js';
import { bannerLines, printBanner } from './banner.js';

const CONVENTIONAL_SKILL_FILES = ['SKILL.md', 'AGENTS.md', 'CLAUDE.md'];
const isMarkdownFile = (filePath: string): boolean => path.extname(filePath).toLowerCase() === '.md';
const CONVENTIONAL_SKILL_FILES = ['SKILL.md', 'AGENTS.md', 'CLAUDE.md', '.cursorrules'];
const isMarkdownFile = (filePath: string): boolean => {
const base = path.basename(filePath);
if (base === '.cursorrules') {
return true;
}
const ext = path.extname(filePath).toLowerCase();
return ext === '.md' || ext === '.mdc';
};

export class CancelledError extends Error {
readonly exitCode = 130;
Expand All @@ -31,7 +38,7 @@ export interface Keypress {
}

export function supportedSkillFilesText(): string {
return `a Markdown (.md) file (e.g. ${CONVENTIONAL_SKILL_FILES.join(', ')}), or a folder containing one`;
return `a Markdown (.md, .mdc) or .cursorrules file (e.g. ${CONVENTIONAL_SKILL_FILES.join(', ')}), or a folder containing one`;
}

export async function directoryHasMarkdown(dirPath: string): Promise<boolean> {
Expand All @@ -51,13 +58,13 @@ export async function validateSkillInput(inputPath: string): Promise<string> {
if (await directoryHasMarkdown(resolved)) {
return inputPath;
}
throw new Error(`That folder has no .md file. Skillcheck only checks Markdown (.md) files — open a folder that contains one, or pick a .md file directly.`);
throw new Error(`That folder has no supported skill file (.md, .mdc, .cursorrules). Skillcheck only checks Markdown (.md, .mdc) and .cursorrules files — open a folder that contains one, or pick a file directly.`);
}
if (stats.isFile()) {
if (isMarkdownFile(resolved)) {
return inputPath;
}
throw new Error(`Skillcheck only checks Markdown (.md) files. "${path.basename(resolved)}" is not a .md file.`);
throw new Error(`Skillcheck only checks Markdown (.md, .mdc) and .cursorrules files. "${path.basename(resolved)}" is not a supported file.`);
}
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
Expand Down Expand Up @@ -328,7 +335,7 @@ export async function selectSkillPath(startDir = process.cwd()): Promise<string>
chosen = entry.fullPath;
break;
}
message = 'We only check Markdown (.md) files. Open a folder to find one, or pick a .md file.';
message = 'We only check Markdown (.md, .mdc) and .cursorrules files. Open a folder to find one, or pick a supported file.';
}
}, { altScreen: true });

Expand Down
29 changes: 28 additions & 1 deletion packages/cli/test/cli.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { mkdtemp, writeFile } from 'node:fs/promises';
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
import { evalResultFixture } from './eval-result-fixture.js';
import { tmpdir } from 'node:os';
import path from 'node:path';
Expand Down Expand Up @@ -74,6 +74,12 @@ describe('friendly CLI check command', () => {
expect(withoutFlag.markdown).toBe(false);
});

it('rejects combining --markdown and --json with a clear error', () => {
expect(() =>
parseCheckOptions(['node', 'skillcheck', 'check', './SKILL.md', '--markdown', '--json'])
).toThrow(/Cannot combine --markdown and --json/);
});

it('accepts options before or after the skill path', () => {
const optsBefore = parseCheckOptions(['node', 'skillcheck', '--explain', './SKILL.md', '--tasks', '5'], 2);
expect(optsBefore.evalOptions.inputPath).toBe('./SKILL.md');
Expand Down Expand Up @@ -198,6 +204,27 @@ describe('friendly CLI check command', () => {
await expect(validateSkillInput(md)).resolves.toBe(md);
});

it('accepts .cursorrules and .cursor/rules/*.mdc files', async () => {
const dir = await mkdtemp(path.join(tmpdir(), 'skillcheck-validate-'));
const cursorrules = path.join(dir, '.cursorrules');
await writeFile(cursorrules, '# Cursor rules\n\nAlways write tests.\n');
await expect(validateSkillInput(cursorrules)).resolves.toBe(cursorrules);

const rulesDir = path.join(dir, '.cursor', 'rules');
await mkdir(rulesDir, { recursive: true });
const mdc = path.join(rulesDir, 'api.mdc');
await writeFile(mdc, '---\ndescription: API rules\n---\n# API rules\n\nFollow REST.\n');
await expect(validateSkillInput(mdc)).resolves.toBe(mdc);

const cursorrulesDir = await mkdtemp(path.join(tmpdir(), 'skillcheck-dir-cursorrules-'));
await writeFile(path.join(cursorrulesDir, '.cursorrules'), '# Rules\n');
await expect(validateSkillInput(cursorrulesDir)).resolves.toBe(cursorrulesDir);

const mdcDir = await mkdtemp(path.join(tmpdir(), 'skillcheck-dir-mdc-'));
await writeFile(path.join(mdcDir, 'rule.mdc'), '# MDC Rule\n');
await expect(validateSkillInput(mdcDir)).resolves.toBe(mdcDir);
});

it('rejects files that are not Markdown', async () => {
const dir = await mkdtemp(path.join(tmpdir(), 'skillcheck-validate-'));
const txt = path.join(dir, 'notes.txt');
Expand Down
16 changes: 16 additions & 0 deletions packages/cli/test/normalize.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,22 @@ describe('normalizeSkill', () => {
expect(skill.domain).toBe('React accessibility reviews');
});

it('normalizes .cursor/rules/*.mdc by path and folder', async () => {
const dir = await mkdtemp(path.join(tmpdir(), 'skillcheck-normalize-'));
const rulesDir = path.join(dir, '.cursor', 'rules');
await mkdir(rulesDir, { recursive: true });
const file = path.join(rulesDir, 'api.mdc');
await writeFile(file, '---\ndescription: API documentation editing\n---\n# API Guide\n\nUse precise docs language.\n');

const skill = await normalizeSkill(file);
expect(skill.format).toBe('mdc');
expect(skill.name).toBe('API Guide');
expect(skill.domain).toBe('API documentation editing');

const dirSkill = await normalizeSkill(rulesDir);
expect(dirSkill.format).toBe('mdc');
});

it('normalizes CLAUDE.md for awesome-claude-md corpus entries', async () => {
const dir = await mkdtemp(path.join(tmpdir(), 'skillcheck-normalize-'));
await writeFile(path.join(dir, 'CLAUDE.md'), '# Next.js Rules\n\ndescription: Next.js app development\n');
Expand Down
37 changes: 37 additions & 0 deletions packages/cli/test/ui-print.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,41 @@ describe('formatMarkdownReport', () => {
expect(report).not.toContain(':warning:');
expect(report).not.toContain('Small sample');
});

it('escapes user mentions, links, HTML tags, backticks, pipes, and newlines in skill name', () => {
const report = formatMarkdownReport(evalResultFixture({
skill: {
name: 'Docs @octocat [claim your prize](http://evil.example) <script> `test` | other\nheading'
}
}));

expect(report).toContain('# Skillcheck Evaluation: Docs @​octocat \\[claim your prize\\](http://evil.example) &lt;script&gt; \\`test\\` \\| other heading');
expect(report).not.toContain('@octocat');
expect(report).not.toContain('[claim your prize]');
expect(report).not.toContain('<script>');
});

it('escapes user mentions, links, HTML tags, backticks, pipes, and newlines in task prompt excerpts', () => {
const report = formatMarkdownReport(evalResultFixture({
tasks: [
{
id: 't001',
prompt: 'Review @octocat [bad link](http://evil.example) <danger> `drop db` | choice\nnext line',
criterion_type: 'rubric',
criterion: 'Picks correctly',
with_skill_pass_rate: 1,
no_skill_pass_rate: 0
}
]
}));

expect(report).toContain('@​octocat');
expect(report).toContain('\\[bad link\\]');
expect(report).toContain('&lt;danger&gt;');
expect(report).toContain('\\`drop db\\`');
expect(report).toContain('\\| choice');
expect(report).not.toContain('@octocat');
expect(report).not.toContain('[bad link]');
expect(report).not.toContain('<danger>');
});
});
Loading