diff --git a/fix.mjs b/fix.mjs index 933fbdf..d835553 100644 --- a/fix.mjs +++ b/fix.mjs @@ -19,11 +19,13 @@ import { loadBrand, resolveUserAgent } from './brand.mjs'; const args = process.argv.slice(2); const flag = (name, dflt) => { const i = args.indexOf(name); return i !== -1 && args[i + 1] ? args[i + 1] : dflt; }; -const valued = new Set(['--out', '--type', '--brand', '--user-agent']); +const valued = new Set(['--out', '--type', '--brand', '--user-agent', '--robots-mode']); const target = args.find((a, i) => !a.startsWith('--') && !valued.has(args[i - 1])); -if (!target) { console.error('usage: node fix.mjs --out [--type auto|person|organization|localbusiness] [--brand ] [--user-agent ]'); process.exit(1); } +if (!target) { console.error('usage: node fix.mjs --out [--type auto|person|organization|localbusiness] [--robots-mode search|all] [--brand ] [--user-agent ]'); process.exit(1); } const TYPE_FLAG = flag('--type', 'auto').toLowerCase(); if (!['auto', 'person', 'organization', 'localbusiness'].includes(TYPE_FLAG)) { console.error('--type must be auto, person, organization or localbusiness'); process.exit(1); } +const ROBOTS_MODE = flag('--robots-mode', 'search').toLowerCase(); +if (!['search', 'all'].includes(ROBOTS_MODE)) { console.error('--robots-mode must be search or all'); process.exit(1); } let brand; try { brand = loadBrand(flag('--brand', null)); } @@ -193,35 +195,69 @@ ${[origin + '/', ...pages.map((p) => p.url)].map((u) => ` ${u.replace `; -const robots = `# --- AI agent crawlers (${robotsExists ? 'append to your existing robots.txt' : 'your site has no robots.txt yet: save this as /robots.txt'}) --- -# Allow = you want these assistants to read and cite your site. -# Change to Disallow if you want to opt out of that crawler entirely. -${robotsExists ? '' : '\nUser-agent: *\nAllow: /\n'} -User-agent: GPTBot -Allow: / +const robotsHeader = `# --- AI agent crawlers (${robotsExists ? 'append to your existing robots.txt' : 'your site has no robots.txt yet: save this as /robots.txt'}) --- +# Mode: ${ROBOTS_MODE === 'search' ? 'search-optimized (citations in ChatGPT, Claude and Perplexity; training scrapers disallowed)' : 'all (both search indexing and AI training allowed)'} +${robotsExists ? '' : '\nUser-agent: *\nAllow: /\n'}`; +const robotsBody = ROBOTS_MODE === 'search' + ? `# 1. AI Search & Citation crawlers — ALLOWED so assistants cite and link your business User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / +User-agent: Claude-SearchBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +# 2. AI Model Training & Bulk Scrapers — DISALLOWED (opt out of training without sacrificing search visibility) +User-agent: GPTBot +Disallow: / + User-agent: ClaudeBot +Disallow: / + +User-agent: CCBot +Disallow: / + +# Google-Extended controls Gemini training/grounding. Google Search / AI Overviews use Googlebot. +User-agent: Google-Extended +Disallow: /` + : `# AI Search & Citation crawlers +User-agent: OAI-SearchBot Allow: / -User-agent: Claude-User +User-agent: ChatGPT-User +Allow: / + +User-agent: Claude-SearchBot Allow: / User-agent: PerplexityBot Allow: / +# AI Model Training & Bulk Scrapers +User-agent: GPTBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-User +Allow: / + User-agent: CCBot Allow: / # Google-Extended controls Gemini grounding AND training on your content. -# Allow = you get cited in Gemini answers. Disallow = you opt out of training. User-agent: Google-Extended -Allow: / +Allow: /`; + +const robots = `${robotsHeader} +${robotsBody} ${sitemapExists ? `Sitemap: ${origin}/sitemap.xml` : `# Your site has no sitemap.xml yet. Upload sitemap.xml from this pack, then uncomment:\n# Sitemap: ${origin}/sitemap.xml`} `; @@ -306,6 +342,7 @@ ${llms}\`\`\` ## 2. \`robots.txt\` → ${robotsExists ? `append to the existing file at \`${origin}/robots.txt\`` : `create \`${origin}/robots.txt\` (none exists today${robotsRes.ok ? '; that URL currently returns an HTML page' : ''})`} ${robotsExists ? 'Append, **do not replace** — the existing rules may be load-bearing.' : 'Serve it as `text/plain`.'} +Mode: **${ROBOTS_MODE === 'search' ? 'Search-first' : 'All crawlers'}** (${ROBOTS_MODE === 'search' ? 'search & citation crawlers allowed; bulk training scrapers disallowed' : 'both search and training crawlers allowed'}). Re-run with \`--robots-mode ${ROBOTS_MODE === 'search' ? 'all' : 'search'}\` to change policy. \`\`\` ${robots}\`\`\` @@ -346,7 +383,7 @@ writeFileSync(join(outDir, 'FIX.md'), fix); console.log(` fix pack → ${outDir}`); console.log(` llms.txt ${llms.length} bytes, ${pages.length} pages, ${todos} TODOs remaining`); -console.log(` robots-ai-block.txt ${robots.split('\n').length} lines (${robotsExists ? 'append' : 'new file'})`); +console.log(` robots-ai-block.txt ${robots.split('\n').length} lines (${robotsExists ? 'append' : 'new file'}, mode: ${ROBOTS_MODE})`); if (!sitemapExists) console.log(` sitemap.xml ${pages.length + 1} URLs (site has none)`); console.log(` schema.jsonld.html ${schemaType} skeleton (${typeWhy})`); console.log(` FIX.md placement + verification instructions`); diff --git a/test/scan.test.mjs b/test/scan.test.mjs index eb5569b..4df5766 100644 --- a/test/scan.test.mjs +++ b/test/scan.test.mjs @@ -467,3 +467,43 @@ test('batch.mjs --brand: leaderboard branded and method line shows custom user-a assert.doesNotMatch(body, /generated by agentready/i); } finally { await ok.close(); } }); + +test('fix.mjs --robots-mode: search policy allows search/citation bots and disallows training scrapers; includes Claude-SearchBot', async () => { + const home = 'Acme Co

Acme Co

'; + const s = await serve((req, res) => send(res, 200, 'text/html', home)); + try { + const out = tmp(); + // 1. Default (search policy) + const rDef = await run('fix.mjs', [s.url, '--out', out]); + assert.equal(rDef.code, 0, rDef.stderr); + const host = new URL(s.url).host; + const robotsDef = readFileSync(join(out, host, 'robots-ai-block.txt'), 'utf8'); + + // Search & citation crawlers allowed + assert.match(robotsDef, /User-agent: Claude-SearchBot\s*\nAllow: \//, 'Claude-SearchBot must be allowed for search'); + assert.match(robotsDef, /User-agent: OAI-SearchBot\s*\nAllow: \//, 'OAI-SearchBot must be allowed for search'); + assert.match(robotsDef, /User-agent: PerplexityBot\s*\nAllow: \//, 'PerplexityBot must be allowed for search'); + assert.match(robotsDef, /User-agent: ChatGPT-User\s*\nAllow: \//, 'ChatGPT-User must be allowed for user browsing'); + + // Training / bulk scrapers disallowed in search mode + assert.match(robotsDef, /User-agent: GPTBot\s*\nDisallow: \//, 'GPTBot training scraper must be disallowed in search mode'); + assert.match(robotsDef, /User-agent: ClaudeBot\s*\nDisallow: \//, 'ClaudeBot training scraper must be disallowed in search mode'); + assert.match(robotsDef, /User-agent: CCBot\s*\nDisallow: \//, 'CCBot bulk scraper must be disallowed in search mode'); + assert.match(robotsDef, /User-agent: Google-Extended\s*\nDisallow: \//, 'Google-Extended must be disallowed in search mode'); + + // 2. Explicit --robots-mode all + const outAll = tmp(); + const rAll = await run('fix.mjs', [s.url, '--out', outAll, '--robots-mode', 'all']); + assert.equal(rAll.code, 0, rAll.stderr); + const robotsAll = readFileSync(join(outAll, host, 'robots-ai-block.txt'), 'utf8'); + assert.match(robotsAll, /User-agent: Claude-SearchBot\s*\nAllow: \//); + assert.match(robotsAll, /User-agent: GPTBot\s*\nAllow: \//); + assert.match(robotsAll, /User-agent: ClaudeBot\s*\nAllow: \//); + assert.match(robotsAll, /User-agent: CCBot\s*\nAllow: \//); + + // 3. Invalid --robots-mode + const rBad = await run('fix.mjs', [s.url, '--out', tmp(), '--robots-mode', 'invalid']); + assert.equal(rBad.code, 1); + assert.match(rBad.stderr, /--robots-mode must be search or all/); + } finally { await s.close(); } +});