Cite local businesses in AI search without opening sites to bulk model training

fix.mjs

@@ -19,11 +19,13 @@ import { loadBrand, resolveUserAgent } from './brand.mjs';
 
 const args = process.argv.slice(2);
 const flag = (name, dflt) => { const i = args.indexOf(name); return i !== -1 && args[i + 1] ? args[i + 1] : dflt; };
-const valued = new Set(['--out', '--type', '--brand', '--user-agent']);
+const valued = new Set(['--out', '--type', '--brand', '--user-agent', '--robots-mode']);
 const target = args.find((a, i) => !a.startsWith('--') && !valued.has(args[i - 1]));
-if (!target) { console.error('usage: node fix.mjs <url> --out <dir> [--type auto|person|organization|localbusiness] [--brand <file.json>] [--user-agent <ua>]'); process.exit(1); }
+if (!target) { console.error('usage: node fix.mjs <url> --out <dir> [--type auto|person|organization|localbusiness] [--robots-mode search|all] [--brand <file.json>] [--user-agent <ua>]'); process.exit(1); }
 const TYPE_FLAG = flag('--type', 'auto').toLowerCase();
 if (!['auto', 'person', 'organization', 'localbusiness'].includes(TYPE_FLAG)) { console.error('--type must be auto, person, organization or localbusiness'); process.exit(1); }
+const ROBOTS_MODE = flag('--robots-mode', 'search').toLowerCase();
+if (!['search', 'all'].includes(ROBOTS_MODE)) { console.error('--robots-mode must be search or all'); process.exit(1); }
 
 let brand;
 try { brand = loadBrand(flag('--brand', null)); }
@@ -193,35 +195,69 @@ ${[origin + '/', ...pages.map((p) => p.url)].map((u) => `  <url><loc>${u.replace
 </urlset>
 `;
 
-const robots = `# --- AI agent crawlers (${robotsExists ? 'append to your existing robots.txt' : 'your site has no robots.txt yet: save this as /robots.txt'}) ---
-# Allow = you want these assistants to read and cite your site.
-# Change to Disallow if you want to opt out of that crawler entirely.
-${robotsExists ? '' : '\nUser-agent: *\nAllow: /\n'}
-User-agent: GPTBot
-Allow: /
+const robotsHeader = `# --- AI agent crawlers (${robotsExists ? 'append to your existing robots.txt' : 'your site has no robots.txt yet: save this as /robots.txt'}) ---
+# Mode: ${ROBOTS_MODE === 'search' ? 'search-optimized (citations in ChatGPT, Claude and Perplexity; training scrapers disallowed)' : 'all (both search indexing and AI training allowed)'}
+${robotsExists ? '' : '\nUser-agent: *\nAllow: /\n'}`;
 
+const robotsBody = ROBOTS_MODE === 'search'
+  ? `# 1. AI Search & Citation crawlers — ALLOWED so assistants cite and link your business
 User-agent: OAI-SearchBot
 Allow: /
 
 User-agent: ChatGPT-User
 Allow: /
 
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+# 2. AI Model Training & Bulk Scrapers — DISALLOWED (opt out of training without sacrificing search visibility)
+User-agent: GPTBot
+Disallow: /
+
 User-agent: ClaudeBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+# Google-Extended controls Gemini training/grounding. Google Search / AI Overviews use Googlebot.
+User-agent: Google-Extended
+Disallow: /`
+  : `# AI Search & Citation crawlers
+User-agent: OAI-SearchBot
 Allow: /
 
-User-agent: Claude-User
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: Claude-SearchBot
 Allow: /
 
 User-agent: PerplexityBot
 Allow: /
 
+# AI Model Training & Bulk Scrapers
+User-agent: GPTBot
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
 User-agent: CCBot
 Allow: /
 
 # Google-Extended controls Gemini grounding AND training on your content.
-# Allow = you get cited in Gemini answers. Disallow = you opt out of training.
 User-agent: Google-Extended
-Allow: /
+Allow: /`;
+
+const robots = `${robotsHeader}
+${robotsBody}
 
 ${sitemapExists ? `Sitemap: ${origin}/sitemap.xml` : `# Your site has no sitemap.xml yet. Upload sitemap.xml from this pack, then uncomment:\n# Sitemap: ${origin}/sitemap.xml`}
 `;
@@ -306,6 +342,7 @@ ${llms}\`\`\`
 ## 2. \`robots.txt\` → ${robotsExists ? `append to the existing file at \`${origin}/robots.txt\`` : `create \`${origin}/robots.txt\` (none exists today${robotsRes.ok ? '; that URL currently returns an HTML page' : ''})`}
 
 ${robotsExists ? 'Append, **do not replace** — the existing rules may be load-bearing.' : 'Serve it as `text/plain`.'}
+Mode: **${ROBOTS_MODE === 'search' ? 'Search-first' : 'All crawlers'}** (${ROBOTS_MODE === 'search' ? 'search & citation crawlers allowed; bulk training scrapers disallowed' : 'both search and training crawlers allowed'}). Re-run with \`--robots-mode ${ROBOTS_MODE === 'search' ? 'all' : 'search'}\` to change policy.
 
 \`\`\`
 ${robots}\`\`\`
@@ -346,7 +383,7 @@ writeFileSync(join(outDir, 'FIX.md'), fix);
 
 console.log(`  fix pack → ${outDir}`);
 console.log(`    llms.txt              ${llms.length} bytes, ${pages.length} pages, ${todos} TODOs remaining`);
-console.log(`    robots-ai-block.txt   ${robots.split('\n').length} lines (${robotsExists ? 'append' : 'new file'})`);
+console.log(`    robots-ai-block.txt   ${robots.split('\n').length} lines (${robotsExists ? 'append' : 'new file'}, mode: ${ROBOTS_MODE})`);
 if (!sitemapExists) console.log(`    sitemap.xml           ${pages.length + 1} URLs (site has none)`);
 console.log(`    schema.jsonld.html    ${schemaType} skeleton (${typeWhy})`);
 console.log(`    FIX.md                placement + verification instructions`);

test/scan.test.mjs

@@ -467,3 +467,43 @@ test('batch.mjs --brand: leaderboard branded and method line shows custom user-a
     assert.doesNotMatch(body, /generated by agentready/i);
   } finally { await ok.close(); }
 });
+
+test('fix.mjs --robots-mode: search policy allows search/citation bots and disallows training scrapers; includes Claude-SearchBot', async () => {
+  const home = '<!DOCTYPE html><html><head><title>Acme Co</title><meta name="description" content="Acme makes excellent widgets for everybody."></head><body><h1>Acme Co</h1></body></html>';
+  const s = await serve((req, res) => send(res, 200, 'text/html', home));
+  try {
+    const out = tmp();
+    // 1. Default (search policy)
+    const rDef = await run('fix.mjs', [s.url, '--out', out]);
+    assert.equal(rDef.code, 0, rDef.stderr);
+    const host = new URL(s.url).host;
+    const robotsDef = readFileSync(join(out, host, 'robots-ai-block.txt'), 'utf8');
+
+    // Search & citation crawlers allowed
+    assert.match(robotsDef, /User-agent: Claude-SearchBot\s*\nAllow: \//, 'Claude-SearchBot must be allowed for search');
+    assert.match(robotsDef, /User-agent: OAI-SearchBot\s*\nAllow: \//, 'OAI-SearchBot must be allowed for search');
+    assert.match(robotsDef, /User-agent: PerplexityBot\s*\nAllow: \//, 'PerplexityBot must be allowed for search');
+    assert.match(robotsDef, /User-agent: ChatGPT-User\s*\nAllow: \//, 'ChatGPT-User must be allowed for user browsing');
+
+    // Training / bulk scrapers disallowed in search mode
+    assert.match(robotsDef, /User-agent: GPTBot\s*\nDisallow: \//, 'GPTBot training scraper must be disallowed in search mode');
+    assert.match(robotsDef, /User-agent: ClaudeBot\s*\nDisallow: \//, 'ClaudeBot training scraper must be disallowed in search mode');
+    assert.match(robotsDef, /User-agent: CCBot\s*\nDisallow: \//, 'CCBot bulk scraper must be disallowed in search mode');
+    assert.match(robotsDef, /User-agent: Google-Extended\s*\nDisallow: \//, 'Google-Extended must be disallowed in search mode');
+
+    // 2. Explicit --robots-mode all
+    const outAll = tmp();
+    const rAll = await run('fix.mjs', [s.url, '--out', outAll, '--robots-mode', 'all']);
+    assert.equal(rAll.code, 0, rAll.stderr);
+    const robotsAll = readFileSync(join(outAll, host, 'robots-ai-block.txt'), 'utf8');
+    assert.match(robotsAll, /User-agent: Claude-SearchBot\s*\nAllow: \//);
+    assert.match(robotsAll, /User-agent: GPTBot\s*\nAllow: \//);
+    assert.match(robotsAll, /User-agent: ClaudeBot\s*\nAllow: \//);
+    assert.match(robotsAll, /User-agent: CCBot\s*\nAllow: \//);
+
+    // 3. Invalid --robots-mode
+    const rBad = await run('fix.mjs', [s.url, '--out', tmp(), '--robots-mode', 'invalid']);
+    assert.equal(rBad.code, 1);
+    assert.match(rBad.stderr, /--robots-mode must be search or all/);
+  } finally { await s.close(); }
+});