{"name":"Foundable crawler access audit","version":"2026-06-05","officialDomain":"foundable.now","siteUrl":"https://foundable.now","canonicalUrl":"https://foundable.now/visibility/crawler-access.json","wellKnownUrl":"https://foundable.now/.well-known/foundable-crawler-access.json","status":"pass","issueCount":0,"issues":[],"outcome":"Keep Foundable public SEO, GEO, AEO, social previews, and machine-readable discovery files reachable to search crawlers, answer-engine crawlers, and preview fetchers while app-only URLs return noindex and sensitive token/API surfaces stay blocked.","passCriteria":["Provider registry user agents match the robots AI/search and social-preview crawler user-agent lists exactly.","OpenAI, Anthropic, Perplexity, Mistral, Google, Bing, DuckDuckGo, You.com, Amazon, Common Crawl, Apple, LinkedIn, X, Slack, Meta, and Discord crawler source URLs are documented.","AI-facing discovery files include identity JSON, llms files, authority evidence, brand SERP, monitoring, indexing, crawler-access, and feed references.","Social-preview crawlers for LinkedIn, X, Slack, Meta, and Discord stay in the machine-readable audit so edge allowlists can be checked after social-card changes.","Authenticated workspace UI, login, and signup URLs stay crawlable enough to receive X-Robots noindex instead of becoming blocked-but-indexed Search Console entries.","Public compliance pages plus machine-readable llms, identity, visibility, and well-known discovery files stay reachable but return X-Robots noindex headers when they should not compete in search results.","OAuth, invite, iMessage, investor-room, dev, and API routes stay disallowed because those URLs can carry sensitive state or are not useful page resources.","Generated render assets and public support assets stay crawl-fetchable for page rendering and browser identity but return X-Robots noindex headers so JS, font, and manifest URLs do not become search results.","The crawler access audit is itself listed in indexing discovery files and exposed through a well-known alias."],"crawlerProviders":[{"id":"openai","label":"OpenAI ChatGPT search, user actions, and discovery","sourceUrl":"https://platform.openai.com/docs/bots","purpose":"Keep Foundable eligible for ChatGPT Search crawl, user-requested fetches, and OpenAI web discovery when users ask idea-to-revenue questions.","userAgents":["OAI-SearchBot","ChatGPT-User","GPTBot"]},{"id":"anthropic","label":"Anthropic Claude search, user actions, and discovery","sourceUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","purpose":"Keep Foundable reachable for Claude search indexing, user-requested web fetches, and Anthropic discovery crawls.","userAgents":["Claude-SearchBot","Claude-User","ClaudeBot"]},{"id":"perplexity","label":"Perplexity answer search and user-requested fetches","sourceUrl":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","purpose":"Keep Foundable reachable when Perplexity answers cite public sources or users request current startup-building guidance.","userAgents":["PerplexityBot","Perplexity-User"]},{"id":"mistral","label":"Mistral Le Chat search indexing and user-requested fetches","sourceUrl":"https://docs.mistral.ai/robots","purpose":"Keep Foundable reachable when Le Chat indexes the web for search answers or fetches current pages on a user's behalf.","userAgents":["MistralAI-User","MistralAI-Index"]},{"id":"google","label":"Google Search, Search Console inspection, Gemini grounding, logo, favicon, image, and video discovery","sourceUrl":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","purpose":"Keep Foundable public pages, current favicon, logo, preview assets, Search Console inspection, and Gemini/Vertex grounding controls crawlable across Google Search and AI discovery surfaces.","userAgents":["Googlebot","Googlebot-Image","Googlebot-Video","Google-InspectionTool","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-CloudVertexBot","Google-Extended"]},{"id":"bing","label":"Bing Search and Microsoft answer surfaces","sourceUrl":"https://www.bing.com/webmasters/help/Verify-Bingbot-2195837f","purpose":"Keep Foundable eligible for Bing indexing, branded result inspection, IndexNow follow-up, and Microsoft answer surfaces.","userAgents":["Bingbot"]},{"id":"duckduckgo-search","label":"DuckDuckGo Search","sourceUrl":"https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/","purpose":"Keep Foundable eligible for DuckDuckGo organic discovery and private search result indexing.","userAgents":["DuckDuckBot"]},{"id":"duckduckgo-ai","label":"DuckDuckGo AI-assisted answers","sourceUrl":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/","purpose":"Keep Foundable eligible as a cited public source in DuckDuckGo AI-assisted answers.","userAgents":["DuckAssistBot"]},{"id":"youcom","label":"You.com search and answer surfaces","sourceUrl":"https://you.com/docs/youbot","purpose":"Keep Foundable eligible for You.com search indexing, real-time answer retrieval, and citation surfaces.","userAgents":["YouBot"]},{"id":"amazon","label":"Amazon Alexa, Rufus, and search answer surfaces","sourceUrl":"https://developer.amazon.com/amazonbot","purpose":"Keep Foundable eligible for Amazon search experiences, Alexa live answers, and Rufus-style assistant retrieval.","userAgents":["Amazonbot","Amzn-SearchBot","Amzn-User"]},{"id":"commoncrawl","label":"Common Crawl open web corpus","sourceUrl":"https://commoncrawl.org/ccbot","purpose":"Keep Foundable public identity and answer pages eligible for the open web corpus used by researchers, search systems, and downstream AI datasets.","userAgents":["CCBot"]},{"id":"apple","label":"Apple Search, Spotlight, Siri, Safari suggestions, and Apple Intelligence","sourceUrl":"https://support.apple.com/en-us/119829","purpose":"Keep Foundable eligible for Applebot discovery in Spotlight, Siri, Safari suggestions, and Apple Intelligence model-quality controls.","userAgents":["Applebot","Applebot-Extended"]}],"socialPreviewCrawlerProviders":[{"id":"linkedin","label":"LinkedIn link previews","sourceUrl":"https://radar.cloudflare.com/bots/directory/linkedin","purpose":"Keep Foundable previews, F-mark imagery, and founder-facing authority pages visible when links are shared on LinkedIn.","userAgents":["LinkedInBot"]},{"id":"x","label":"X / Twitter Cards","sourceUrl":"https://developer.x.com/cards/overview","purpose":"Keep Foundable's X cards and launch-share previews eligible for title, description, and image refreshes.","userAgents":["Twitterbot"]},{"id":"slack","label":"Slack link unfurls","sourceUrl":"https://api.slack.com/robots","purpose":"Keep Foundable page metadata and OG images unfurling correctly when users share pages in Slack workspaces.","userAgents":["Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot"]},{"id":"meta","label":"Meta link previews","sourceUrl":"https://radar.cloudflare.com/bots/directory/facebook","purpose":"Keep Foundable Open Graph previews available across Facebook, Instagram, Messenger, and related Meta sharing surfaces.","userAgents":["facebookexternalhit"]},{"id":"discord","label":"Discord embeds","sourceUrl":"https://radar.cloudflare.com/bots/directory/discord-bot","purpose":"Keep Foundable title, description, and image embeds available when links are shared in Discord communities.","userAgents":["Discordbot"]}],"userAgents":["OAI-SearchBot","ChatGPT-User","GPTBot","Claude-SearchBot","Claude-User","ClaudeBot","PerplexityBot","Perplexity-User","MistralAI-User","MistralAI-Index","Googlebot","Googlebot-Image","Googlebot-Video","Google-InspectionTool","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-CloudVertexBot","Google-Extended","Bingbot","DuckDuckBot","DuckAssistBot","YouBot","Amazonbot","Amzn-SearchBot","Amzn-User","CCBot","Applebot","Applebot-Extended","LinkedInBot","Twitterbot","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","facebookexternalhit","Discordbot"],"aiSearchUserAgents":["OAI-SearchBot","ChatGPT-User","GPTBot","Claude-SearchBot","Claude-User","ClaudeBot","PerplexityBot","Perplexity-User","MistralAI-User","MistralAI-Index","Googlebot","Googlebot-Image","Googlebot-Video","Google-InspectionTool","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-CloudVertexBot","Google-Extended","Bingbot","DuckDuckBot","DuckAssistBot","YouBot","Amazonbot","Amzn-SearchBot","Amzn-User","CCBot","Applebot","Applebot-Extended"],"socialPreviewUserAgents":["LinkedInBot","Twitterbot","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","facebookexternalhit","Discordbot"],"requiredUserAgents":["OAI-SearchBot","ChatGPT-User","GPTBot","Claude-SearchBot","Claude-User","ClaudeBot","PerplexityBot","Perplexity-User","MistralAI-User","MistralAI-Index","Googlebot","Googlebot-Image","Googlebot-Video","Google-InspectionTool","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-CloudVertexBot","Google-Extended","Bingbot","DuckDuckBot","DuckAssistBot","YouBot","Amazonbot","Amzn-SearchBot","Amzn-User","CCBot","Applebot","Applebot-Extended"],"requiredSocialPreviewUserAgents":["LinkedInBot","Twitterbot","Slackbot-LinkExpanding","Slack-ImgProxy","Slackbot","facebookexternalhit","Discordbot"],"publicDiscoveryPaths":["/","/foundable","/official-foundable","/foundable-vs-foundable-ai","/foundable-vs-fundable","/foundable-vs-tryfundable","/build","/grow","/llms.txt","/llms-full.txt","/foundable.json","/.well-known/foundable.json","/.well-known/llms.txt","/.well-known/llms-full.txt","/visibility/canonical-answer-coverage.json","/visibility/canonical-answers.json","/visibility/ad-experiment-log.csv","/visibility/ad-landing-matrix.json","/visibility/ai-search-prompts.json","/visibility/authority-evidence.json","/visibility/brand-serp.json","/visibility/crawler-access.json","/visibility/indexing-runbook.json","/visibility/performance-budget.json","/visibility/search-result-experiments.json","/visibility/structured-data.json","/visibility/search-monitoring.json","/.well-known/foundable-ad-experiment-log.csv","/.well-known/foundable-ad-landing-matrix.json","/.well-known/foundable-answer-coverage.json","/.well-known/foundable-answers.json","/.well-known/foundable-authority-evidence.json","/.well-known/foundable-brand-serp.json","/.well-known/foundable-crawler-access.json","/.well-known/foundable-indexing-runbook.json","/.well-known/foundable-performance-budget.json","/.well-known/foundable-search-result-experiments.json","/.well-known/foundable-visibility.json","/.well-known/foundable-structured-data.json","/.well-known/foundable-search-monitoring.json","/feed.xml"],"publicDiscoveryUrls":["https://foundable.now/","https://foundable.now/foundable","https://foundable.now/official-foundable","https://foundable.now/foundable-vs-foundable-ai","https://foundable.now/foundable-vs-fundable","https://foundable.now/foundable-vs-tryfundable","https://foundable.now/build","https://foundable.now/grow","https://foundable.now/llms.txt","https://foundable.now/llms-full.txt","https://foundable.now/foundable.json","https://foundable.now/.well-known/foundable.json","https://foundable.now/.well-known/llms.txt","https://foundable.now/.well-known/llms-full.txt","https://foundable.now/visibility/canonical-answer-coverage.json","https://foundable.now/visibility/canonical-answers.json","https://foundable.now/visibility/ad-experiment-log.csv","https://foundable.now/visibility/ad-landing-matrix.json","https://foundable.now/visibility/ai-search-prompts.json","https://foundable.now/visibility/authority-evidence.json","https://foundable.now/visibility/brand-serp.json","https://foundable.now/visibility/crawler-access.json","https://foundable.now/visibility/indexing-runbook.json","https://foundable.now/visibility/performance-budget.json","https://foundable.now/visibility/search-result-experiments.json","https://foundable.now/visibility/structured-data.json","https://foundable.now/visibility/search-monitoring.json","https://foundable.now/.well-known/foundable-ad-experiment-log.csv","https://foundable.now/.well-known/foundable-ad-landing-matrix.json","https://foundable.now/.well-known/foundable-answer-coverage.json","https://foundable.now/.well-known/foundable-answers.json","https://foundable.now/.well-known/foundable-authority-evidence.json","https://foundable.now/.well-known/foundable-brand-serp.json","https://foundable.now/.well-known/foundable-crawler-access.json","https://foundable.now/.well-known/foundable-indexing-runbook.json","https://foundable.now/.well-known/foundable-performance-budget.json","https://foundable.now/.well-known/foundable-search-result-experiments.json","https://foundable.now/.well-known/foundable-visibility.json","https://foundable.now/.well-known/foundable-structured-data.json","https://foundable.now/.well-known/foundable-search-monitoring.json","https://foundable.now/feed.xml"],"requiredDiscoveryPaths":["/","/foundable","/official-foundable","/foundable-vs-foundable-ai","/foundable-vs-fundable","/foundable-vs-tryfundable","/build","/grow","/llms.txt","/llms-full.txt","/foundable.json","/.well-known/foundable.json","/visibility/authority-evidence.json","/.well-known/foundable-authority-evidence.json","/visibility/brand-serp.json","/.well-known/foundable-brand-serp.json","/visibility/crawler-access.json","/.well-known/foundable-crawler-access.json","/visibility/ai-search-prompts.json","/visibility/indexing-runbook.json","/visibility/search-monitoring.json","/feed.xml"],"privateDisallowPaths":["/auth/","/imessage/","/investor-room/","/invite/","/dev/","/api/"],"requiredPrivateDisallowPaths":["/api/","/auth/","/imessage/","/investor-room/","/invite/","/dev/"],"crawlableNoIndexPolicy":{"xRobotsTag":"noindex, nofollow, noarchive","headerRoutes":["/account","/account/:path*","/admin","/admin/:path*","/chat","/chat/:path*","/companies/:path*","/dashboard","/dashboard/:path*","/design","/design/:path*","/directory","/directory/:path*","/finder","/finder/:path*","/forgot-password","/growth","/growth/:path*","/home","/home/:path*","/inbox","/inbox/:path*","/login","/mail","/mail/:path*","/marketplace","/marketplace/:path*","/new","/new/:path*","/onboarding","/onboarding/:path*","/payments","/payments/:path*","/reset-password","/revenue","/revenue/:path*","/schedule","/schedule/:path*","/scheduled","/scheduled/:path*","/settings","/settings/:path*","/signup","/api/:path*","/auth/:path*","/dev/:path*","/imessage/:path*","/investor-room","/investor-room/:path*","/invite/:path*"],"crawlableHeaderRoutes":["/account","/account/:path*","/admin","/admin/:path*","/chat","/chat/:path*","/companies/:path*","/dashboard","/dashboard/:path*","/design","/design/:path*","/directory","/directory/:path*","/finder","/finder/:path*","/forgot-password","/growth","/growth/:path*","/home","/home/:path*","/inbox","/inbox/:path*","/login","/mail","/mail/:path*","/marketplace","/marketplace/:path*","/new","/new/:path*","/onboarding","/onboarding/:path*","/payments","/payments/:path*","/reset-password","/revenue","/revenue/:path*","/schedule","/schedule/:path*","/scheduled","/scheduled/:path*","/settings","/settings/:path*","/signup"],"requiredCrawlableHeaderRoutes":["/account","/account/:path*","/admin","/admin/:path*","/chat","/chat/:path*","/companies/:path*","/dashboard","/dashboard/:path*","/finder","/finder/:path*","/forgot-password","/growth","/growth/:path*","/home","/home/:path*","/login","/mail","/mail/:path*","/payments","/payments/:path*","/reset-password","/revenue","/revenue/:path*","/schedule","/schedule/:path*","/settings","/settings/:path*","/signup"],"robotsDisallowConflicts":[],"expectedGscOutcome":"Authenticated UI and auth URLs should leave crawled/indexed buckets after Google recrawls and sees X-Robots noindex; they should not appear as indexed though blocked by robots.txt."},"publicNoIndexPolicy":{"xRobotsTag":"noindex, noarchive","headerRoutes":["/sitemap","/sitemap/:path*","/data-deletion","/cookies","/dpa","/llms.txt","/llms-full.txt","/foundable.json","/.well-known/llms.txt","/.well-known/llms-full.txt","/visibility/:path*","/.well-known/foundable.json","/.well-known/foundable-ad-experiment-log.csv","/.well-known/foundable-ad-landing-matrix.json","/.well-known/foundable-answer-coverage.json","/.well-known/foundable-answers.json","/.well-known/foundable-authority-evidence.json","/.well-known/foundable-brand-serp.json","/.well-known/foundable-crawler-access.json","/.well-known/foundable-indexing-runbook.json","/.well-known/foundable-performance-budget.json","/.well-known/foundable-search-monitoring.json","/.well-known/foundable-search-result-experiments.json","/.well-known/foundable-structured-data.json","/.well-known/foundable-visibility.json"],"requiredHeaderRoutes":["/data-deletion","/cookies","/dpa","/llms.txt","/llms-full.txt","/foundable.json","/.well-known/llms.txt","/.well-known/llms-full.txt","/visibility/:path*","/.well-known/foundable.json","/.well-known/foundable-ad-experiment-log.csv","/.well-known/foundable-ad-landing-matrix.json","/.well-known/foundable-answer-coverage.json","/.well-known/foundable-answers.json","/.well-known/foundable-authority-evidence.json","/.well-known/foundable-brand-serp.json","/.well-known/foundable-crawler-access.json","/.well-known/foundable-indexing-runbook.json","/.well-known/foundable-performance-budget.json","/.well-known/foundable-search-monitoring.json","/.well-known/foundable-search-result-experiments.json","/.well-known/foundable-structured-data.json","/.well-known/foundable-visibility.json"],"robotsDisallowConflicts":[],"expectedGscOutcome":"Public compliance pages and machine-readable discovery files should stay fetchable for reviewers, crawlers, and answer engines but leave search result impressions after Google recrawls the X-Robots noindex header."},"assetIndexingPolicy":{"xRobotsTag":"noindex, noarchive","headerRoutes":["/_next/static/:path*","/fonts/:path*","/manifest.webmanifest"],"requiredHeaderRoutes":["/_next/static/:path*","/fonts/:path*","/manifest.webmanifest"],"fetchRoots":["/_next/static/","/fonts/","/manifest.webmanifest"],"robotsDisallowConflicts":[],"rationale":"Search crawlers need JS, font, and manifest assets to render public pages and identify the site, but Search Console should see those asset URLs as non-indexable resources rather than page candidates."},"indexingDiscoveryUrls":["https://foundable.now/sitemap.xml","https://foundable.now/robots.txt","https://foundable.now/foundable.json","https://foundable.now/.well-known/foundable.json","https://foundable.now/llms.txt","https://foundable.now/.well-known/llms.txt","https://foundable.now/llms-full.txt","https://foundable.now/.well-known/llms-full.txt","https://foundable.now/feed.xml","https://foundable.now/visibility/ai-search-prompts.json","https://foundable.now/.well-known/foundable-visibility.json","https://foundable.now/visibility/canonical-answers.json","https://foundable.now/.well-known/foundable-answers.json","https://foundable.now/visibility/canonical-answer-coverage.json","https://foundable.now/.well-known/foundable-answer-coverage.json","https://foundable.now/visibility/brand-serp.json","https://foundable.now/.well-known/foundable-brand-serp.json","https://foundable.now/visibility/authority-evidence.json","https://foundable.now/.well-known/foundable-authority-evidence.json","https://foundable.now/visibility/structured-data.json","https://foundable.now/.well-known/foundable-structured-data.json","https://foundable.now/visibility/crawler-access.json","https://foundable.now/.well-known/foundable-crawler-access.json","https://foundable.now/visibility/search-monitoring.json","https://foundable.now/.well-known/foundable-search-monitoring.json","https://foundable.now/visibility/search-result-experiments.json","https://foundable.now/.well-known/foundable-search-result-experiments.json","https://foundable.now/visibility/indexing-runbook.json","https://foundable.now/.well-known/foundable-indexing-runbook.json","https://foundable.now/visibility/performance-budget.json","https://foundable.now/.well-known/foundable-performance-budget.json","https://foundable.now/visibility/ad-landing-matrix.json","https://foundable.now/.well-known/foundable-ad-landing-matrix.json","https://foundable.now/visibility/ad-experiment-log.csv","https://foundable.now/.well-known/foundable-ad-experiment-log.csv"],"validationCommands":["bun run seo:crawler-access -- --json --fail-on-issue","bun --cwd ui vitest run src/__tests__/public-seo-visibility-reports.test.ts","bun run --filter '@foundable/ui' test public-seo proxy"],"relatedReferences":["https://foundable.now/robots.txt","https://foundable.now/sitemap.xml","https://foundable.now/foundable.json","https://foundable.now/.well-known/foundable.json","https://foundable.now/llms.txt","https://foundable.now/llms-full.txt","https://foundable.now/visibility/authority-evidence.json","https://foundable.now/.well-known/foundable-authority-evidence.json","https://foundable.now/visibility/brand-serp.json","https://foundable.now/.well-known/foundable-brand-serp.json","https://foundable.now/visibility/search-monitoring.json","https://foundable.now/.well-known/foundable-search-monitoring.json","https://foundable.now/visibility/indexing-runbook.json","https://foundable.now/.well-known/foundable-indexing-runbook.json","https://foundable.now/visibility/crawler-access.json","https://foundable.now/.well-known/foundable-crawler-access.json"]}