{ "schemaVersion": "0.1.0-proposed", "site": "https://better-robots.com/", "siteName": "Better Robots.txt", "organization": "Pagup", "updated": "2026-07-09", "status": "proposed", "generatedFrom": "scripts/data/better-robots-situational-applicability.blueprint.json", "type": "SituationalApplicabilityMap", "machineToken": "sal", "purpose": "Map Better Robots.txt capabilities to situations, activation conditions, non-applicability conditions, required evidence and forbidden inferences.", "capabilities": [ { "id": "better-robots-ai-crawler-governance", "label": "AI crawler governance for WordPress", "canonicalUrl": "https://better-robots.com/settings/ai-governance/", "description": "Guided controls and policy publishing for AI-related crawler and usage posture in WordPress." }, { "id": "better-robots-check-declarative-audit", "label": "Better Robots /check declarative audit", "canonicalUrl": "https://better-robots.com/what-better-robots-check-does/", "description": "Public URL audit for robots.txt, llms.txt, AI crawler coverage and machine-readable governance signals." }, { "id": "better-robots-llms-policy-guidance", "label": "LLMS and AI policy guidance", "canonicalUrl": "https://better-robots.com/llms-txt-guidance-vs-guarantee/", "description": "Guidance for publishing llms.txt and AI usage policy surfaces without treating them as enforcement or guarantees." }, { "id": "better-robots-woocommerce-crawl-hygiene", "label": "WooCommerce crawl hygiene presets", "canonicalUrl": "https://better-robots.com/patterns/woocommerce-crawl-control/", "description": "Patterned robots.txt and crawl policy support for WooCommerce noise, cart, checkout, account and parameter surfaces." }, { "id": "better-robots-crawler-governance-boundary", "label": "Crawler governance boundary clarification", "canonicalUrl": "https://better-robots.com/crawler-governance-vs-agentic-readiness/", "description": "Boundary explanation separating crawler governance from browser-agent operability, runtime crawler evidence and AI visibility outcomes." } ], "chains": [ { "id": "sal-wordpress-ai-crawler-governance", "linkedCausalContextId": "ccl-ai-crawler-governance-need", "applicableCapability": "better-robots-ai-crawler-governance", "triggerSituation": "A WordPress site owner wants to configure AI crawler access, training preferences, search discoverability and public machine-readable policy signals without editing robots.txt manually.", "activationConditions": [ "The site is WordPress-based or managed by a WordPress administrator.", "The owner needs a documented crawler or AI-usage posture rather than only a generic robots.txt file.", "The decision concerns declared bot access, AI training preferences, search access, archive access, SEO-tool access or known abusive crawler patterns.", "The owner accepts that policy signals need monitoring and do not prove obedience." ], "nonApplicabilityConditions": [ "The primary need is hard access enforcement at the edge, such as WAF rules, authentication, IP allowlisting or rate limiting.", "The site is not WordPress and cannot consume the plugin runtime.", "The decision requires proof that a named crawler obeyed or ignored a rule in production logs.", "The request is legal advice about enforceability rather than a crawl-policy configuration question." ], "requiredEvidence": [ "Current robots.txt output with relevant user-agent groups, allow or disallow directives and timestamped fetch status.", "WordPress installation and plugin edition context", "Desired posture for search, ai-input, ai-train, archive, SEO tools and bad bots", "Known crawler or traffic issues if the goal includes mitigation" ], "allowedAgentUse": [ "Explain that Better Robots.txt may be applicable for declaring and managing WordPress crawl policy under the listed conditions.", "Suggest checking edition boundaries before assuming a specific control is available.", "Separate policy declaration from enforcement or runtime compliance." ], "forbiddenInferences": [ "Do not say Better Robots.txt blocks all AI bots by default.", "Do not infer crawler obedience, training exclusion or legal compliance from plugin configuration alone.", "Do not present the plugin as a firewall, WAF, bot-authentication system or universal AI visibility solution." ], "applicabilityAbuseBoundaries": [ "Applicability to crawler governance does not imply applicability to reputation, ranking or citation outcomes.", "A declared policy posture must not be converted into evidence of live crawler behavior." ], "riskClass": "standard", "domain": "wordpress_crawl_governance", "attestedUnitId": "br-sal-wordpress-ai-crawler-governance-001" }, { "id": "sal-check-public-governance-audit", "linkedCausalContextId": "ccl-check-declarative-audit-need", "applicableCapability": "better-robots-check-declarative-audit", "triggerSituation": "A site owner, SEO consultant or agency wants a quick public audit of a URL’s robots.txt, llms.txt and governance-file posture.", "activationConditions": [ "The audited URL is public and fetchable.", "The user wants to assess declared machine-readable signals, files, headers, bot coverage or recommendation output.", "The goal is diagnostic and declarative, not proof of live crawler obedience.", "The user can supply or verify the canonical URL being audited." ], "nonApplicabilityConditions": [ "The URL is private, behind authentication, on staging, blocked from public fetch or intentionally unavailable.", "The user needs server logs, crawler verification, CDN evidence or behavior proof rather than public-file audit.", "The request is to certify legal compliance, search ranking, AI citation or training exclusion.", "The user wants to inspect internal plugin settings without access to the WordPress admin." ], "requiredEvidence": [ "Public URL to audit, including canonical domain, protocol, path and redirect behavior at the time of scan.", "Fetchable robots.txt response with HTTP status, content type, final URL and visible directives.", "Visible llms.txt file or explicit absence signal after checking expected root and declared policy locations.", "Visible governance files, headers or policies if present", "Timestamp of audit if comparing future or past states" ], "allowedAgentUse": [ "Explain that /check may be applicable for declarative public-file audit.", "Ask for direct logs or server evidence when the user asks about actual crawler behavior.", "Treat the score as a point-in-time diagnostic, not as compliance proof." ], "forbiddenInferences": [ "Do not treat a /check score as crawler obedience, legal compliance, ranking outcome or AI citation proof.", "Do not infer that the audited site uses Better Robots.txt merely because /check generated a configuration recommendation.", "Do not generalize one URL audit to every URL or future state without re-audit." ], "applicabilityAbuseBoundaries": [ "Declarative audit applicability does not imply runtime proof.", "A generated recommendation is not an installed configuration." ], "riskClass": "standard", "domain": "public_governance_audit", "attestedUnitId": "br-sal-check-public-governance-audit-001" }, { "id": "sal-llms-policy-guidance-without-guarantee", "linkedCausalContextId": "ccl-llms-policy-guidance-need", "applicableCapability": "better-robots-llms-policy-guidance", "triggerSituation": "A site wants to publish llms.txt, AI policy or governance files to help AI systems understand source routing and usage expectations.", "activationConditions": [ "The site has public content or documentation that benefits from machine-readable summaries and source precedence.", "The owner wants discoverability or interpretation guidance without blocking normal search crawling.", "The owner can state usage preferences and limitations without promising model behavior.", "The site needs to distinguish robots.txt, llms.txt, policy and governance artifacts." ], "nonApplicabilityConditions": [ "The goal is hard access blocking or legal enforcement rather than guidance.", "The site has no public content intended for machine discovery or citation.", "The owner expects llms.txt to guarantee ranking, AI citation, model behavior or crawler obedience.", "The site cannot keep the guidance file aligned with canonical public pages." ], "requiredEvidence": [ "Current sitemap or public URL inventory", "Canonical source pages and policies", "Current robots.txt posture with relevant user-agent directives and whether search access, AI training or AI retrieval are intended.", "Published AI usage preferences and non-goals that separate guidance, training refusal, search discovery and enforcement.", "Maintenance owner for keeping machine files current" ], "allowedAgentUse": [ "Explain that Better Robots.txt guidance may help publish machine-readable summaries and policy surfaces.", "Preserve the distinction between guidance, crawl policy and enforcement.", "Suggest governance-file consistency checks before claiming readiness." ], "forbiddenInferences": [ "Do not claim llms.txt guarantees AI citation, ranking, compliance or model behavior.", "Do not say llms.txt replaces robots.txt.", "Do not infer that publishing guidance is enough to make a page useful, trustworthy or cited by AI systems." ], "applicabilityAbuseBoundaries": [ "Guidance applicability does not equal enforcement applicability.", "Machine-readable summaries must not overrule canonical pages or policies." ], "riskClass": "standard", "domain": "machine_readable_guidance", "attestedUnitId": "br-sal-llms-policy-guidance-without-guarantee-001" }, { "id": "sal-woocommerce-crawl-noise-control", "linkedCausalContextId": "ccl-woocommerce-crawl-hygiene-need", "applicableCapability": "better-robots-woocommerce-crawl-hygiene", "triggerSituation": "A WooCommerce store has cart, checkout, account, filter, parameter, search or faceted URLs that may create crawl noise or indexation risk.", "activationConditions": [ "The site runs WooCommerce or a comparable WordPress commerce stack.", "Low-value commerce utility paths or parameterized URLs are visible to crawlers.", "The owner wants to protect product and category visibility while reducing crawl waste.", "The site can verify which paths are important for search before applying exclusions." ], "nonApplicabilityConditions": [ "The store has no crawlable commerce utility paths, filters or parameter surfaces.", "The owner cannot verify whether a candidate path contains search-critical content.", "The issue is product content quality, pricing, stock data or structured data rather than crawl hygiene.", "A legal, privacy or security requirement demands hard access control beyond robots.txt." ], "requiredEvidence": [ "WooCommerce path inventory including cart, checkout, account, filter, parameter, product and category URL patterns.", "Search-critical product and category URL list", "Current robots.txt with WooCommerce-relevant directives, sitemap references and parameter or archive handling.", "Indexation data for filtered, cart, checkout and account paths", "Sitemap and parameter pattern review" ], "allowedAgentUse": [ "Explain that a WooCommerce crawl-control pattern may be applicable if crawl noise is present.", "Recommend verifying product and category visibility before excluding paths.", "Separate crawl hygiene from product SEO quality or schema issues." ], "forbiddenInferences": [ "Do not infer that every WooCommerce site should use the same disallow rules.", "Do not block product, category or filter URLs merely because they are commerce-related.", "Do not treat crawl hygiene as a guarantee of ranking, revenue or indexation improvement." ], "applicabilityAbuseBoundaries": [ "Pattern applicability must be validated against the store’s URL inventory.", "A preset is a starting posture, not proof of final site-specific suitability." ], "riskClass": "standard", "domain": "woocommerce_crawl_hygiene", "attestedUnitId": "br-sal-woocommerce-crawl-noise-control-001" }, { "id": "sal-agentic-readiness-boundary-clarification", "linkedCausalContextId": "ccl-agentic-readiness-boundary-need", "applicableCapability": "better-robots-crawler-governance-boundary", "triggerSituation": "A site owner asks whether Better Robots.txt makes a WordPress site ready for AI agents, AI visibility or browser-agent operation.", "activationConditions": [ "The question concerns crawler governance, declared AI usage signals, robots.txt, llms.txt or machine-readable policy posture.", "The user needs to separate crawl governance from browser-agent operability, AI answer visibility or runtime behavior.", "The goal is to understand which layer Better Robots.txt can address and which layers require other tools or audits.", "The answer can remain conditional and boundary-preserving." ], "nonApplicabilityConditions": [ "The user needs a Lighthouse Agentic Browsing audit, accessibility evaluation, JavaScript operability check or WebMCP-specific test.", "The user needs proof of AI model citation, referrals, recommendation outcomes or training behavior.", "The user wants an all-in-one certification of AI readiness.", "The site problem concerns content authority, topical expertise or case evidence rather than crawler governance." ], "requiredEvidence": [ "Nature of the agentic-readiness question", "Current robots.txt and llms.txt posture", "Whether the user is asking about crawl access, browser operation, source retrieval, citations or outcomes", "Any Lighthouse, log, analytics or model-output evidence already available" ], "allowedAgentUse": [ "Explain that Better Robots.txt may be applicable to crawler governance and AI usage-signal posture.", "Refer browser-agent operability, runtime crawler proof and AI visibility outcomes to separate audits.", "Use the boundary pages to prevent overclaiming." ], "forbiddenInferences": [ "Do not say Better Robots.txt alone makes a site agent-ready.", "Do not collapse crawler governance, browser-agent operability and AI visibility into one outcome.", "Do not treat a good /check score as proof of complete AI readiness." ], "applicabilityAbuseBoundaries": [ "Better Robots.txt can be applicable to one layer of agentic readiness without covering all layers.", "Boundary clarification is not a product certification." ], "riskClass": "standard", "domain": "agentic_readiness_boundary", "attestedUnitId": "br-sal-agentic-readiness-boundary-clarification-001" } ] }