# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

User-agent: *
Allow: /

# Note: We rely on meta "noindex" tags (served via index.php) for auth/admin routes
# to ensure they are properly removed from search indexes rather than just blocked.

# `Disallow: /api/` REMOVED 2026-09-01 — real bug found via Google Search
# Console's Live Test (Ryan): every guide/news page came back "Page
# cannot be indexed: Soft 404", and the resource list named the actual
# cause directly -- "Googlebot blocked by robots.txt" on
# /api/guides/get.php, /api/classes/list.php, /api/banners/active.php,
# /api/auth/me.php, etc. This site is a fully client-rendered SPA: every
# page's real content comes from a client-side fetch to these exact
# endpoints AFTER the HTML loads. Disallowing /api/ was meant to keep
# raw JSON responses out of the index (never a real risk -- JSON isn't
# indexed as a page anyway, it has no title/HTML), but it had a much
# bigger, unintended effect: Googlebot's renderer refuses to make any
# fetch a robots.txt rule disallows, so it could never actually see a
# guide, filter, news article, or anything else built from an API
# response -- every JS-rendered page looked broken/empty to Google
# specifically, while a real browser (or Google's own initial HTML
# fetch, which only sees the pre-rendered meta tags) never showed a
# problem. Real access control was never robots.txt's job anyway --
# every endpoint gates on its own Auth/isAdmin() checks server-side
# regardless of what robots.txt says, so removing this doesn't change
# what's actually protected.

# Sitemap
Sitemap: https://diablofilter.com/sitemap.xml