# # robots.txt for QICDRC - Qatar International Court and Dispute Resolution Centre # # Managed by the RobotsTxt module at /admin/config/search/robotstxt. # Editing the robots.txt file in the webroot has no effect while that module # is enabled - and the physical file has been renamed to robots.txt.disabled # so the module's content is what actually gets served. # User-agent: * # Crawl-delay is respected by Bing and Yandex, ignored by Google. Crawl-delay: 10 # Core Drupal paths. Disallow: /core/ Disallow: /profiles/ Disallow: /README.md Disallow: /INSTALL.txt Disallow: /LICENSE.txt Disallow: /example.gitignore Disallow: /modules/README.txt Disallow: /sites/README.txt Disallow: /themes/README.txt Disallow: /update.php Disallow: /web.config # Authentication and account management. Disallow: /admin/ Disallow: /user/ Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /user/logout # Node action paths. Disallow: /node/add/ Disallow: /*/delete Disallow: /*/edit Disallow: /*/revisions Disallow: /*/devel # Query strings that generate unbounded duplicate URLs. Disallow: /*?destination= Disallow: /*?page=*& Disallow: /*&page= Disallow: /*?sort= Disallow: /*&sort= Disallow: /*?order= Disallow: /*&order= Disallow: /*?search_api_fulltext= Disallow: /index.php/ # Transactional flows - must not be indexed or shared. Disallow: /webform-payment/ Disallow: /payment/return Disallow: /payment/success Disallow: /payment/failure Disallow: /payment/webhook # Chat endpoints. Disallow: /qicdrc-chat/ Disallow: /customgpt-chat/ # Private events are not for public discovery. Disallow: /private-events/ # Let crawlers fetch the assets needed to render pages. Blocking these causes # "page cannot be rendered" warnings in Search Console. Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /themes/ Allow: /modules/ Allow: /sites/default/files/ # # Answer engines / AI crawlers. # # These are explicitly allowed. QICDRC's judgements, rules and guidance are # public legal information, and being cited by ChatGPT, Perplexity, Claude, # Gemini and Copilot is the point of the AEO work. # # Change this at /admin/config/search/fuego-seo. # User-agent: GPTBot Disallow: User-agent: OAI-SearchBot Disallow: User-agent: ChatGPT-User Disallow: User-agent: ClaudeBot Disallow: User-agent: Claude-User Disallow: User-agent: PerplexityBot Disallow: User-agent: Google-Extended Disallow: User-agent: Applebot-Extended Disallow: User-agent: CCBot Disallow: User-agent: Bingbot Disallow: # Sitemaps Sitemap: https://www.qicdrc.gov.qa/sitemap.xml