# Block aggressive or non-essential bots
User-agent: Barkrowler
Disallow: /

User-agent: Bytespider
Disallow: /

# Allow major crawlers
User-agent: Googlebot
Disallow:

User-agent: YandexBot
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: bingbot
Disallow:

User-agent: PetalBot
Disallow:

# All other bots
User-agent: *
Allow: /
Crawl-delay: 15

# Basic exclusions
Disallow: /cgi-bin/

# Core Drupal folders
Disallow: /.well-known/
Disallow: /core/
Disallow: /vendor/
Disallow: /modules/
Disallow: /profiles/
Disallow: /sites/
Disallow: /themes/
Disallow: /libraries/
Disallow: /tmp/

# Custom project directories
Disallow: /.git/
Disallow: /drush/
Disallow: /patches/

# Sensitive config and root-level files
Disallow: /.htaccess
Disallow: /autoload.php
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /example.gitignore
Disallow: /install.txt
Disallow: /README.md
Disallow: /README.txt
Disallow: /update.php
Disallow: /php.ini
Disallow: /web.config
Disallow: /.user.ini
Disallow: /.editorconfig
Disallow: /.gitattributes
Disallow: /.gitignore

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /media/oembed/
Disallow: /*/media/oembed/

# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/register/
Disallow: /index.php/user/password/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/
Disallow: /index.php/media/oembed/
Disallow: /index.php/*/media/oembed/

# Sitemap
Sitemap: https://texfash.com/sitemap.xml