User-agent: meta-externalagent Disallow: /c/ # Disallow a specific directory User-agent: facebookexternalhit Disallow: /c/ # Disallow a specific directory User-agent: GPTBot Disallow: / User-agent: SeekportBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: barkrowler Disallow: / User-agent: dotbot Disallow: / User-agent: GeedoBot Disallow: / User-agent: PetalBot Disallow: / User-agent: AwarioBot Disallow: / User-agent: YandexBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: Baiduspider Disallow: / User-agent: Pinterestbot Disallow: / User-agent: * Crawl-delay: 15