diff --git a/srv/nginx/robots.txt b/srv/nginx/robots.txt index 9ad0312..f7c6e2b 100644 --- a/srv/nginx/robots.txt +++ b/srv/nginx/robots.txt @@ -32,6 +32,8 @@ User-agent: ChatGPT-User User-agent: Claude-Web User-agent: ClaudeBot User-agent: cohere-ai +User-agent: cohere-training-data-crawler +User-agent: Crawlspace User-agent: Diffbot User-agent: DuckAssistBot User-agent: FacebookBot @@ -56,6 +58,8 @@ User-agent: PanguBot User-agent: PerplexityBot User-agent: PetalBot User-agent: Scrapy +User-agent: SemrushBot-OCOB +User-agent: SemrushBot-SWA User-agent: Sidetrade indexer bot User-agent: Timpibot User-agent: VelenPublicWebCrawler