Update robots.txt

srv/nginx/robots.txt

@@ -8,10 +8,9 @@ Disallow: /
 Allow: /ads.txt
 Allow: /app-ads.txt
 
-User-agent: peer39_crawler
 User-agent: peer39_crawler/1.0
 User-agent: TurnitinBot
-User-agent: NPBot
+User-agent: AcademicBotRTU
 User-agent: SlySearch
 User-agent: BLEXBot
 User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html)
@@ -24,6 +23,7 @@ User-agent: MJ12bot
 User-agent: AI2Bot
 User-agent: Ai2Bot-Dolma
 User-agent: Amazonbot
+User-agent: anthropic-ai
 User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
@@ -31,21 +31,28 @@ User-agent: CCBot
 User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
+User-agent: cohere-ai
 User-agent: Diffbot
+User-agent: DuckAssistBot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
-User-agent: GPTBot
 User-agent: Google-Extended
 User-agent: GoogleOther
 User-agent: GoogleOther-Image
 User-agent: GoogleOther-Video
+User-agent: GPTBot
+User-agent: iaskspider/2.0
 User-agent: ICC-Crawler
-User-agent: ISSCyberRiskCrawler
 User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
 User-agent: Kangaroo Bot
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
 User-agent: OAI-SearchBot
+User-agent: omgili
+User-agent: omgilibot
+User-agent: PanguBot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
@@ -54,11 +61,4 @@ User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended
 User-agent: YouBot
-User-agent: anthropic-ai
-User-agent: cohere-ai
-User-agent: facebookexternalhit
-User-agent: iaskspider/2.0
-User-agent: img2dataset
-User-agent: omgili
-User-agent: omgilibot
 Disallow: /