From 0f0305614b05c8774cf719d12f1567ec3d2403d3 Mon Sep 17 00:00:00 2001
From: Tommy <contact@tommytran.io>
Date: Mon, 10 Feb 2025 04:59:59 -0700
Subject: [PATCH] Update robots.txt Matches
 https://github.com/ai-robots-txt/ai.robots.txt/blob/bebffccc0ced8c420276c93f3109c2e71cd5ca0c/robots.txt

Signed-off-by: Tommy <contact@tommytran.io>
---
 srv/nginx/robots.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/srv/nginx/robots.txt b/srv/nginx/robots.txt
index 9ad0312..f7c6e2b 100644
--- a/srv/nginx/robots.txt
+++ b/srv/nginx/robots.txt
@@ -32,6 +32,8 @@ User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: cohere-ai
+User-agent: cohere-training-data-crawler
+User-agent: Crawlspace
 User-agent: Diffbot
 User-agent: DuckAssistBot
 User-agent: FacebookBot
@@ -56,6 +58,8 @@ User-agent: PanguBot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
+User-agent: SemrushBot-OCOB
+User-agent: SemrushBot-SWA
 User-agent: Sidetrade indexer bot
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler