From 879b25072566c6a717042f6b1fb107071dbd8323 Mon Sep 17 00:00:00 2001 From: nico Date: Thu, 13 Feb 2025 11:27:54 +0100 Subject: [PATCH] feat(robots.txt): add new bots --- public/robots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/robots.txt b/public/robots.txt index e003819..2f11bf6 100755 --- a/public/robots.txt +++ b/public/robots.txt @@ -36,7 +36,8 @@ User-agent: PanguBot User-agent: PerplexityBot User-agent: PetalBot User-agent: Scrapy -User-agent: SemrushBot +User-agent: SemrushBot-OCOB +User-agent: SemrushBot-SWA User-agent: Sidetrade indexer bot User-agent: Timpibot User-agent: VelenPublicWebCrawler