diff options
| author | Leitwolf11 | 2026-11-05 06:50:56 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-11-05 06:50:56 +0300 |
| commit | 67423dfb0d867326c4e6f1f1f0d4ee3495837258 (patch) | |
| tree | 2ab2db624c0bc400600fd1b5790ba21904889ba1 | |
| parent | 9b4cc75dade5d9ab7e03284d52916bb2a6fddf36 (diff) | |
| download | website-67423dfb0d867326c4e6f1f1f0d4ee3495837258.tar.gz | |
Use the crawler rules of progmasoft.com on tala.runseo/robots-txt
tala.run and its talac, tazel, talafmt and talalint hosts take the
rules progmasoft.com and tazel.net use: Googlebot, Google-Extended,
OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User and
ClaudeBot are allowed; GPTBot, CCBot and Bytespider are turned away;
every other crawler is allowed. The rules for PerplexityBot,
Perplexity-User, meta-externalagent, meta-externalfetcher, FacebookBot,
DuckAssistBot, Amazonbot, Google-CloudVertexBot and MistralAI-User go,
so they fall under the rule for every crawler. Each host keeps its own
sitemap.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
| -rw-r--r-- | frontend/public/hosts/talac/robots.txt | 29 | ||||
| -rw-r--r-- | frontend/public/hosts/talafmt/robots.txt | 29 | ||||
| -rw-r--r-- | frontend/public/hosts/talalint/robots.txt | 29 | ||||
| -rw-r--r-- | frontend/public/hosts/tazel/robots.txt | 29 | ||||
| -rw-r--r-- | frontend/public/robots.txt | 29 |
5 files changed, 5 insertions, 140 deletions
diff --git a/frontend/public/hosts/talac/robots.txt b/frontend/public/hosts/talac/robots.txt index 1bb6b21..fcb318b 100644 --- a/frontend/public/hosts/talac/robots.txt +++ b/frontend/public/hosts/talac/robots.txt @@ -25,37 +25,10 @@ Allow: / User-agent: ClaudeBot Allow: / -User-agent: PerplexityBot -Disallow: / - -User-agent: Perplexity-User -Disallow: / - -User-agent: Bytespider -Disallow: / - User-agent: CCBot Disallow: / -User-agent: meta-externalagent -Disallow: / - -User-agent: meta-externalfetcher -Disallow: / - -User-agent: FacebookBot -Disallow: / - -User-agent: DuckAssistBot -Allow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Google-CloudVertexBot -Disallow: / - -User-agent: MistralAI-User +User-agent: Bytespider Disallow: / User-agent: * diff --git a/frontend/public/hosts/talafmt/robots.txt b/frontend/public/hosts/talafmt/robots.txt index 150b53e..3ebcc0a 100644 --- a/frontend/public/hosts/talafmt/robots.txt +++ b/frontend/public/hosts/talafmt/robots.txt @@ -25,37 +25,10 @@ Allow: / User-agent: ClaudeBot Allow: / -User-agent: PerplexityBot -Disallow: / - -User-agent: Perplexity-User -Disallow: / - -User-agent: Bytespider -Disallow: / - User-agent: CCBot Disallow: / -User-agent: meta-externalagent -Disallow: / - -User-agent: meta-externalfetcher -Disallow: / - -User-agent: FacebookBot -Disallow: / - -User-agent: DuckAssistBot -Allow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Google-CloudVertexBot -Disallow: / - -User-agent: MistralAI-User +User-agent: Bytespider Disallow: / User-agent: * diff --git a/frontend/public/hosts/talalint/robots.txt b/frontend/public/hosts/talalint/robots.txt index 4fadfbd..ff81354 100644 --- a/frontend/public/hosts/talalint/robots.txt +++ b/frontend/public/hosts/talalint/robots.txt @@ -25,37 +25,10 @@ Allow: / User-agent: ClaudeBot Allow: / -User-agent: PerplexityBot -Disallow: / - -User-agent: Perplexity-User -Disallow: / - -User-agent: Bytespider -Disallow: / - User-agent: CCBot Disallow: / -User-agent: meta-externalagent -Disallow: / - -User-agent: meta-externalfetcher -Disallow: / - -User-agent: FacebookBot -Disallow: / - -User-agent: DuckAssistBot -Allow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Google-CloudVertexBot -Disallow: / - -User-agent: MistralAI-User +User-agent: Bytespider Disallow: / User-agent: * diff --git a/frontend/public/hosts/tazel/robots.txt b/frontend/public/hosts/tazel/robots.txt index 61d4db2..12092e5 100644 --- a/frontend/public/hosts/tazel/robots.txt +++ b/frontend/public/hosts/tazel/robots.txt @@ -25,37 +25,10 @@ Allow: / User-agent: ClaudeBot Allow: / -User-agent: PerplexityBot -Disallow: / - -User-agent: Perplexity-User -Disallow: / - -User-agent: Bytespider -Disallow: / - User-agent: CCBot Disallow: / -User-agent: meta-externalagent -Disallow: / - -User-agent: meta-externalfetcher -Disallow: / - -User-agent: FacebookBot -Disallow: / - -User-agent: DuckAssistBot -Allow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Google-CloudVertexBot -Disallow: / - -User-agent: MistralAI-User +User-agent: Bytespider Disallow: / User-agent: * diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt index 2b84f74..deac40b 100644 --- a/frontend/public/robots.txt +++ b/frontend/public/robots.txt @@ -25,37 +25,10 @@ Allow: / User-agent: ClaudeBot Allow: / -User-agent: PerplexityBot -Disallow: / - -User-agent: Perplexity-User -Disallow: / - -User-agent: Bytespider -Disallow: / - User-agent: CCBot Disallow: / -User-agent: meta-externalagent -Disallow: / - -User-agent: meta-externalfetcher -Disallow: / - -User-agent: FacebookBot -Disallow: / - -User-agent: DuckAssistBot -Allow: / - -User-agent: Amazonbot -Disallow: / - -User-agent: Google-CloudVertexBot -Disallow: / - -User-agent: MistralAI-User +User-agent: Bytespider Disallow: / User-agent: * |