From 2e1d6b3080835b947aaf82bf457c8c497be0d0b6 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 17 Jun 2020 20:52:56 +0300 Subject: [PATCH 1/3] Clean up robots.txt Signed-off-by: Dmitriy Zaporozhets --- public/robots.txt | 87 +++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 49 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 20cd97a596f52c..12ceba8839576f 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -8,7 +8,10 @@ # Only some crawlers respect this setting, e.g. Googlebot does not # Crawl-delay: 1 -# Based on details in https://gitlab.com/gitlab-org/gitlab/blob/master/config/routes.rb, https://gitlab.com/gitlab-org/gitlab/blob/master/spec/routing, and using application +# Based on details in https://gitlab.com/gitlab-org/gitlab/blob/master/config/routes.rb, +# https://gitlab.com/gitlab-org/gitlab/blob/master/spec/routing, and using application + +# Global routes User-Agent: * Disallow: /autocomplete/users Disallow: /search @@ -16,62 +19,48 @@ Disallow: /api Disallow: /admin Disallow: /profile Disallow: /dashboard -Disallow: /projects/new -Disallow: /groups/new -Disallow: /groups/*/edit Disallow: /users Disallow: /help +Disallow: /s/ # Only specifically allow the Sign In page to avoid very ugly search results Allow: /users/sign_in -# Global snippets +# Generic resource routes like new, edit, raw +# This will block routes like: +# - /projects/new +# - /gitlab-org/gitlab-foss/issues/123/-/edit User-Agent: * -Disallow: /s/ -Disallow: /snippets/new -Disallow: /snippets/*/edit -Disallow: /snippets/*/raw +Disallow: /*/new +Disallow: /*/edit +Disallow: /*/raw + +# Group details +User-Agent: * +Disallow: /groups/*/analytics +Disallow: /groups/*/contribution_analytics +Disallow: /groups/*/group_members # Project details User-Agent: * Disallow: /*/*.git -Disallow: /*/*/fork/new -Disallow: /*/-/archive/ -Disallow: /*/*/repository/archive* -Disallow: /*/*/activity -Disallow: /*/*/new -Disallow: /*/*/edit -Disallow: /*/*/raw -Disallow: /*/*/blame -Disallow: /*/*/commits/*/* -Disallow: /*/*/commit/*.patch -Disallow: /*/*/commit/*.diff -Disallow: /*/*/compare -Disallow: /*/*/branches/new -Disallow: /*/*/tags/new -Disallow: /*/*/network -Disallow: /*/*/graphs -Disallow: /*/*/milestones/new -Disallow: /*/*/milestones/*/edit -Disallow: /*/*/issues/new -Disallow: /*/*/issues/*/edit -Disallow: /*/*/-/merge_requests/new -Disallow: /*/*/-/merge_requests/*.patch -Disallow: /*/*/-/merge_requests/*.diff -Disallow: /*/*/-/merge_requests/*/edit -Disallow: /*/*/-/merge_requests/*/diffs -Disallow: /*/*/project_members/import -Disallow: /*/*/labels/new -Disallow: /*/*/labels/*/edit -Disallow: /*/*/wikis/*/edit -Disallow: /*/*/snippets/new -Disallow: /*/*/snippets/*/edit -Disallow: /*/*/snippets/*/raw -Disallow: /*/*/deploy_keys -Disallow: /*/*/hooks -Disallow: /*/*/services -Disallow: /*/*/protected_branches -Disallow: /*/*/uploads/ -Disallow: /*/-/group_members +Disallow: /*/archive/ +Disallow: /*/repository/archive* +Disallow: /*/activity +Disallow: /*/blame +Disallow: /*/commits +Disallow: /*/commit +Disallow: /*/commit/*.patch +Disallow: /*/commit/*.diff +Disallow: /*/compare +Disallow: /*/network +Disallow: /*/graphs +Disallow: /*/merge_requests/*.patch +Disallow: /*/merge_requests/*.diff +Disallow: /*/merge_requests/*/diffs +Disallow: /*/deploy_keys +Disallow: /*/hooks +Disallow: /*/services +Disallow: /*/protected_branches +Disallow: /*/uploads/ Disallow: /*/project_members -Disallow: /groups/*/-/contribution_analytics -Disallow: /groups/*/-/analytics +Disallow: /*/settings -- GitLab From 8c3772c3ab154e1baeff66af5775f30f6c9831b0 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 18 Jun 2020 13:12:42 +0300 Subject: [PATCH 2/3] Remove no longer used /s/ from robots.txt Signed-off-by: Dmitriy Zaporozhets --- public/robots.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/public/robots.txt b/public/robots.txt index 12ceba8839576f..e9e497bcd943cd 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -21,7 +21,6 @@ Disallow: /profile Disallow: /dashboard Disallow: /users Disallow: /help -Disallow: /s/ # Only specifically allow the Sign In page to avoid very ugly search results Allow: /users/sign_in -- GitLab From 5d6e13d9386cf6f91af3523578f7c004c34f16cf Mon Sep 17 00:00:00 2001 From: Dmytro Zaporozhets Date: Thu, 18 Jun 2020 16:15:21 +0000 Subject: [PATCH 3/3] Revert "Remove no longer used /s/ from robots.txt" This reverts commit 8c3772c3ab154e1baeff66af5775f30f6c9831b0 --- public/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/public/robots.txt b/public/robots.txt index e9e497bcd943cd..12ceba8839576f 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -21,6 +21,7 @@ Disallow: /profile Disallow: /dashboard Disallow: /users Disallow: /help +Disallow: /s/ # Only specifically allow the Sign In page to avoid very ugly search results Allow: /users/sign_in -- GitLab