From 40218b9c33f11fad92a5f8341b802b3ae69648b6 Mon Sep 17 00:00:00 2001 From: Ethan Reesor Date: Tue, 14 Apr 2020 00:49:58 -0500 Subject: [PATCH 1/3] Add Go to the Packages API - Add Go to Package types - Add a worker for refreshing Go packages - Add a service to schedule the worker - Add an API to call the service - Update GraphQL schema with new package type - Add regexes --- app/models/blob_viewer/go_mod.rb | 10 +-- .../graphql/reference/gitlab_schema.graphql | 5 ++ doc/api/graphql/reference/gitlab_schema.json | 6 ++ ee/app/finders/packages/go/version_finder.rb | 3 +- ee/app/models/packages/go/module.rb | 25 +++++-- ee/app/models/packages/go/module_version.rb | 27 ++++++- ee/app/models/packages/package.rb | 3 +- .../packages/go/create_package_service.rb | 50 +++++++++++++ .../packages/go/refresh_packages_service.rb | 22 ++++++ ee/app/workers/all_queues.yml | 8 +++ ee/app/workers/packages/go/refresh_worker.rb | 30 ++++++++ ee/lib/api/project_packages.rb | 34 +++++++++ ee/spec/factories/packages.rb | 6 ++ .../graphql/types/package_type_enum_spec.rb | 2 +- ee/spec/requests/api/project_packages_spec.rb | 22 ++++++ .../go/create_package_service_spec.rb | 69 ++++++++++++++++++ .../go/refresh_packages_service_spec.rb | 53 ++++++++++++++ .../services/packages_shared_examples.rb | 1 + .../packages/go/refresh_worker_spec.rb | 71 +++++++++++++++++++ lib/gitlab/golang.rb | 18 ++++- lib/gitlab/regex.rb | 15 ++++ spec/lib/gitlab/regex_spec.rb | 13 ++++ 22 files changed, 470 insertions(+), 23 deletions(-) create mode 100644 ee/app/services/packages/go/create_package_service.rb create mode 100644 ee/app/services/packages/go/refresh_packages_service.rb create mode 100644 ee/app/workers/packages/go/refresh_worker.rb create mode 100644 ee/spec/services/packages/go/create_package_service_spec.rb create mode 100644 ee/spec/services/packages/go/refresh_packages_service_spec.rb create mode 100644 ee/spec/workers/packages/go/refresh_worker_spec.rb diff --git a/app/models/blob_viewer/go_mod.rb b/app/models/blob_viewer/go_mod.rb index ae57e2c0526853..28eb06bf8fca39 100644 --- a/app/models/blob_viewer/go_mod.rb +++ b/app/models/blob_viewer/go_mod.rb @@ -5,14 +5,6 @@ class GoMod < DependencyManager include ServerSide include Gitlab::Utils::StrongMemoize - MODULE_REGEX = / - \A (?# beginning of file) - module\s+ (?# module directive) - (?.*?) (?# module name) - \s*(?:\/\/.*)? (?# comment) - (?:\n|\z) (?# newline or end of file) - /x.freeze - self.file_types = %i(go_mod go_sum) def manager_name @@ -30,7 +22,7 @@ def package_type def package_name strong_memoize(:package_name) do next if blob.name != 'go.mod' - next unless match = MODULE_REGEX.match(blob.data) + next unless match = Gitlab::Regex.go_mod_module_regex.match(blob.data) match[:name] end diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index 23507172c6f167..e418148e7c8148 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -8130,6 +8130,11 @@ enum PackageTypeEnum { """ CONAN + """ + Packages from the golang package manager + """ + GOLANG + """ Packages from the maven package manager """ diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index 06d5d50a4b0f74..e81a047b660ec5 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -24257,6 +24257,12 @@ "description": "Packages from the composer package manager", "isDeprecated": false, "deprecationReason": null + }, + { + "name": "GOLANG", + "description": "Packages from the golang package manager", + "isDeprecated": false, + "deprecationReason": null } ], "possibleTypes": null diff --git a/ee/app/finders/packages/go/version_finder.rb b/ee/app/finders/packages/go/version_finder.rb index 8e2fab8ba354af..e3c6a298f220cc 100644 --- a/ee/app/finders/packages/go/version_finder.rb +++ b/ee/app/finders/packages/go/version_finder.rb @@ -23,7 +23,8 @@ def find(target) when String if pseudo_version? target semver = parse_semver(target) - commit = pseudo_version_commit(@mod.project, semver) + sha, timestamp = parse_pseudo_version(semver) + commit = validate_pseudo_version(@mod.project, sha, timestamp) Packages::Go::ModuleVersion.new(@mod, :pseudo, commit, name: target, semver: semver) else @mod.version_by(ref: target) diff --git a/ee/app/models/packages/go/module.rb b/ee/app/models/packages/go/module.rb index b38b691ed6cdd9..4be2c6a8d9ce38 100644 --- a/ee/app/models/packages/go/module.rb +++ b/ee/app/models/packages/go/module.rb @@ -5,6 +5,8 @@ module Go class Module include Gitlab::Utils::StrongMemoize + PATH_VERSION_REGEX = /\/v(\d+)$/i.freeze + attr_reader :project, :name, :path def initialize(project, name, path) @@ -13,6 +15,10 @@ def initialize(project, name, path) @path = path end + def inspect + "#<#{self.class.name.split('::').last} #{name}>" + end + def versions strong_memoize(:versions) { Packages::Go::VersionFinder.new(self).execute } end @@ -33,7 +39,17 @@ def version_by(ref: nil, commit: nil) end def path_valid?(major) - m = /\/v(\d+)$/i.match(@name) + # Ensure that go.mod contains an import path that correctly corresponds to + # the major version. + # + # https://blog.golang.org/v2-go-modules + # + # example.com/go/lib@v0.0.1 is imported as "example.com/go/lib" + # example.com/go/lib@v1.0.0 is imported as "example.com/go/lib" + # example.com/go/lib@v2.0.0 is imported as "example.com/go/lib/v2" + # example.com/go/lib@v3.0.0 is imported as "example.com/go/lib/v3" + + m = PATH_VERSION_REGEX.match(@name) case major when 0, 1 @@ -44,11 +60,10 @@ def path_valid?(major) end def gomod_valid?(gomod) - if Feature.enabled?(:go_proxy_disable_gomod_validation, @project) - return gomod&.start_with?("module ") - end + return false unless m = Gitlab::Regex.go_mod_module_regex.match(gomod) + return true if Feature.enabled?(:go_proxy_disable_gomod_validation, @project) - gomod&.split("\n", 2)&.first == "module #{@name}" + return m[:name] == @name end private diff --git a/ee/app/models/packages/go/module_version.rb b/ee/app/models/packages/go/module_version.rb index a50c78f8e69954..a0f447b36e979f 100644 --- a/ee/app/models/packages/go/module_version.rb +++ b/ee/app/models/packages/go/module_version.rb @@ -4,6 +4,7 @@ module Packages module Go class ModuleVersion include Gitlab::Utils::StrongMemoize + include Gitlab::Golang VALID_TYPES = %i[ref commit pseudo].freeze @@ -30,9 +31,13 @@ def initialize(mod, type, commit, name: nil, semver: nil, ref: nil) @mod = mod @type = type @commit = commit - @name = name if name - @semver = semver if semver - @ref = ref if ref + @name = name + @semver = semver + @ref = ref + end + + def inspect + "#<#{self.class.name.split('::').last} #{full_name}>" end def name @@ -80,7 +85,23 @@ def excluded end end + def package + strong_memoize(:package) do + @mod.project.packages.golang + .with_name(mod.name) + .with_version(self.name) + .first + end + end + + def clear_package_memoization + clear_memoization(:package) + end + def valid? + # assume the module version is valid if a corresponding Package exists + return true unless package.nil? + @mod.path_valid?(major) && @mod.gomod_valid?(gomod) end diff --git a/ee/app/models/packages/package.rb b/ee/app/models/packages/package.rb index bef002dd916142..a9f846d7cb5912 100644 --- a/ee/app/models/packages/package.rb +++ b/ee/app/models/packages/package.rb @@ -36,8 +36,9 @@ class Packages::Package < ApplicationRecord validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { npm? || nuget? } validates :name, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? validates :version, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? + validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :golang? - enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, composer: 6 } + enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, composer: 6, golang: 7 } scope :with_name, ->(name) { where(name: name) } scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) } diff --git a/ee/app/services/packages/go/create_package_service.rb b/ee/app/services/packages/go/create_package_service.rb new file mode 100644 index 00000000000000..a8374cdd4b6621 --- /dev/null +++ b/ee/app/services/packages/go/create_package_service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Go + class CreatePackageService < BaseService + attr_accessor :version + + def initialize(project, user = nil, version:) + super(project, user) + + @version = version + end + + def execute + # find existing package + package = version.package + + ActiveRecord::Base.transaction do + # create new package if necessary + unless package + package = project.packages.create!( + name: version.mod.name, + version: version.name, + package_type: 'golang', + created_at: version.commit.committed_date + ) + version.clear_package_memoization + end + + # files to create + files = {} + files[:info] = -> { { Version: version.name, Time: version.commit.committed_date }.to_json } + files[:mod] = -> { version.gomod } + files[:zip] = -> { version.archive.string } + + # create files if neccessary + files.each do |type, generate| + file_name = "#{version.name}.#{type}" + next if package.package_files.with_file_name(file_name).any? + + file = CarrierWaveStringFile.new(generate.call) + CreatePackageFileService.new(package, file_name: file_name, file: file).execute + end + + package + end + end + end + end +end diff --git a/ee/app/services/packages/go/refresh_packages_service.rb b/ee/app/services/packages/go/refresh_packages_service.rb new file mode 100644 index 00000000000000..aa697e5921db8f --- /dev/null +++ b/ee/app/services/packages/go/refresh_packages_service.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Packages + module Go + class RefreshPackagesService < BaseService + include Gitlab::Golang + + def initialize(project, user) + super(project, user) + + raise ArgumentError, 'project is required' unless project + raise ArgumentError, 'user is required' unless user + + raise 'Unauthorized' unless user.can?(:create_package, project) + end + + def execute_async + Packages::Go::RefreshWorker.perform_async(current_user.id, project.id) + end + end + end +end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 5ecf9591f4a036..82d0bbdee629f8 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -459,6 +459,14 @@ :weight: 1 :idempotent: :tags: [] +- :name: package_repositories:packages_go_refresh + :feature_category: :package_registry + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: package_repositories:packages_nuget_extraction :feature_category: :package_registry :has_external_dependencies: diff --git a/ee/app/workers/packages/go/refresh_worker.rb b/ee/app/workers/packages/go/refresh_worker.rb new file mode 100644 index 00000000000000..b4dca8ea4d9287 --- /dev/null +++ b/ee/app/workers/packages/go/refresh_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Packages + module Go + class RefreshWorker + include ApplicationWorker + include Gitlab::Golang + + queue_namespace :package_repositories + feature_category :package_registry + + idempotent! + + def perform(user_id, project_id) + user = User.find_by_id(user_id) + project = Project.find_by_id(project_id) + return unless user && project + + module_name = go_path(project) # ignores modules in subdirs + versions = Packages::Go::ModuleFinder.new(project, module_name).execute.versions + + ActiveRecord::Base.transaction do + versions.map do |v| + Packages::Go::CreatePackageService.new(project, user, version: v).execute + end + end + end + end + end +end diff --git a/ee/lib/api/project_packages.rb b/ee/lib/api/project_packages.rb index a9d94b457a9800..2e242f08270aca 100644 --- a/ee/lib/api/project_packages.rb +++ b/ee/lib/api/project_packages.rb @@ -10,6 +10,14 @@ class ProjectPackages < Grape::API helpers ::API::Helpers::PackagesHelpers + helpers do + def obtain_new_lease(key, timeout:) + Gitlab::ExclusiveLease + .new(key, timeout: timeout) + .try_obtain + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -38,6 +46,32 @@ class ProjectPackages < Grape::API present paginate(packages), with: EE::API::Entities::Package, user: current_user end + desc 'Refresh packages' do + detail 'Rebuild the list of packages, for packages dynamically derived from the repository' + end + params do + requires :package_type, type: String, values: %w[go golang], + desc: 'Refresh packages of this type' + end + put ':id/packages/refresh' do + case params[:package_type] + when 'go', 'golang' + not_found!('Package Type') unless Feature.enabled?(:go_packages, user_project) + + authorize_create_package! + + message = 'This request has already been made. It may take some time to refresh packages. You can run this at most once an hour for a given project and ref.' + render_api_error!(message, 409) unless obtain_new_lease("project_packages:refresh:go:#{user_project.id}", timeout: 1.hour) + + ::Packages::Go::RefreshPackagesService.new(user_project, current_user).execute_async + + accepted! + + else + not_found!('Package Type') + end + end + desc 'Get a single project package' do detail 'This feature was introduced in GitLab 11.9' success EE::API::Entities::Package diff --git a/ee/spec/factories/packages.rb b/ee/spec/factories/packages.rb index 5906d38aea38bf..1d34a1a4c79f7c 100644 --- a/ee/spec/factories/packages.rb +++ b/ee/spec/factories/packages.rb @@ -73,6 +73,12 @@ package_type { :composer } end + factory :golang_package do + sequence(:name) { |n| "golang.org/x/pkg-#{n}"} + sequence(:version) { |n| "v1.0.#{n}" } + package_type { :golang } + end + factory :conan_package do conan_metadatum diff --git a/ee/spec/graphql/types/package_type_enum_spec.rb b/ee/spec/graphql/types/package_type_enum_spec.rb index fadec9744ed612..d344456c133f76 100644 --- a/ee/spec/graphql/types/package_type_enum_spec.rb +++ b/ee/spec/graphql/types/package_type_enum_spec.rb @@ -4,6 +4,6 @@ RSpec.describe GitlabSchema.types['PackageTypeEnum'] do it 'exposes all package types' do - expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER]) + expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER GOLANG]) end end diff --git a/ee/spec/requests/api/project_packages_spec.rb b/ee/spec/requests/api/project_packages_spec.rb index 662e69912adf27..61080055d2a08b 100644 --- a/ee/spec/requests/api/project_packages_spec.rb +++ b/ee/spec/requests/api/project_packages_spec.rb @@ -317,4 +317,26 @@ end end end + + describe 'PUT /projects/:id/packages/refresh' do + let(:url) { "/projects/#{project.id}/packages/refresh?package_type=#{package_type}" } + + before do + project.add_developer(user) + stub_licensed_features(packages: true) + end + + context 'with Go package type' do + let_it_be(:project) { create :project } + let(:package_type) { 'golang' } + + it 'schedules a package refresh' do + expect(::Packages::Go::RefreshWorker).to receive(:perform_async).once + + put api(url, user) + + expect(response).to have_gitlab_http_status(:accepted) + end + end + end end diff --git a/ee/spec/services/packages/go/create_package_service_spec.rb b/ee/spec/services/packages/go/create_package_service_spec.rb new file mode 100644 index 00000000000000..776ce742610b5a --- /dev/null +++ b/ee/spec/services/packages/go/create_package_service_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Packages::Go::CreatePackageService do + let_it_be(:project) { create :project_empty_repo, path: 'my-go-lib' } + let_it_be(:mod) { create :go_module, project: project } + + before :all do + create :go_module_commit, :module, project: project, tag: 'v1.0.0' + end + + shared_examples 'a package' do |files:| + it "returns a valid package with #{files ? files.to_s : 'no'} file(s)" do + expect(subject).to be_valid + expect(subject.name).to eq(version.mod.name) + expect(subject.version).to eq(version.name) + expect(subject.package_type).to eq('golang') + expect(subject.created_at).to eq(version.commit.committed_date) + expect(subject.package_files.count).to eq(files) + end + end + + shared_examples 'a package file' do |type| + it "returns a package with a #{type} file" do + file_name = "#{version.name}.#{type}" + expect(subject.package_files.map { |f| f.file_name }).to include(file_name) + + file = subject.package_files.with_file_name(file_name).first + expect(file).not_to be_nil + expect(file.file_name).to eq(file_name) + end + end + + describe '#execute' do + subject { described_class.new(project, nil, version: version).execute } + + let(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.0' } + + context 'with no existing package' do + it_behaves_like 'a package', files: 3 + it_behaves_like 'a package file', :info + it_behaves_like 'a package file', :mod + it_behaves_like 'a package file', :zip + + it 'creates a new package' do + expect { subject } + .to change { project.packages.count }.by(1) + .and change { Packages::PackageFile.count }.by(3) + end + end + + context 'with an existing package' do + before do + described_class.new(project, version: version).execute + end + + it_behaves_like 'a package', files: 3 + it_behaves_like 'a package file', :info + it_behaves_like 'a package file', :mod + it_behaves_like 'a package file', :zip + + it 'does not create a package or files' do + expect { subject }.to not_change { project.packages.count } + expect { subject }.to not_change { Packages::PackageFile.count } + end + end + end +end diff --git a/ee/spec/services/packages/go/refresh_packages_service_spec.rb b/ee/spec/services/packages/go/refresh_packages_service_spec.rb new file mode 100644 index 00000000000000..a087b1603d6b51 --- /dev/null +++ b/ee/spec/services/packages/go/refresh_packages_service_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::RefreshPackagesService do + let_it_be(:maintainer) { create :user } + let_it_be(:guest) { create :user } + let_it_be(:project) { create :project_empty_repo } + let(:params) { { info: true, mod: true, zip: true } } + + before :all do + create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' } + create :go_module_commit, :module, project: project, tag: 'v1.0.1' + create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg' + create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod' + create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" } + create :go_module_commit, :module, project: project, name: 'v2' + create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } + + project.add_maintainer(maintainer) + end + + describe '#execute_async' do + it 'schedules a package refresh' do + expect(::Packages::Go::RefreshWorker).to receive(:perform_async).once + + described_class.new(project, maintainer).execute_async + end + end + + describe '#initialize' do + context 'without a project' do + it 'raises an error' do + expect { described_class.new(nil, maintainer) } + .to raise_error(ArgumentError, 'project is required') + end + end + + context 'without a user' do + it 'raises an error' do + expect { described_class.new(project, nil) } + .to raise_error(ArgumentError, 'user is required') + end + end + + context 'with a user not authorized to create packages' do + it 'raises an error' do + expect { described_class.new(project, guest) } + .to raise_error('Unauthorized') + end + end + end +end diff --git a/ee/spec/support/shared_examples/services/packages_shared_examples.rb b/ee/spec/support/shared_examples/services/packages_shared_examples.rb index bae20fca60c07f..f3d0adb40df775 100644 --- a/ee/spec/support/shared_examples/services/packages_shared_examples.rb +++ b/ee/spec/support/shared_examples/services/packages_shared_examples.rb @@ -161,6 +161,7 @@ def group_filter_url(filter, param) let_it_be(:package4) { create(:nuget_package, project: project) } let_it_be(:package5) { create(:pypi_package, project: project) } let_it_be(:package6) { create(:composer_package, project: project) } + let_it_be(:package7) { create(:golang_package, project: project) } Packages::Package.package_types.keys.each do |package_type| context "for package type #{package_type}" do diff --git a/ee/spec/workers/packages/go/refresh_worker_spec.rb b/ee/spec/workers/packages/go/refresh_worker_spec.rb new file mode 100644 index 00000000000000..0fa6fc62cb3e62 --- /dev/null +++ b/ee/spec/workers/packages/go/refresh_worker_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Packages::Go::RefreshWorker, type: :worker do + let_it_be(:user) { create :user } + let_it_be(:project) { create :project_empty_repo, :public } + let(:params) { { info: true, mod: true, zip: true } } + + subject { described_class.new.perform(user.id, project.id) } + + before :all do + create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' } + create :go_module_commit, :module, project: project, tag: 'v1.0.1' + create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg' + create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod' + create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" } + create :go_module_commit, :module, project: project, name: 'v2' + create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } + + project.add_maintainer(user) + end + + shared_examples 'a package' do |path, version, files: nil| + it "returns a package for example.com/project#{path.empty? ? '' : '/' + path}@#{version}" do + mod = create :go_module, project: project, path: path + ver = create :go_module_version, :tagged, mod: mod, name: version + expect(subject.map { |v| "#{v.name}@#{v.version}" }).to include(ver.full_name) + + pkg = subject.find { |v| v.name == mod.name && v.version == ver.name } + expect(pkg).not_to be_nil + expect(pkg.package_type).to eq('golang') + expect(pkg.created_at).to eq(ver.commit.committed_date) + expect(pkg.package_files.count).to eq(files) unless files.nil? + end + end + + describe '#perform' do + it_behaves_like 'a package', '', 'v1.0.1' + it_behaves_like 'a package', '', 'v1.0.2' + it_behaves_like 'a package', '', 'v1.0.3' + + # subdirectories (mod, v2) are not scanned + + context 'with no existing packages' do + it 'creates a package for each version of each module' do + expect { subject } + .to change { project.packages.count }.by(3) + .and change { Packages::PackageFile.count }.by(9) + + expect(subject.count).to eq(3) + end + end + + context 'with existing packages' do + before do + mod = create :go_module, project: project + ver = create :go_module_version, :tagged, mod: mod, name: 'v1.0.1' + Packages::Go::CreatePackageService.new(project, nil, version: ver).execute + end + + it 'creates the missing package entries' do + expect { subject } + .to change { project.packages.count }.by(2) + .and change { Packages::PackageFile.count }.by(6) + + expect(subject.count).to eq(3) + end + end + end +end diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index f2dc668c482a03..88b68ab25a1d8e 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -37,11 +37,11 @@ def pseudo_version?(version) end # This pattern is intentionally more forgiving than the patterns - # above. Correctness is verified by #pseudo_version_commit. + # above. Correctness is verified by #validate_pseudo_version. /\A\d{14}-\h+\z/.freeze.match? pre end - def pseudo_version_commit(project, semver) + def parse_pseudo_version(semver) # Per Go's implementation of pseudo-versions, a tag should be # considered a pseudo-version if it matches one of the patterns # listed in #pseudo_version?, regardless of the content of the @@ -57,7 +57,11 @@ def pseudo_version_commit(project, semver) # Go ignores anything before '.' or after the second '-', so we will do the same timestamp, sha = semver.prerelease.split('-').last 2 timestamp = timestamp.split('.').last - commit = project.repository.commit_by(oid: sha) + [sha, timestamp] + end + + def validate_pseudo_version(project, sha, timestamp, commit = nil) + commit ||= project.repository.commit_by(oid: sha) # Error messages are based on the responses of proxy.golang.org @@ -77,6 +81,14 @@ def parse_semver(str) Packages::SemVer.parse(str, prefixed: true) end + def go_path(project, path = nil) + if path.nil? || path.empty? + "#{local_module_prefix}/#{project.full_path}" + else + "#{local_module_prefix}/#{project.full_path}/#{path}" + end + end + def pkg_go_dev_url(name, version = nil) if version "https://pkg.go.dev/#{name}@#{version}" diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4caff8ae6798e5..8e5f8ab7434f45 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -66,6 +66,11 @@ def semver_regex @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end + def prefixed_semver_regex + # identical to semver_regex, except starting with 'v' + @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + def go_package_regex # A Go package name looks like a URL but is not; it: # - Must not have a scheme, such as http:// or https:// @@ -85,6 +90,16 @@ def go_package_regex \b (?# word boundary) /ix.freeze end + + def go_mod_module_regex + @go_mod_module_regex ||= / + \A (?# beginning of file) + module\s+ (?# module directive) + (?.*?) (?# module name) + \s*(?:\/\/.*)? (?# comment) + (?:\n|\z) (?# newline or end of file) + /x.freeze + end end extend self diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index 2f220272651921..740ffb897c43bb 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -295,4 +295,17 @@ it { is_expected.not_to match('1.2') } it { is_expected.not_to match('1./2.3') } end + + describe '.prefixed_semver_regex' do + subject { described_class.prefixed_semver_regex } + + it { is_expected.to match('v1.2.3') } + it { is_expected.to match('v1.2.3-beta') } + it { is_expected.to match('v1.2.3-alpha.3') } + it { is_expected.not_to match('v1') } + it { is_expected.not_to match('v1.2') } + it { is_expected.not_to match('v1./2.3') } + it { is_expected.not_to match('v../../../../../1.2.3') } + it { is_expected.not_to match('v%2e%2e%2f1.2.3') } + end end -- GitLab From 3b5b3fa3fe26f81508e5b1e4011e3959948b61b9 Mon Sep 17 00:00:00 2001 From: Ethan Reesor Date: Tue, 14 Apr 2020 00:53:03 -0500 Subject: [PATCH 2/3] Add Go to the Packages UI --- .../packages/details/components/app.vue | 4 ++ .../details/components/go_installation.vue | 60 +++++++++++++++++ .../javascripts/packages/details/constants.js | 4 ++ .../packages/details/store/getters.js | 6 ++ .../list/components/packages_list_app.vue | 30 ++++++++- .../javascripts/packages/list/constants.js | 6 ++ .../javascripts/packages/shared/constants.js | 1 + .../javascripts/packages/shared/utils.js | 2 + .../projects/packages/packages_controller.rb | 4 ++ ee/app/helpers/ee/packages_helper.rb | 3 +- .../projects/packages/packages/show.html.haml | 4 +- .../packages/details/components/app_spec.js | 4 ++ .../components/go_installation_spec.js | 64 +++++++++++++++++++ .../packages/details/store/getters_spec.js | 22 +++++++ .../packages_list_app_spec.js.snap | 62 ++++++++++++++++++ ee/spec/frontend/packages/mock_data.js | 12 ++++ locale/gitlab.pot | 24 +++++++ 17 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 ee/app/assets/javascripts/packages/details/components/go_installation.vue create mode 100644 ee/spec/frontend/packages/details/components/go_installation_spec.js diff --git a/ee/app/assets/javascripts/packages/details/components/app.vue b/ee/app/assets/javascripts/packages/details/components/app.vue index da4429f5134781..30535c4cf3d330 100644 --- a/ee/app/assets/javascripts/packages/details/components/app.vue +++ b/ee/app/assets/javascripts/packages/details/components/app.vue @@ -18,6 +18,7 @@ import PackageActivity from './activity.vue'; import PackageInformation from './information.vue'; import PackageTitle from './package_title.vue'; import ConanInstallation from './conan_installation.vue'; +import GoInstallation from './go_installation.vue'; import MavenInstallation from './maven_installation.vue'; import NpmInstallation from './npm_installation.vue'; import NugetInstallation from './nuget_installation.vue'; @@ -50,6 +51,7 @@ export default { PackageInformation, PackageTitle, ConanInstallation, + GoInstallation, MavenInstallation, NpmInstallation, NugetInstallation, @@ -79,6 +81,8 @@ export default { switch (this.packageEntity.package_type) { case PackageType.CONAN: return ConanInstallation; + case PackageType.GO: + return GoInstallation; case PackageType.MAVEN: return MavenInstallation; case PackageType.NPM: diff --git a/ee/app/assets/javascripts/packages/details/components/go_installation.vue b/ee/app/assets/javascripts/packages/details/components/go_installation.vue new file mode 100644 index 00000000000000..0b097c2d558115 --- /dev/null +++ b/ee/app/assets/javascripts/packages/details/components/go_installation.vue @@ -0,0 +1,60 @@ + + + diff --git a/ee/app/assets/javascripts/packages/details/constants.js b/ee/app/assets/javascripts/packages/details/constants.js index d12f93dd277004..0008eff7a749d7 100644 --- a/ee/app/assets/javascripts/packages/details/constants.js +++ b/ee/app/assets/javascripts/packages/details/constants.js @@ -3,6 +3,7 @@ import { s__ } from '~/locale'; export const TrackingLabels = { CODE_INSTRUCTION: 'code_instruction', CONAN_INSTALLATION: 'conan_installation', + GO_INSTALLATION: 'go_installation', MAVEN_INSTALLATION: 'maven_installation', NPM_INSTALLATION: 'npm_installation', NUGET_INSTALLATION: 'nuget_installation', @@ -16,6 +17,9 @@ export const TrackingActions = { COPY_CONAN_COMMAND: 'copy_conan_command', COPY_CONAN_SETUP_COMMAND: 'copy_conan_setup_command', + COPY_GO_GET_COMMAND: 'copy_go_get_command', + COPY_GO_ENV_COMMAND: 'copy_go_env_command', + COPY_MAVEN_XML: 'copy_maven_xml', COPY_MAVEN_COMMAND: 'copy_maven_command', COPY_MAVEN_SETUP: 'copy_maven_setup_xml', diff --git a/ee/app/assets/javascripts/packages/details/store/getters.js b/ee/app/assets/javascripts/packages/details/store/getters.js index bcf74713f03cbc..100334e1786d1f 100644 --- a/ee/app/assets/javascripts/packages/details/store/getters.js +++ b/ee/app/assets/javascripts/packages/details/store/getters.js @@ -30,6 +30,12 @@ export const conanSetupCommand = ({ conanPath }) => // eslint-disable-next-line @gitlab/require-i18n-strings `conan remote add gitlab ${conanPath}`; +export const goInstallationCommand = ({ packageEntity }) => + // eslint-disable-next-line @gitlab/require-i18n-strings + `go get ${packageEntity.name}@${packageEntity.version}`; + +export const goSetupCommand = ({ goPath }) => `go env -w GOPROXY="${goPath},$(go env GOPROXY)"`; + export const mavenInstallationXml = ({ packageEntity = {} }) => { const { app_group: appGroup = '', diff --git a/ee/app/assets/javascripts/packages/list/components/packages_list_app.vue b/ee/app/assets/javascripts/packages/list/components/packages_list_app.vue index dabeb7f21f1f5f..6d68f886c9f041 100644 --- a/ee/app/assets/javascripts/packages/list/components/packages_list_app.vue +++ b/ee/app/assets/javascripts/packages/list/components/packages_list_app.vue @@ -2,6 +2,7 @@ import { mapActions, mapState } from 'vuex'; import { GlEmptyState, GlTab, GlTabs, GlLink, GlSprintf } from '@gitlab/ui'; import { s__, sprintf } from '~/locale'; +import glFeatureFlagsMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; import PackageFilter from './packages_filter.vue'; import PackageList from './packages_list.vue'; import PackageSort from './packages_sort.vue'; @@ -20,12 +21,14 @@ export default { PackageSort, PackagesComingSoon, }, + mixins: [glFeatureFlagsMixin()], computed: { ...mapState({ emptyListIllustration: state => state.config.emptyListIllustration, emptyListHelpUrl: state => state.config.emptyListHelpUrl, comingSoon: state => state.config.comingSoon, filterQuery: state => state.filterQuery, + goHelpPath: state => state.config.goHelpPath, }), tabsToRender() { return PACKAGE_REGISTRY_TABS; @@ -50,11 +53,17 @@ export default { this.requestPackagesList(); } }, - emptyStateTitle({ title, type }) { + emptyStateTitle({ title, type, featureFlag }) { if (this.filterQuery) { return s__('PackageRegistry|Sorry, your filter produced no results'); } + if (featureFlag && !this.glFeatures[featureFlag]) { + return sprintf(s__('PackageRegistry|%{packageType} packages are disabled'), { + packageType: title, + }); + } + if (type) { return sprintf(s__('PackageRegistry|There are no %{packageType} packages yet'), { packageType: title, @@ -63,12 +72,21 @@ export default { return s__('PackageRegistry|There are no packages yet'); }, + tabDisabled({ featureFlag }) { + return featureFlag && !this.glFeatures[featureFlag]; + }, + tabFeatureHelpPath({ featureHelpPath }) { + return this[featureHelpPath]; + }, }, i18n: { widenFilters: s__('PackageRegistry|To widen your search, change or remove the filters above.'), noResults: s__( 'PackageRegistry|Learn how to %{noPackagesLinkStart}publish and share your packages%{noPackagesLinkEnd} with GitLab.', ), + featureHelpText: s__( + 'PackageRegistry|Use feature flags to enable %{packageType} packages. %{linkStart}See the documentation%{linkEnd} to find out more.', + ), }, }; @@ -87,7 +105,15 @@ export default {