From 075a22f225d3bf28991eb03fbc31bb17104a1d57 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Wed, 26 Aug 2020 16:44:10 -0700 Subject: [PATCH 1/7] Add new secret key for encrypted configuration - Adds support for a new (optional) secret key from secrets.yml - Adds a encrypted configuration class than can decrypt/encrypt files using the provided key - Adds a new encrypted setting helper method to Settings for the new secret key --- config/settings.rb | 10 ++ doc/development/application_secrets.md | 1 + lib/gitlab/encrypted_configuration.rb | 84 +++++++++++++ spec/config/settings_spec.rb | 21 ++++ .../gitlab/encrypted_configuration_spec.rb | 112 ++++++++++++++++++ 5 files changed, 228 insertions(+) create mode 100644 lib/gitlab/encrypted_configuration.rb create mode 100644 spec/lib/gitlab/encrypted_configuration_spec.rb diff --git a/config/settings.rb b/config/settings.rb index c681fa32491712..3eaa7d8260867e 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -152,6 +152,16 @@ def attr_encrypted_db_key_base Gitlab::Application.secrets.db_key_base end + def encrypted(path) + return Gitlab::EncryptedConfiguration.new unless Gitlab::Application.secrets.enc_settings_key_base + + Gitlab::EncryptedConfiguration.new( + content_path: Settings.absolute(path), + key: Gitlab::Application.secrets.enc_settings_key_base, + previous_keys: Gitlab::Application.secrets.rotated_enc_settings_key_base || [] + ) + end + def load_dynamic_cron_schedules! cron_jobs['gitlab_usage_ping_worker']['cron'] ||= cron_for_usage_ping end diff --git a/doc/development/application_secrets.md b/doc/development/application_secrets.md index abc5ff7b98539b..59d0a54a153c42 100644 --- a/doc/development/application_secrets.md +++ b/doc/development/application_secrets.md @@ -16,6 +16,7 @@ This page is a development guide for application secrets. | `otp_key_base` | The base key for One Time Passwords, described in [User management](../raketasks/user_management.md#rotate-two-factor-authentication-encryption-key) | |`db_key_base` | The base key to encrypt the data for `attr_encrypted` columns | |`openid_connect_signing_key` | The singing key for OpenID Connect | +| `enc_settings_key_base` | The base key to encrypt settings files with | ## Where the secrets are stored diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb new file mode 100644 index 00000000000000..db195320a2f4dc --- /dev/null +++ b/lib/gitlab/encrypted_configuration.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + class EncryptedConfiguration + delegate :[], :fetch, to: :config + delegate_missing_to :options + attr_reader :content_path, :key, :previous_keys + + CIPHER = "aes-128-gcm" + + def initialize(content_path: nil, key: nil, previous_keys: []) + @content_path = Pathname.new(content_path).yield_self { |path| path.symlink? ? path.realpath : path } if content_path + @key = key + @previous_keys = previous_keys + end + + def read + if !key.nil? && content_path&.exist? + decrypt content_path.binread + else + "" + end + end + + def write(contents) + # ensure contents are valid to deserialize before write + deserialize(contents) + + temp_file = Tempfile.new(File.basename(content_path), File.dirname(content_path)) + File.open(temp_file.path, 'wb') do |file| + file.write(encrypt(contents)) + end + FileUtils.mv temp_file.path, content_path + ensure + temp_file&.unlink + end + + def config + @config ||= deserialize(read).deep_symbolize_keys + end + + def change(&block) + writing read, &block + end + + private + + def writing(contents) + updated_contents = yield contents + + write(updated_contents) if updated_contents != contents + end + + def encrypt(contents) + encryptor.encrypt_and_sign contents + end + + def decrypt(contents) + encryptor.decrypt_and_verify contents + end + + def encryptor + return @encryptor if @encryptor + + @encryptor = ActiveSupport::MessageEncryptor.new([key].pack("H*"), cipher: CIPHER) + + # Allow fallback to previous keys + @previous_keys.each do |key| + @encryptor.rotate([key].pack("H*")) + end + + @encryptor + end + + def options + # Allows top level keys to be referenced using dot syntax + @options ||= ActiveSupport::InheritableOptions.new(config) + end + + def deserialize(contents) + YAML.safe_load(contents, permitted_classes: [Symbol]).presence || {} + end + end +end diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index ed873478fc999c..d83d042c8ebf36 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -134,4 +134,25 @@ end end end + + describe '.encrypted' do + before do + allow(Gitlab::Application.secrets).to receive(:enc_settings_key_base).and_return(SecureRandom.hex(64)) + end + + it 'defaults to using the enc_settings_key_base for the key' do + expect(Gitlab::EncryptedConfiguration).to receive(:new).with(hash_including(key: Gitlab::Application.secrets.enc_settings_key_base)) + Settings.encrypted('tmp/tests/test.enc') + end + + it 'defaults the configpath within the rails root' do + expect(Settings.encrypted('tmp/tests/test.enc').content_path.fnmatch?(File.join(Rails.root, '**'))).to be true + end + + it 'returns empty encrypted config when a key has not been set' do + allow(Gitlab::Application.secrets).to receive(:enc_settings_key_base).and_return(nil) + expect(Gitlab::EncryptedConfiguration).to receive(:new).with(no_args) + Settings.encrypted('tmp/tests/test.enc') + end + end end diff --git a/spec/lib/gitlab/encrypted_configuration_spec.rb b/spec/lib/gitlab/encrypted_configuration_spec.rb new file mode 100644 index 00000000000000..de41a522baa97d --- /dev/null +++ b/spec/lib/gitlab/encrypted_configuration_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::EncryptedConfiguration do + subject(:configuration) { described_class.new } + + describe '#initialize' do + it 'accepts all args as optional fields' do + expect { configuration }.not_to raise_exception + + expect(configuration.key).to be_nil + expect(configuration.previous_keys).to be_empty + end + end + + context 'when provided key and config file' do + let!(:config_tmp_dir) { Dir.mktmpdir('config-') } + let(:credentials_config_path) { File.join(config_tmp_dir, 'credentials.yml.enc') } + let(:credentials_key) { ActiveSupport::EncryptedConfiguration.generate_key } + + after do + FileUtils.rm_f(config_tmp_dir) + end + + describe '#write' do + it 'encrypts the file using the provided key' do + encryptor = ActiveSupport::MessageEncryptor.new([credentials_key].pack('H*'), cipher: 'aes-128-gcm') + config = described_class.new(content_path: credentials_config_path, key: credentials_key) + + config.write('sample-content') + expect(encryptor.decrypt_and_verify(File.read(credentials_config_path))).to eq('sample-content') + end + end + + describe '#read' do + it 'reads yaml configuration' do + config = described_class.new(content_path: credentials_config_path, key: credentials_key) + + config.write({ foo: { bar: true } }.to_yaml) + expect(config[:foo][:bar]).to be true + end + + it 'allows referencing top level keys via dot syntax' do + config = described_class.new(content_path: credentials_config_path, key: credentials_key) + + config.write({ foo: { bar: true } }.to_yaml) + expect(config.foo[:bar]).to be true + end + end + + describe '#change' do + it 'changes yaml configuration' do + config = described_class.new(content_path: credentials_config_path, key: credentials_key) + + config.write({ foo: { bar: true } }.to_yaml) + config.change do |unencrypted_contents| + contents = YAML.safe_load(unencrypted_contents, permitted_classes: [Symbol]) + contents.merge(beef: "stew").to_yaml + end + expect(config.foo[:bar]).to be true + expect(config.beef).to eq('stew') + end + end + + context 'when provided previous_keys for rotation' do + let!(:config_tmp_dir) { Dir.mktmpdir('config-') } + let(:credential_key_original) { ActiveSupport::EncryptedConfiguration.generate_key } + let(:credential_key_latest) { ActiveSupport::EncryptedConfiguration.generate_key } + let(:config_path_original) { File.join(config_tmp_dir, 'credentials-orig.yml.enc') } + let(:config_path_latest) { File.join(config_tmp_dir, 'credentials-latest.yml.enc') } + + after do + FileUtils.rm_f(config_tmp_dir) + end + + def encryptor(key) + ActiveSupport::MessageEncryptor.new([key].pack('H*'), cipher: 'aes-128-gcm') + end + + describe '#write' do + it 'rotates the key when provided a new key' do + config1 = described_class.new(content_path: config_path_original, key: credential_key_original) + config1.write('sample-content1') + + config2 = described_class.new(content_path: config_path_latest, key: credential_key_latest, previous_keys: [credential_key_original]) + config2.write('sample-content2') + + original_key_encryptor = encryptor(credential_key_original) # can read with the initial key + latest_key_encryptor = encryptor(credential_key_latest) # can read with the new key + both_key_encryptor = encryptor(credential_key_latest) # can read with either key + both_key_encryptor.rotate([credential_key_original].pack("H*")) + + expect(original_key_encryptor.decrypt_and_verify(File.read(config_path_original))).to eq('sample-content1') + expect(both_key_encryptor.decrypt_and_verify(File.read(config_path_original))).to eq('sample-content1') + expect(latest_key_encryptor.decrypt_and_verify(File.read(config_path_latest))).to eq('sample-content2') + expect(both_key_encryptor.decrypt_and_verify(File.read(config_path_latest))).to eq('sample-content2') + expect { original_key_encryptor.decrypt_and_verify(File.read(config_path_latest)) }.to raise_error(ActiveSupport::MessageEncryptor::InvalidMessage) + end + end + + describe '#read' do + it 'supports reading using rotated config' do + described_class.new(content_path: config_path_original, key: credential_key_original).write({ foo: { bar: true } }.to_yaml) + + config = described_class.new(content_path: config_path_original, key: credential_key_latest, previous_keys: [credential_key_original]) + expect(config[:foo][:bar]).to be true + end + end + end + end +end -- GitLab From 5fe1d0925b2291617221d0f66bd342b9ea8c569b Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Thu, 1 Oct 2020 13:11:37 -0700 Subject: [PATCH 2/7] Use keygenerate to ensure encyprtion key is of correct length - Generates a new key or correct length pased on the provided base key - Updates the naming of the key param passed to EncryptedConfiguration - Adds tests for key length to the spec tests Increase cipher to 256 The 128 was the default from upstream configuration encryption but is lower then the message encryptor default. Not sure why, so we might as well go with the larger one. --- config/settings.rb | 2 +- lib/gitlab/encrypted_configuration.rb | 19 +++++--- spec/config/settings_spec.rb | 4 +- .../gitlab/encrypted_configuration_spec.rb | 44 +++++++++++++------ 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/config/settings.rb b/config/settings.rb index 3eaa7d8260867e..4eaadf0ddc0189 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -157,7 +157,7 @@ def encrypted(path) Gitlab::EncryptedConfiguration.new( content_path: Settings.absolute(path), - key: Gitlab::Application.secrets.enc_settings_key_base, + base_key: Gitlab::Application.secrets.enc_settings_key_base, previous_keys: Gitlab::Application.secrets.rotated_enc_settings_key_base || [] ) end diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index db195320a2f4dc..8001bf4c822783 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -6,11 +6,20 @@ class EncryptedConfiguration delegate_missing_to :options attr_reader :content_path, :key, :previous_keys - CIPHER = "aes-128-gcm" + CIPHER = "aes-256-gcm" + SALT = "GitLabEncryptedConfigSalt" - def initialize(content_path: nil, key: nil, previous_keys: []) + def self.generate_key(base_key) + # Because the salt is static, we want uniqueness to be coming from the base_key + # Error if the base_key is empty or suspiciously short + raise 'Base key too small' if base_key.blank? || base_key.length < 16 + + ActiveSupport::KeyGenerator.new(base_key).generate_key(SALT, ActiveSupport::MessageEncryptor.key_len(CIPHER)) + end + + def initialize(content_path: nil, base_key: nil, previous_keys: []) @content_path = Pathname.new(content_path).yield_self { |path| path.symlink? ? path.realpath : path } if content_path - @key = key + @key = self.class.generate_key(base_key) if base_key @previous_keys = previous_keys end @@ -62,11 +71,11 @@ def decrypt(contents) def encryptor return @encryptor if @encryptor - @encryptor = ActiveSupport::MessageEncryptor.new([key].pack("H*"), cipher: CIPHER) + @encryptor = ActiveSupport::MessageEncryptor.new(key, cipher: CIPHER) # Allow fallback to previous keys @previous_keys.each do |key| - @encryptor.rotate([key].pack("H*")) + @encryptor.rotate(self.class.generate_key(key)) end @encryptor diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index d83d042c8ebf36..97286ce197d8dd 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -141,11 +141,11 @@ end it 'defaults to using the enc_settings_key_base for the key' do - expect(Gitlab::EncryptedConfiguration).to receive(:new).with(hash_including(key: Gitlab::Application.secrets.enc_settings_key_base)) + expect(Gitlab::EncryptedConfiguration).to receive(:new).with(hash_including(base_key: Gitlab::Application.secrets.enc_settings_key_base)) Settings.encrypted('tmp/tests/test.enc') end - it 'defaults the configpath within the rails root' do + it 'defaults the config path within the rails root' do expect(Settings.encrypted('tmp/tests/test.enc').content_path.fnmatch?(File.join(Rails.root, '**'))).to be true end diff --git a/spec/lib/gitlab/encrypted_configuration_spec.rb b/spec/lib/gitlab/encrypted_configuration_spec.rb index de41a522baa97d..969343537d2f47 100644 --- a/spec/lib/gitlab/encrypted_configuration_spec.rb +++ b/spec/lib/gitlab/encrypted_configuration_spec.rb @@ -12,12 +12,28 @@ expect(configuration.key).to be_nil expect(configuration.previous_keys).to be_empty end + + it 'generates 32 byte key when provided a larger base key' do + configuration = described_class.new(base_key: 'A' * 64) + + expect(configuration.key.bytesize).to eq 32 + end + + it 'generates 32 byte key when provided a smaller base key' do + configuration = described_class.new(base_key: 'A' * 16) + + expect(configuration.key.bytesize).to eq 32 + end + + it 'throws an error when the base key is too small' do + expect { described_class.new(base_key: 'A' * 12) }.to raise_error 'Base key too small' + end end context 'when provided key and config file' do let!(:config_tmp_dir) { Dir.mktmpdir('config-') } let(:credentials_config_path) { File.join(config_tmp_dir, 'credentials.yml.enc') } - let(:credentials_key) { ActiveSupport::EncryptedConfiguration.generate_key } + let(:credentials_key) { SecureRandom.hex(64) } after do FileUtils.rm_f(config_tmp_dir) @@ -25,8 +41,8 @@ describe '#write' do it 'encrypts the file using the provided key' do - encryptor = ActiveSupport::MessageEncryptor.new([credentials_key].pack('H*'), cipher: 'aes-128-gcm') - config = described_class.new(content_path: credentials_config_path, key: credentials_key) + encryptor = ActiveSupport::MessageEncryptor.new(Gitlab::EncryptedConfiguration.generate_key(credentials_key), cipher: 'aes-256-gcm') + config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) config.write('sample-content') expect(encryptor.decrypt_and_verify(File.read(credentials_config_path))).to eq('sample-content') @@ -35,14 +51,14 @@ describe '#read' do it 'reads yaml configuration' do - config = described_class.new(content_path: credentials_config_path, key: credentials_key) + config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) config.write({ foo: { bar: true } }.to_yaml) expect(config[:foo][:bar]).to be true end it 'allows referencing top level keys via dot syntax' do - config = described_class.new(content_path: credentials_config_path, key: credentials_key) + config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) config.write({ foo: { bar: true } }.to_yaml) expect(config.foo[:bar]).to be true @@ -51,7 +67,7 @@ describe '#change' do it 'changes yaml configuration' do - config = described_class.new(content_path: credentials_config_path, key: credentials_key) + config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) config.write({ foo: { bar: true } }.to_yaml) config.change do |unencrypted_contents| @@ -65,8 +81,8 @@ context 'when provided previous_keys for rotation' do let!(:config_tmp_dir) { Dir.mktmpdir('config-') } - let(:credential_key_original) { ActiveSupport::EncryptedConfiguration.generate_key } - let(:credential_key_latest) { ActiveSupport::EncryptedConfiguration.generate_key } + let(:credential_key_original) { SecureRandom.hex(64) } + let(:credential_key_latest) { SecureRandom.hex(64) } let(:config_path_original) { File.join(config_tmp_dir, 'credentials-orig.yml.enc') } let(:config_path_latest) { File.join(config_tmp_dir, 'credentials-latest.yml.enc') } @@ -75,21 +91,21 @@ end def encryptor(key) - ActiveSupport::MessageEncryptor.new([key].pack('H*'), cipher: 'aes-128-gcm') + ActiveSupport::MessageEncryptor.new(Gitlab::EncryptedConfiguration.generate_key(key), cipher: 'aes-256-gcm') end describe '#write' do it 'rotates the key when provided a new key' do - config1 = described_class.new(content_path: config_path_original, key: credential_key_original) + config1 = described_class.new(content_path: config_path_original, base_key: credential_key_original) config1.write('sample-content1') - config2 = described_class.new(content_path: config_path_latest, key: credential_key_latest, previous_keys: [credential_key_original]) + config2 = described_class.new(content_path: config_path_latest, base_key: credential_key_latest, previous_keys: [credential_key_original]) config2.write('sample-content2') original_key_encryptor = encryptor(credential_key_original) # can read with the initial key latest_key_encryptor = encryptor(credential_key_latest) # can read with the new key both_key_encryptor = encryptor(credential_key_latest) # can read with either key - both_key_encryptor.rotate([credential_key_original].pack("H*")) + both_key_encryptor.rotate(Gitlab::EncryptedConfiguration.generate_key(credential_key_original)) expect(original_key_encryptor.decrypt_and_verify(File.read(config_path_original))).to eq('sample-content1') expect(both_key_encryptor.decrypt_and_verify(File.read(config_path_original))).to eq('sample-content1') @@ -101,9 +117,9 @@ def encryptor(key) describe '#read' do it 'supports reading using rotated config' do - described_class.new(content_path: config_path_original, key: credential_key_original).write({ foo: { bar: true } }.to_yaml) + described_class.new(content_path: config_path_original, base_key: credential_key_original).write({ foo: { bar: true } }.to_yaml) - config = described_class.new(content_path: config_path_original, key: credential_key_latest, previous_keys: [credential_key_original]) + config = described_class.new(content_path: config_path_original, base_key: credential_key_latest, previous_keys: [credential_key_original]) expect(config[:foo][:bar]).to be true end end -- GitLab From 3159beae35a13ff24ee89dbe2698d1f076f51ca6 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Thu, 19 Nov 2020 13:03:58 -0800 Subject: [PATCH 3/7] Bring in changes from the ldap impelmentation MR This adds: - Path for storing encrypted settings in config - optional base token generation - Additional error handling for missing config and keys --- config/gitlab.yml.example | 4 ++ config/initializers/01_secret_token.rb | 3 ++ config/initializers/1_settings.rb | 9 +++-- config/settings.rb | 2 - lib/gitlab/encrypted_configuration.rb | 32 +++++++++++++++- spec/config/settings_spec.rb | 3 +- spec/initializers/secret_token_spec.rb | 27 +++++++++++++- .../gitlab/encrypted_configuration_spec.rb | 37 ++++++++++++++----- 8 files changed, 97 insertions(+), 20 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index ae9475aa60d01d..5db0e22ed10228 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -1042,6 +1042,10 @@ production: &base shared: # path: /mnt/gitlab # Default: shared + # Encrypted Settings configuration + encrypted_settings: + # path: /mnt/gitlab/encrypted_settings # Default: shared/encrypted_settings + # Gitaly settings gitaly: # Path to the directory containing Gitaly client executables. diff --git a/config/initializers/01_secret_token.rb b/config/initializers/01_secret_token.rb index 5949f463457ced..430cca0b6b220c 100644 --- a/config/initializers/01_secret_token.rb +++ b/config/initializers/01_secret_token.rb @@ -34,6 +34,9 @@ def create_tokens openid_connect_signing_key: generate_new_rsa_private_key } + # enc_settings_key_base is optional for now + defaults[:enc_settings_key_base] = generate_new_secure_token if ENV['GITLAB_GENERATE_ENC_SETTINGS_KEY_BASE'] + missing_secrets = set_missing_keys(defaults) write_secrets_yml(missing_secrets) unless missing_secrets.empty? diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 723937c59879ae..f5d1ac1127ee73 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -3,6 +3,12 @@ require_relative '../smime_signature_settings' # Default settings +Settings['shared'] ||= Settingslogic.new({}) +Settings.shared['path'] = Settings.absolute(Settings.shared['path'] || "shared") + +Settings['encrypted_settings'] ||= Settingslogic.new({}) +Settings.encrypted_settings['path'] = Settings.absolute(Settings.encrypted_settings['path'] || File.join(Settings.shared['path'], "encrypted_settings")) + Settings['ldap'] ||= Settingslogic.new({}) Settings.ldap['enabled'] = false if Settings.ldap['enabled'].nil? Settings.ldap['prevent_ldap_sign_in'] = false if Settings.ldap['prevent_ldap_sign_in'].blank? @@ -140,9 +146,6 @@ Settings.omniauth.providers << Settingslogic.new({ 'name' => 'group_saml' }) end -Settings['shared'] ||= Settingslogic.new({}) -Settings.shared['path'] = Settings.absolute(Settings.shared['path'] || "shared") - Settings['issues_tracker'] ||= {} # diff --git a/config/settings.rb b/config/settings.rb index 4eaadf0ddc0189..2207f300f82403 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -153,8 +153,6 @@ def attr_encrypted_db_key_base end def encrypted(path) - return Gitlab::EncryptedConfiguration.new unless Gitlab::Application.secrets.enc_settings_key_base - Gitlab::EncryptedConfiguration.new( content_path: Settings.absolute(path), base_key: Gitlab::Application.secrets.enc_settings_key_base, diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index 8001bf4c822783..f1d4a9d72f8f5d 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -9,6 +9,18 @@ class EncryptedConfiguration CIPHER = "aes-256-gcm" SALT = "GitLabEncryptedConfigSalt" + class MissingKeyError < RuntimeError + def initialize(msg = "Missing encryption key to encrypt/decrypt file with.") + super + end + end + + class InvalidConfigError < RuntimeError + def initialize(msg = "Content was not a valid yml config file") + super + end + end + def self.generate_key(base_key) # Because the salt is static, we want uniqueness to be coming from the base_key # Error if the base_key is empty or suspiciously short @@ -23,8 +35,12 @@ def initialize(content_path: nil, base_key: nil, previous_keys: []) @previous_keys = previous_keys end + def active? + content_path&.exist? + end + def read - if !key.nil? && content_path&.exist? + if active? decrypt content_path.binread else "" @@ -45,7 +61,13 @@ def write(contents) end def config - @config ||= deserialize(read).deep_symbolize_keys + return @config if @config + + contents = deserialize(read) + + raise InvalidConfigError.new unless contents.is_a?(Hash) + + @config = contents.deep_symbolize_keys end def change(&block) @@ -61,10 +83,12 @@ def writing(contents) end def encrypt(contents) + handle_missing_key! encryptor.encrypt_and_sign contents end def decrypt(contents) + handle_missing_key! encryptor.decrypt_and_verify contents end @@ -89,5 +113,9 @@ def options def deserialize(contents) YAML.safe_load(contents, permitted_classes: [Symbol]).presence || {} end + + def handle_missing_key! + raise MissingKeyError.new if @key.nil? + end end end diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index 97286ce197d8dd..02863b192093c3 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -151,8 +151,7 @@ it 'returns empty encrypted config when a key has not been set' do allow(Gitlab::Application.secrets).to receive(:enc_settings_key_base).and_return(nil) - expect(Gitlab::EncryptedConfiguration).to receive(:new).with(no_args) - Settings.encrypted('tmp/tests/test.enc') + expect(Settings.encrypted('tmp/tests/test.enc').read).to be_empty end end end diff --git a/spec/initializers/secret_token_spec.rb b/spec/initializers/secret_token_spec.rb index 51a14aca4adfdf..d12ae7b2899675 100644 --- a/spec/initializers/secret_token_spec.rb +++ b/spec/initializers/secret_token_spec.rb @@ -24,7 +24,7 @@ describe 'ensure acknowledged secrets in any installations' do let(:acknowledged_secrets) do - %w[secret_key_base otp_key_base db_key_base openid_connect_signing_key] + %w[secret_key_base otp_key_base db_key_base openid_connect_signing_key enc_settings_key_base rotated_enc_settings_key_base] end it 'does not allow to add a new secret without a proper handling' do @@ -90,6 +90,7 @@ expect(new_secrets['otp_key_base']).to eq(secrets.otp_key_base) expect(new_secrets['db_key_base']).to eq(secrets.db_key_base) expect(new_secrets['openid_connect_signing_key']).to eq(secrets.openid_connect_signing_key) + expect(new_secrets['enc_settings_key_base']).to eq(secrets.enc_settings_key_base) end create_tokens @@ -106,6 +107,7 @@ before do secrets.db_key_base = 'db_key_base' secrets.openid_connect_signing_key = 'openid_connect_signing_key' + secrets.enc_settings_key_base = 'enc_settings_key_base' allow(File).to receive(:exist?).with('.secret').and_return(true) stub_file_read('.secret', content: 'file_key') @@ -158,6 +160,7 @@ expect(secrets.otp_key_base).to eq('otp_key_base') expect(secrets.db_key_base).to eq('db_key_base') expect(secrets.openid_connect_signing_key).to eq('openid_connect_signing_key') + expect(secrets.enc_settings_key_base).to eq('enc_settings_key_base') end it 'deletes the .secret file' do @@ -208,12 +211,34 @@ create_tokens end end + + context 'when rotated_enc_settings_key_base does not exist' do + before do + secrets.secret_key_base = 'secret_key_base' + secrets.otp_key_base = 'otp_key_base' + secrets.openid_connect_signing_key = 'openid_connect_signing_key' + secrets.enc_settings_key_base = 'enc_settings_key_base' + end + + it 'does not warns about the missing secrets' do + expect(self).not_to receive(:warn_missing_secret).with('rotated_enc_settings_key_base') + + create_tokens + end + + it 'does not update secrets.yml' do + expect(File).not_to receive(:write) + + create_tokens + end + end end context 'when db_key_base is blank but exists in secrets.yml' do before do secrets.otp_key_base = 'otp_key_base' secrets.secret_key_base = 'secret_key_base' + secrets.enc_settings_key_base = 'enc_settings_key_base' yaml_secrets = secrets.to_h.stringify_keys.merge('db_key_base' => '<%= an_erb_expression %>') allow(File).to receive(:exist?).with('.secret').and_return(false) diff --git a/spec/lib/gitlab/encrypted_configuration_spec.rb b/spec/lib/gitlab/encrypted_configuration_spec.rb index 969343537d2f47..69c26b8ceb34b9 100644 --- a/spec/lib/gitlab/encrypted_configuration_spec.rb +++ b/spec/lib/gitlab/encrypted_configuration_spec.rb @@ -5,6 +5,12 @@ RSpec.describe Gitlab::EncryptedConfiguration do subject(:configuration) { described_class.new } + let!(:config_tmp_dir) { Dir.mktmpdir('config-') } + + after do + FileUtils.rm_f(config_tmp_dir) + end + describe '#initialize' do it 'accepts all args as optional fields' do expect { configuration }.not_to raise_exception @@ -30,15 +36,24 @@ end end + context 'when provided a config file but no key' do + let(:config_path) { File.join(config_tmp_dir, 'credentials.yml.enc') } + + it 'throws an error when writing without a key' do + expect { described_class.new(content_path: config_path).write('test') }.to raise_error Gitlab::EncryptedConfiguration::MissingKeyError + end + + it 'throws an error when reading without a key' do + config = described_class.new(content_path: config_path) + File.write(config_path, 'test') + expect { config.read }.to raise_error Gitlab::EncryptedConfiguration::MissingKeyError + end + end + context 'when provided key and config file' do - let!(:config_tmp_dir) { Dir.mktmpdir('config-') } let(:credentials_config_path) { File.join(config_tmp_dir, 'credentials.yml.enc') } let(:credentials_key) { SecureRandom.hex(64) } - after do - FileUtils.rm_f(config_tmp_dir) - end - describe '#write' do it 'encrypts the file using the provided key' do encryptor = ActiveSupport::MessageEncryptor.new(Gitlab::EncryptedConfiguration.generate_key(credentials_key), cipher: 'aes-256-gcm') @@ -63,6 +78,13 @@ config.write({ foo: { bar: true } }.to_yaml) expect(config.foo[:bar]).to be true end + + it 'throws a custom error when deferencing an invalid key map config' do + config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) + + config.write("stringcontent") + expect { config[:foo] }.to raise_error Gitlab::EncryptedConfiguration::InvalidConfigError + end end describe '#change' do @@ -80,16 +102,11 @@ end context 'when provided previous_keys for rotation' do - let!(:config_tmp_dir) { Dir.mktmpdir('config-') } let(:credential_key_original) { SecureRandom.hex(64) } let(:credential_key_latest) { SecureRandom.hex(64) } let(:config_path_original) { File.join(config_tmp_dir, 'credentials-orig.yml.enc') } let(:config_path_latest) { File.join(config_tmp_dir, 'credentials-latest.yml.enc') } - after do - FileUtils.rm_f(config_tmp_dir) - end - def encryptor(key) ActiveSupport::MessageEncryptor.new(Gitlab::EncryptedConfiguration.generate_key(key), cipher: 'aes-256-gcm') end -- GitLab From 1e8d0c0f77941cc9303451f52b08a689cc7444df Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sun, 22 Nov 2020 05:48:20 +0000 Subject: [PATCH 4/7] Apply 1 suggestion(s) to 1 file(s) --- spec/initializers/secret_token_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/initializers/secret_token_spec.rb b/spec/initializers/secret_token_spec.rb index d12ae7b2899675..b8d49256eaf8b6 100644 --- a/spec/initializers/secret_token_spec.rb +++ b/spec/initializers/secret_token_spec.rb @@ -220,7 +220,7 @@ secrets.enc_settings_key_base = 'enc_settings_key_base' end - it 'does not warns about the missing secrets' do + it 'does not warn about the missing secrets' do expect(self).not_to receive(:warn_missing_secret).with('rotated_enc_settings_key_base') create_tokens -- GitLab From d33611a6db7ada9dfc5771941f7443ee8d064ccb Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Mon, 23 Nov 2020 14:27:56 -0800 Subject: [PATCH 5/7] Change the secret key name From `enc_settings_key_base` to `encrypted_settings_key_base` --- config/initializers/01_secret_token.rb | 4 ++-- config/settings.rb | 4 ++-- doc/development/application_secrets.md | 2 +- spec/config/settings_spec.rb | 8 ++++---- spec/initializers/secret_token_spec.rb | 16 ++++++++-------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/config/initializers/01_secret_token.rb b/config/initializers/01_secret_token.rb index 430cca0b6b220c..d7e725477ebb08 100644 --- a/config/initializers/01_secret_token.rb +++ b/config/initializers/01_secret_token.rb @@ -34,8 +34,8 @@ def create_tokens openid_connect_signing_key: generate_new_rsa_private_key } - # enc_settings_key_base is optional for now - defaults[:enc_settings_key_base] = generate_new_secure_token if ENV['GITLAB_GENERATE_ENC_SETTINGS_KEY_BASE'] + # encrypted_settings_key_base is optional for now + defaults[:encrypted_settings_key_base] = generate_new_secure_token if ENV['GITLAB_GENERATE_ENCRYPTED_SETTINGS_KEY_BASE'] missing_secrets = set_missing_keys(defaults) write_secrets_yml(missing_secrets) unless missing_secrets.empty? diff --git a/config/settings.rb b/config/settings.rb index 2207f300f82403..f49d37bd8c6822 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -155,8 +155,8 @@ def attr_encrypted_db_key_base def encrypted(path) Gitlab::EncryptedConfiguration.new( content_path: Settings.absolute(path), - base_key: Gitlab::Application.secrets.enc_settings_key_base, - previous_keys: Gitlab::Application.secrets.rotated_enc_settings_key_base || [] + base_key: Gitlab::Application.secrets.encrypted_settings_key_base, + previous_keys: Gitlab::Application.secrets.rotated_encrypted_settings_key_base || [] ) end diff --git a/doc/development/application_secrets.md b/doc/development/application_secrets.md index 59d0a54a153c42..6e70b0dc2271af 100644 --- a/doc/development/application_secrets.md +++ b/doc/development/application_secrets.md @@ -16,7 +16,7 @@ This page is a development guide for application secrets. | `otp_key_base` | The base key for One Time Passwords, described in [User management](../raketasks/user_management.md#rotate-two-factor-authentication-encryption-key) | |`db_key_base` | The base key to encrypt the data for `attr_encrypted` columns | |`openid_connect_signing_key` | The singing key for OpenID Connect | -| `enc_settings_key_base` | The base key to encrypt settings files with | +| `encrypted_settings_key_base` | The base key to encrypt settings files with | ## Where the secrets are stored diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index 02863b192093c3..7fc9c527d18f07 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -137,11 +137,11 @@ describe '.encrypted' do before do - allow(Gitlab::Application.secrets).to receive(:enc_settings_key_base).and_return(SecureRandom.hex(64)) + allow(Gitlab::Application.secrets).to receive(:encryped_settings_key_base).and_return(SecureRandom.hex(64)) end - it 'defaults to using the enc_settings_key_base for the key' do - expect(Gitlab::EncryptedConfiguration).to receive(:new).with(hash_including(base_key: Gitlab::Application.secrets.enc_settings_key_base)) + it 'defaults to using the encrypted_settings_key_base for the key' do + expect(Gitlab::EncryptedConfiguration).to receive(:new).with(hash_including(base_key: Gitlab::Application.secrets.encrypted_settings_key_base)) Settings.encrypted('tmp/tests/test.enc') end @@ -150,7 +150,7 @@ end it 'returns empty encrypted config when a key has not been set' do - allow(Gitlab::Application.secrets).to receive(:enc_settings_key_base).and_return(nil) + allow(Gitlab::Application.secrets).to receive(:encrypted_settings_key_base).and_return(nil) expect(Settings.encrypted('tmp/tests/test.enc').read).to be_empty end end diff --git a/spec/initializers/secret_token_spec.rb b/spec/initializers/secret_token_spec.rb index b8d49256eaf8b6..ab16dbad3fc4dd 100644 --- a/spec/initializers/secret_token_spec.rb +++ b/spec/initializers/secret_token_spec.rb @@ -24,7 +24,7 @@ describe 'ensure acknowledged secrets in any installations' do let(:acknowledged_secrets) do - %w[secret_key_base otp_key_base db_key_base openid_connect_signing_key enc_settings_key_base rotated_enc_settings_key_base] + %w[secret_key_base otp_key_base db_key_base openid_connect_signing_key encrypted_settings_key_base rotated_encrypted_settings_key_base] end it 'does not allow to add a new secret without a proper handling' do @@ -90,7 +90,7 @@ expect(new_secrets['otp_key_base']).to eq(secrets.otp_key_base) expect(new_secrets['db_key_base']).to eq(secrets.db_key_base) expect(new_secrets['openid_connect_signing_key']).to eq(secrets.openid_connect_signing_key) - expect(new_secrets['enc_settings_key_base']).to eq(secrets.enc_settings_key_base) + expect(new_secrets['encrypted_settings_key_base']).to eq(secrets.encrypted_settings_key_base) end create_tokens @@ -107,7 +107,7 @@ before do secrets.db_key_base = 'db_key_base' secrets.openid_connect_signing_key = 'openid_connect_signing_key' - secrets.enc_settings_key_base = 'enc_settings_key_base' + secrets.encrypted_settings_key_base = 'encrypted_settings_key_base' allow(File).to receive(:exist?).with('.secret').and_return(true) stub_file_read('.secret', content: 'file_key') @@ -160,7 +160,7 @@ expect(secrets.otp_key_base).to eq('otp_key_base') expect(secrets.db_key_base).to eq('db_key_base') expect(secrets.openid_connect_signing_key).to eq('openid_connect_signing_key') - expect(secrets.enc_settings_key_base).to eq('enc_settings_key_base') + expect(secrets.encrypted_settings_key_base).to eq('encrypted_settings_key_base') end it 'deletes the .secret file' do @@ -212,16 +212,16 @@ end end - context 'when rotated_enc_settings_key_base does not exist' do + context 'when rotated_encrypted_settings_key_base does not exist' do before do secrets.secret_key_base = 'secret_key_base' secrets.otp_key_base = 'otp_key_base' secrets.openid_connect_signing_key = 'openid_connect_signing_key' - secrets.enc_settings_key_base = 'enc_settings_key_base' + secrets.encrypted_settings_key_base = 'encrypted_settings_key_base' end it 'does not warn about the missing secrets' do - expect(self).not_to receive(:warn_missing_secret).with('rotated_enc_settings_key_base') + expect(self).not_to receive(:warn_missing_secret).with('rotated_encrypted_settings_key_base') create_tokens end @@ -238,7 +238,7 @@ before do secrets.otp_key_base = 'otp_key_base' secrets.secret_key_base = 'secret_key_base' - secrets.enc_settings_key_base = 'enc_settings_key_base' + secrets.encrypted_settings_key_base = 'encrypted_settings_key_base' yaml_secrets = secrets.to_h.stringify_keys.merge('db_key_base' => '<%= an_erb_expression %>') allow(File).to receive(:exist?).with('.secret').and_return(false) -- GitLab From add38cc7c6c79e6f127dea80f1171a0b18d53885 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Mon, 23 Nov 2020 14:39:06 -0800 Subject: [PATCH 6/7] Cleanup encrypted path syntax --- config/initializers/1_settings.rb | 3 ++- config/settings.rb | 2 +- spec/config/settings_spec.rb | 4 ---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index f5d1ac1127ee73..2da7b50cbfc8e8 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -7,7 +7,8 @@ Settings.shared['path'] = Settings.absolute(Settings.shared['path'] || "shared") Settings['encrypted_settings'] ||= Settingslogic.new({}) -Settings.encrypted_settings['path'] = Settings.absolute(Settings.encrypted_settings['path'] || File.join(Settings.shared['path'], "encrypted_settings")) +Settings.encrypted_settings['path'] ||= File.join(Settings.shared['path'], "encrypted_settings") +Settings.encrypted_settings['path'] = Settings.absolute(Settings.encrypted_settings['path']) Settings['ldap'] ||= Settingslogic.new({}) Settings.ldap['enabled'] = false if Settings.ldap['enabled'].nil? diff --git a/config/settings.rb b/config/settings.rb index f49d37bd8c6822..3369f2a448053b 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -154,7 +154,7 @@ def attr_encrypted_db_key_base def encrypted(path) Gitlab::EncryptedConfiguration.new( - content_path: Settings.absolute(path), + content_path: path, base_key: Gitlab::Application.secrets.encrypted_settings_key_base, previous_keys: Gitlab::Application.secrets.rotated_encrypted_settings_key_base || [] ) diff --git a/spec/config/settings_spec.rb b/spec/config/settings_spec.rb index 7fc9c527d18f07..6525ae653c98ef 100644 --- a/spec/config/settings_spec.rb +++ b/spec/config/settings_spec.rb @@ -145,10 +145,6 @@ Settings.encrypted('tmp/tests/test.enc') end - it 'defaults the config path within the rails root' do - expect(Settings.encrypted('tmp/tests/test.enc').content_path.fnmatch?(File.join(Rails.root, '**'))).to be true - end - it 'returns empty encrypted config when a key has not been set' do allow(Gitlab::Application.secrets).to receive(:encrypted_settings_key_base).and_return(nil) expect(Settings.encrypted('tmp/tests/test.enc').read).to be_empty -- GitLab From e6cc5f2aa4979732b34a72fb5d004a02c2d76af0 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Mon, 23 Nov 2020 22:57:44 +0000 Subject: [PATCH 7/7] Apply 6 suggestion(s) to 2 file(s) --- lib/gitlab/encrypted_configuration.rb | 10 +++++----- spec/lib/gitlab/encrypted_configuration_spec.rb | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index f1d4a9d72f8f5d..fe49af3ab33757 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -41,7 +41,7 @@ def active? def read if active? - decrypt content_path.binread + decrypt(content_path.binread) else "" end @@ -55,7 +55,7 @@ def write(contents) File.open(temp_file.path, 'wb') do |file| file.write(encrypt(contents)) end - FileUtils.mv temp_file.path, content_path + FileUtils.mv(temp_file.path, content_path) ensure temp_file&.unlink end @@ -71,7 +71,7 @@ def config end def change(&block) - writing read, &block + writing(read, &block) end private @@ -84,12 +84,12 @@ def writing(contents) def encrypt(contents) handle_missing_key! - encryptor.encrypt_and_sign contents + encryptor.encrypt_and_sign(contents) end def decrypt(contents) handle_missing_key! - encryptor.decrypt_and_verify contents + encryptor.decrypt_and_verify(contents) end def encryptor diff --git a/spec/lib/gitlab/encrypted_configuration_spec.rb b/spec/lib/gitlab/encrypted_configuration_spec.rb index 69c26b8ceb34b9..eadc2cf71a78d6 100644 --- a/spec/lib/gitlab/encrypted_configuration_spec.rb +++ b/spec/lib/gitlab/encrypted_configuration_spec.rb @@ -79,7 +79,7 @@ expect(config.foo[:bar]).to be true end - it 'throws a custom error when deferencing an invalid key map config' do + it 'throws a custom error when referencing an invalid key map config' do config = described_class.new(content_path: credentials_config_path, base_key: credentials_key) config.write("stringcontent") -- GitLab