From 77002bf6fca63865197f96e6225cfea953a54e2c Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Wed, 30 Aug 2017 19:24:05 -0700 Subject: [PATCH 01/16] first (broken) commit --- dojo/cli.py | 9 +++++++-- dojo/encrypt.py | 30 ++++++++++++++++++++++++++++++ dojo/run.py | 31 +++++++++++-------------------- 3 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 dojo/encrypt.py diff --git a/dojo/cli.py b/dojo/cli.py index 4be8ecc..02e4af4 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -2,6 +2,7 @@ import click import logging from .run import Entrypoint +from .encrypt import Encrypt @click.command(help='Run a job') @@ -9,9 +10,13 @@ from .run import Entrypoint @click.option('--runner', default=None, help='specify a runner for the job') @click.option('--config', default='config', help='path to directory containing configuration files to be merged') @click.option('--env', default='development', help='environment used to select configuration and secrets') +@click.option('--encrypt', is_flag=True, help='encrypt your project secrets') @click.pass_context -def cli(context, name, runner, config, env): +def cli(context, name, runner, config, env, encrypt): if context.obj is None: context.obj = {} logging.getLogger().setLevel(logging.INFO) - Entrypoint().run(name, runner, config, env) + if encrypt: + Encrypt().run(config, env) + else: + Entrypoint().run(name, runner, config, env) diff --git a/dojo/encrypt.py b/dojo/encrypt.py new file mode 100644 index 0000000..a9e6809 --- /dev/null +++ b/dojo/encrypt.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import os +import click + +from cryptography.fernet import Fernet + +class Encrypt(object): + + def run(self, config, env): + encrypt_key = os.environ.get('DOJO_DECRYPT_KEY') + if not encrypt_key and click.confirm('No key found in your environment variables. Would you like to generate a key?'): + encrypt_key = Fernet.generate_key() + click.echo(click.style('Your Key: %s (keep this safe)', fg='red', bold=True) % encrypt_key) + env_json_secrets_path = os.path.join(config, 'secrets.%s.json' % (env, )) + env_json_secrets = self._read_file(env_json_secrets_path) + if not env_json_secrets: + raise ValueError('File %s does not exist or is empty.' % env_json_secrets_path) + else: + fernet = Fernet(encrypt_key) + token = fernet.encrypt(env_json_secrets) + with open(env_json_secrets_path + '.enc', 'w') as secret_file: + secret_file.write(token) + + def _read_file(self, path): + if os.path.isfile(path): + with open(path, 'r') as f: + return f.read() + else: + return {} \ No newline at end of file diff --git a/dojo/run.py b/dojo/run.py index ba88cf3..39b01ce 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -8,6 +8,7 @@ import fnmatch import subprocess from datetime import datetime +from cryptography.fernet import Fernet from .util import deep_merge @@ -31,25 +32,15 @@ class Entrypoint(object): config = deep_merge(base_config, env_config) # Build secrets by decrypting available EJSONs. - env_ejson_secrets_path = os.path.join(config, 'secrets.%s.ejson' % (env, )) - env_ejson_secrets = self._read_json(env_ejson_secrets_path) - ejson_public_key = env_ejson_secrets['_public_key'] - ejson_private_key_path = os.path.join('/opt/ejson/keys/%s' % (ejson_public_key, )) - if not os.path.isfile(ejson_private_key_path): - if not os.environ.get('EJSON_PRIVATE_KEY'): - raise ValueError('ENV[EJSON_PRIVATE_KEY] must be set or %s must exist containing it.' % (ejson_private_key_path, )) - else: - ejson_private_key_dir = os.path.dirname(ejson_private_key_path) - if not os.path.exists(ejson_private_key_dir): - os.makedirs(ejson_private_key_dir) - with open(ejson_private_key_path, 'w') as f: - f.write(os.environ['EJSON_PRIVATE_KEY']) - print('%s written.' % (ejson_private_key_path, )) + env_json_secrets_path = os.path.join(base_config_path, 'secrets.%s.json.enc' % (env, )) + decrypt_key = os.environ.get('DOJO_DECRYPT_KEY') + if not decrypt_key: + raise ValueError('ENV[DOJO_DECRYPT_KEY] must be set.') try: - out = subprocess.check_output(['ejson', 'decrypt', env_ejson_secrets_path], stderr=subprocess.STDOUT) - out = '\n'.join([line for line in out.split('\n') if not line.startswith('warning:')]) - except subprocess.CalledProcessError as e: - raise ValueError(e.output) + fernet = Fernet(decrypt_key) + out = fernet.decrypt(env_json_secrets_path) + except ValueError as e: + raise ValueError(e) try: secrets = json.loads(out) except ValueError as e: @@ -102,10 +93,10 @@ class Entrypoint(object): return job_class(job_config, job_secrets) - def _read_json(self, path): + def _read_file(self, path): if os.path.isfile(path): with open(path, 'r') as f: - return json.loads(f.read()) + return f.read() else: return {} -- GitLab From b4de2a2a92b7bbb5df04375874ad932e3bc2d22b Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:05:40 -0700 Subject: [PATCH 02/16] cleaned up the secrets class, added test coverage, moved decrypt to the secrets file --- dojo/cli.py | 4 ++-- dojo/run.py | 18 ++++-------------- dojo/{encrypt.py => secrets.py} | 28 ++++++++++++++++++++++++---- tests/conftest.py | 7 +++++++ tests/test_secrets.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 20 deletions(-) rename dojo/{encrypt.py => secrets.py} (51%) create mode 100644 tests/test_secrets.py diff --git a/dojo/cli.py b/dojo/cli.py index 02e4af4..fbe68a7 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -2,7 +2,7 @@ import click import logging from .run import Entrypoint -from .encrypt import Encrypt +from .secrets import Secrets @click.command(help='Run a job') @@ -17,6 +17,6 @@ def cli(context, name, runner, config, env, encrypt): context.obj = {} logging.getLogger().setLevel(logging.INFO) if encrypt: - Encrypt().run(config, env) + Secrets().encrypt(config, env) else: Entrypoint().run(name, runner, config, env) diff --git a/dojo/run.py b/dojo/run.py index 39b01ce..f1645b6 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -8,7 +8,8 @@ import fnmatch import subprocess from datetime import datetime -from cryptography.fernet import Fernet +from .secrets import Secrets + from .util import deep_merge @@ -31,20 +32,9 @@ class Entrypoint(object): env_config = self._read_yaml(env_config_path) or {} config = deep_merge(base_config, env_config) - # Build secrets by decrypting available EJSONs. + # Build secrets by decrypting available JSONs env_json_secrets_path = os.path.join(base_config_path, 'secrets.%s.json.enc' % (env, )) - decrypt_key = os.environ.get('DOJO_DECRYPT_KEY') - if not decrypt_key: - raise ValueError('ENV[DOJO_DECRYPT_KEY] must be set.') - try: - fernet = Fernet(decrypt_key) - out = fernet.decrypt(env_json_secrets_path) - except ValueError as e: - raise ValueError(e) - try: - secrets = json.loads(out) - except ValueError as e: - raise ValueError(e, out) + secrets = Secrets.decrypt(env_json_secrets_path) # Build the job. job = self._build_job(name, config, secrets, runner) diff --git a/dojo/encrypt.py b/dojo/secrets.py similarity index 51% rename from dojo/encrypt.py rename to dojo/secrets.py index a9e6809..3b4d253 100644 --- a/dojo/encrypt.py +++ b/dojo/secrets.py @@ -1,13 +1,14 @@ from __future__ import absolute_import, print_function, unicode_literals import os +import json import click from cryptography.fernet import Fernet -class Encrypt(object): +class Secrets(object): - def run(self, config, env): + def encrypt(self, config, env): encrypt_key = os.environ.get('DOJO_DECRYPT_KEY') if not encrypt_key and click.confirm('No key found in your environment variables. Would you like to generate a key?'): encrypt_key = Fernet.generate_key() @@ -19,8 +20,27 @@ class Encrypt(object): else: fernet = Fernet(encrypt_key) token = fernet.encrypt(env_json_secrets) - with open(env_json_secrets_path + '.enc', 'w') as secret_file: - secret_file.write(token) + with open(env_json_secrets_path + '.enc', 'w') as encrypted_file: + encrypted_file.write(token) + return encrypted_file + + def decrypt(self, json_secrets_path): + encrypt_key = os.environ.get('DOJO_DECRYPT_KEY') + if not encrypt_key or not json_secrets_path: + raise ValueError('Missing a requirement for decrypting. Secrets path or DOJO_DECRYPT_KEY.') + secrets_file = self._read_file(json_secrets_path).encode() + try: + fernet = Fernet(encrypt_key) + out = fernet.decrypt(secrets_file) + #TODO proper error handling? https://cryptography.io/en/latest/fernet/#cryptography.fernet.Fernet.decrypt + except ValueError as e: + raise ValueError(e) + try: + secrets = json.loads(out) + except ValueError as e: + raise ValueError(e, out) + + return secrets def _read_file(self, path): if os.path.isfile(path): diff --git a/tests/conftest.py b/tests/conftest.py index bc673a4..7a26586 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,3 +8,10 @@ import tempfile def temp_dir(): with tempfile.TemporaryDirectory() as d: yield d + +@pytest.fixture(scope='session') +def temp_config(tmpdir_factory): + config_dir = tmpdir_factory.mktemp('config') + json_file = config_dir.join('secrets.development.json') + json_file.write('{"some":"test"}') + return config_dir \ No newline at end of file diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..b234e3c --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,28 @@ +from __future__ import absolute_import, print_function, unicode_literals + +import os +import pytest + +from dojo.secrets import Secrets + + +class TestSecrets(object): + + def test_encrypt(self, temp_config, monkeypatch): + monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') + s = Secrets() + encrypted_file = s.encrypt(temp_config.strpath, 'development') + file = open(encrypted_file.name,'r').read() + + temp_file = os.path.join(temp_config.strpath, 'secrets.development.json.enc') + + assert os.path.isfile(temp_file) + assert len(file) == 100 + + #TypeError: unbound method decrypt() must be called with Secrets instance as first argument (got unicode instance instead) + def test_decrypt(self, temp_config, monkeypatch): + monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') + json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') + s = Secrets() + secrets = s.decrypt(json_secrets_path) + assert secrets == {'some': 'test'} -- GitLab From 7dadf5493383b78bc090148b87d16d1c8ac4dda7 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:07:26 -0700 Subject: [PATCH 03/16] autolintz --- dojo/cli.py | 2 +- dojo/run.py | 2 -- dojo/secrets.py | 9 +++++---- tests/conftest.py | 3 ++- tests/test_secrets.py | 7 +++---- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index fbe68a7..ed26da4 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -19,4 +19,4 @@ def cli(context, name, runner, config, env, encrypt): if encrypt: Secrets().encrypt(config, env) else: - Entrypoint().run(name, runner, config, env) + Entrypoint().run(name, runner, config, env) diff --git a/dojo/run.py b/dojo/run.py index f1645b6..99c21da 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -2,10 +2,8 @@ from __future__ import absolute_import, print_function, unicode_literals import os import yaml -import json import importlib import fnmatch -import subprocess from datetime import datetime from .secrets import Secrets diff --git a/dojo/secrets.py b/dojo/secrets.py index 3b4d253..531bf2c 100644 --- a/dojo/secrets.py +++ b/dojo/secrets.py @@ -6,13 +6,14 @@ import click from cryptography.fernet import Fernet + class Secrets(object): def encrypt(self, config, env): encrypt_key = os.environ.get('DOJO_DECRYPT_KEY') if not encrypt_key and click.confirm('No key found in your environment variables. Would you like to generate a key?'): encrypt_key = Fernet.generate_key() - click.echo(click.style('Your Key: %s (keep this safe)', fg='red', bold=True) % encrypt_key) + click.echo(click.style('Your Key: %s (keep this safe)', fg='red', bold=True) % encrypt_key) env_json_secrets_path = os.path.join(config, 'secrets.%s.json' % (env, )) env_json_secrets = self._read_file(env_json_secrets_path) if not env_json_secrets: @@ -32,7 +33,7 @@ class Secrets(object): try: fernet = Fernet(encrypt_key) out = fernet.decrypt(secrets_file) - #TODO proper error handling? https://cryptography.io/en/latest/fernet/#cryptography.fernet.Fernet.decrypt + # TODO proper error handling? https://cryptography.io/en/latest/fernet/#cryptography.fernet.Fernet.decrypt except ValueError as e: raise ValueError(e) try: @@ -41,10 +42,10 @@ class Secrets(object): raise ValueError(e, out) return secrets - + def _read_file(self, path): if os.path.isfile(path): with open(path, 'r') as f: return f.read() else: - return {} \ No newline at end of file + return {} diff --git a/tests/conftest.py b/tests/conftest.py index 7a26586..64144b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,9 +9,10 @@ def temp_dir(): with tempfile.TemporaryDirectory() as d: yield d + @pytest.fixture(scope='session') def temp_config(tmpdir_factory): config_dir = tmpdir_factory.mktemp('config') json_file = config_dir.join('secrets.development.json') json_file.write('{"some":"test"}') - return config_dir \ No newline at end of file + return config_dir diff --git a/tests/test_secrets.py b/tests/test_secrets.py index b234e3c..df4f68d 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,25 +1,24 @@ from __future__ import absolute_import, print_function, unicode_literals import os -import pytest from dojo.secrets import Secrets class TestSecrets(object): - + def test_encrypt(self, temp_config, monkeypatch): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') s = Secrets() encrypted_file = s.encrypt(temp_config.strpath, 'development') - file = open(encrypted_file.name,'r').read() + file = open(encrypted_file.name, 'r').read() temp_file = os.path.join(temp_config.strpath, 'secrets.development.json.enc') assert os.path.isfile(temp_file) assert len(file) == 100 - #TypeError: unbound method decrypt() must be called with Secrets instance as first argument (got unicode instance instead) + # TypeError: unbound method decrypt() must be called with Secrets instance as first argument (got unicode instance instead) def test_decrypt(self, temp_config, monkeypatch): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') -- GitLab From e0ae292cbb52e647f2d50587fdc69bba5cb92d66 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:42:25 -0700 Subject: [PATCH 04/16] checks to see if there is an encrypted file --- dojo/cli.py | 11 ++++++++++- dojo/secrets.py | 30 +++++++++++++++++++----------- tests/test_secrets.py | 9 +++++++++ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index ed26da4..1a3533f 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -1,6 +1,9 @@ import click import logging +from raven.handlers.logging import SentryHandler +from raven.conf import setup_logging + from .run import Entrypoint from .secrets import Secrets @@ -15,7 +18,13 @@ from .secrets import Secrets def cli(context, name, runner, config, env, encrypt): if context.obj is None: context.obj = {} - logging.getLogger().setLevel(logging.INFO) + + if os.environ.get('SENTRY_DSN'): + handler = SentryHandler(os.environ.get('SENTRY_DSN')) + setup_logging(handler) + else: + logging.getLogger().setLevel(logging.INFO) + if encrypt: Secrets().encrypt(config, env) else: diff --git a/dojo/secrets.py b/dojo/secrets.py index 531bf2c..ee4b09a 100644 --- a/dojo/secrets.py +++ b/dojo/secrets.py @@ -29,17 +29,25 @@ class Secrets(object): encrypt_key = os.environ.get('DOJO_DECRYPT_KEY') if not encrypt_key or not json_secrets_path: raise ValueError('Missing a requirement for decrypting. Secrets path or DOJO_DECRYPT_KEY.') - secrets_file = self._read_file(json_secrets_path).encode() - try: - fernet = Fernet(encrypt_key) - out = fernet.decrypt(secrets_file) - # TODO proper error handling? https://cryptography.io/en/latest/fernet/#cryptography.fernet.Fernet.decrypt - except ValueError as e: - raise ValueError(e) - try: - secrets = json.loads(out) - except ValueError as e: - raise ValueError(e, out) + secrets_file = self._read_file(json_secrets_path) + + # If there aren't any encrypted files, try loading the unecrypted file instead + if os.path.isfile(json_secrets_path.split('.enc')[0]) and not os.path.isfile(json_secrets_path): + try: + secrets = json.loads(open(json_secrets_path.split('.enc')[0]).read()) + except ValueError as e: + raise ValueError(e) + else: + try: + fernet = Fernet(encrypt_key) + out = fernet.decrypt(secrets_file.encode()) + # TODO proper error handling? https://cryptography.io/en/latest/fernet/#cryptography.fernet.Fernet.decrypt + except ValueError as e: + raise ValueError(e) + try: + secrets = json.loads(out) + except ValueError as e: + raise ValueError(e, out) return secrets diff --git a/tests/test_secrets.py b/tests/test_secrets.py index df4f68d..444e02c 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -25,3 +25,12 @@ class TestSecrets(object): s = Secrets() secrets = s.decrypt(json_secrets_path) assert secrets == {'some': 'test'} + + def test_decrypt_no_enc(self, temp_config, monkeypatch): + monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') + json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') + os.remove(json_secrets_path) + s = Secrets() + import pdb; pdb.set_trace() + secrets = s.decrypt(json_secrets_path) + assert secrets == {'some': 'test'} \ No newline at end of file -- GitLab From 6ee7dbed5d1272cde7b0c6fb4e74b962b830221d Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:43:29 -0700 Subject: [PATCH 05/16] crypto dep --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0a746f4..ead449a 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ setup( install_requires=[ 'pyyaml', 'jsonschema', + 'cryptography', 'python-dateutil', 'click', ], -- GitLab From e862f8456b9ae61ade258f3c54d20a8a1f9910f0 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:49:15 -0700 Subject: [PATCH 06/16] autolint --- dojo/cli.py | 10 +++++----- tests/test_secrets.py | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index 1a3533f..3aeb15e 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -18,12 +18,12 @@ from .secrets import Secrets def cli(context, name, runner, config, env, encrypt): if context.obj is None: context.obj = {} - - if os.environ.get('SENTRY_DSN'): - handler = SentryHandler(os.environ.get('SENTRY_DSN')) - setup_logging(handler) + + if os.environ.get('SENTRY_DSN'): + handler = SentryHandler(os.environ.get('SENTRY_DSN')) + setup_logging(handler) else: - logging.getLogger().setLevel(logging.INFO) + logging.getLogger().setLevel(logging.INFO) if encrypt: Secrets().encrypt(config, env) diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 444e02c..d2cf0b0 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -31,6 +31,5 @@ class TestSecrets(object): json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') os.remove(json_secrets_path) s = Secrets() - import pdb; pdb.set_trace() secrets = s.decrypt(json_secrets_path) - assert secrets == {'some': 'test'} \ No newline at end of file + assert secrets == {'some': 'test'} -- GitLab From 35784a4b7c2e1a26ab5586f79c7e99c296649462 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:50:55 -0700 Subject: [PATCH 07/16] autolint --- dojo/cli.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index 3aeb15e..6b2a4d7 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -1,9 +1,6 @@ import click import logging -from raven.handlers.logging import SentryHandler -from raven.conf import setup_logging - from .run import Entrypoint from .secrets import Secrets @@ -18,14 +15,8 @@ from .secrets import Secrets def cli(context, name, runner, config, env, encrypt): if context.obj is None: context.obj = {} - - if os.environ.get('SENTRY_DSN'): - handler = SentryHandler(os.environ.get('SENTRY_DSN')) - setup_logging(handler) - else: - logging.getLogger().setLevel(logging.INFO) - + logging.getLogger().setLevel(logging.INFO) if encrypt: Secrets().encrypt(config, env) else: - Entrypoint().run(name, runner, config, env) + Entrypoint().run(name, runner, config, env) \ No newline at end of file -- GitLab From b2256f6751c28aa2af3f85db6b5dd5c2946064ad Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:54:43 -0700 Subject: [PATCH 08/16] autolint --- dojo/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/cli.py b/dojo/cli.py index 6b2a4d7..ed26da4 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -19,4 +19,4 @@ def cli(context, name, runner, config, env, encrypt): if encrypt: Secrets().encrypt(config, env) else: - Entrypoint().run(name, runner, config, env) \ No newline at end of file + Entrypoint().run(name, runner, config, env) -- GitLab From d806189c761a8fe11c71845c836c2cd4ac567a5d Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 02:10:28 -0700 Subject: [PATCH 09/16] cleaning up tests --- tests/test_secrets.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_secrets.py b/tests/test_secrets.py index d2cf0b0..3cb1b3a 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -9,8 +9,7 @@ class TestSecrets(object): def test_encrypt(self, temp_config, monkeypatch): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') - s = Secrets() - encrypted_file = s.encrypt(temp_config.strpath, 'development') + encrypted_file = Secrets().encrypt(temp_config.strpath, 'development') file = open(encrypted_file.name, 'r').read() temp_file = os.path.join(temp_config.strpath, 'secrets.development.json.enc') @@ -18,18 +17,15 @@ class TestSecrets(object): assert os.path.isfile(temp_file) assert len(file) == 100 - # TypeError: unbound method decrypt() must be called with Secrets instance as first argument (got unicode instance instead) def test_decrypt(self, temp_config, monkeypatch): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') - s = Secrets() - secrets = s.decrypt(json_secrets_path) + secrets = Secrets().decrypt(json_secrets_path) assert secrets == {'some': 'test'} def test_decrypt_no_enc(self, temp_config, monkeypatch): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') json_secrets_path = os.path.join(temp_config.strpath, 'secrets.development.json.enc') os.remove(json_secrets_path) - s = Secrets() - secrets = s.decrypt(json_secrets_path) + secrets = Secrets().decrypt(json_secrets_path) assert secrets == {'some': 'test'} -- GitLab From 5dd51b0372950d41c655004b18d225d52b22de45 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 02:11:11 -0700 Subject: [PATCH 10/16] autolint --- tests/test_secrets.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 3cb1b3a..82924cd 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -11,9 +11,7 @@ class TestSecrets(object): monkeypatch.setenv('DOJO_DECRYPT_KEY', 'Xwti2BVVb-ReShrBCI0A0C54x0yc8zmagz39gCsw0kA=') encrypted_file = Secrets().encrypt(temp_config.strpath, 'development') file = open(encrypted_file.name, 'r').read() - temp_file = os.path.join(temp_config.strpath, 'secrets.development.json.enc') - assert os.path.isfile(temp_file) assert len(file) == 100 -- GitLab From 42359cc2729b9d856a2ab1af1ff73d92c96eb774 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Fri, 8 Sep 2017 15:10:10 -0700 Subject: [PATCH 11/16] Run encrypt with else default to --- dojo/cli.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index ed26da4..0d86480 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -10,13 +10,9 @@ from .secrets import Secrets @click.option('--runner', default=None, help='specify a runner for the job') @click.option('--config', default='config', help='path to directory containing configuration files to be merged') @click.option('--env', default='development', help='environment used to select configuration and secrets') -@click.option('--encrypt', is_flag=True, help='encrypt your project secrets') @click.pass_context def cli(context, name, runner, config, env, encrypt): if context.obj is None: context.obj = {} logging.getLogger().setLevel(logging.INFO) - if encrypt: - Secrets().encrypt(config, env) - else: - Entrypoint().run(name, runner, config, env) + Secrets().encrypt(config, env) if name == 'encrypt' else Entrypoint().run(name, runner, config, env) \ No newline at end of file -- GitLab From 82196081ff5a0dc305c531897a11dc23594aacba Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Fri, 8 Sep 2017 15:12:33 -0700 Subject: [PATCH 12/16] removes encrypt variable from being passed into cli() --- dojo/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/cli.py b/dojo/cli.py index 0d86480..982957d 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -11,7 +11,7 @@ from .secrets import Secrets @click.option('--config', default='config', help='path to directory containing configuration files to be merged') @click.option('--env', default='development', help='environment used to select configuration and secrets') @click.pass_context -def cli(context, name, runner, config, env, encrypt): +def cli(context, name, runner, config, env): if context.obj is None: context.obj = {} logging.getLogger().setLevel(logging.INFO) -- GitLab From 6818adb7a3a90772c202f7a49c6eff11acf43dee Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Fri, 8 Sep 2017 15:13:38 -0700 Subject: [PATCH 13/16] autolint --- dojo/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/cli.py b/dojo/cli.py index 982957d..bcfc251 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -15,4 +15,4 @@ def cli(context, name, runner, config, env): if context.obj is None: context.obj = {} logging.getLogger().setLevel(logging.INFO) - Secrets().encrypt(config, env) if name == 'encrypt' else Entrypoint().run(name, runner, config, env) \ No newline at end of file + Secrets().encrypt(config, env) if name == 'encrypt' else Entrypoint().run(name, runner, config, env) -- GitLab From e9ce6eea26f8c43f792aabe1c542c6671397f9a6 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Wed, 13 Sep 2017 16:17:18 -0700 Subject: [PATCH 14/16] updating cli to use proper grouping system. Updates the README to reflect dojo encryption key --- README.md | 15 ++++++++++++++- dojo/cli.py | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index abeb89b..431e559 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ A data framework encapsulating common data warehousing patterns and best practices, including: + - Defining and scheduling jobs locally, on Google Cloud Dataflow, and on Google Cloud ML Engine - Schemas on datasets and integrity of inputs and outputs - Versioned datasets in storage, never updated @@ -15,8 +16,20 @@ script/test script/distribute ``` +##### Generate your decryption key: + +``` +dojo encrypt +``` +or use a pre-exsiting key: +``` +$ export DOJO_DECRYPT_KEY=yourkey +``` + +##### Run your job + ``` -dojo --runner cloud --env production +dojo run --runner cloud --env production ``` ``` diff --git a/dojo/cli.py b/dojo/cli.py index bcfc251..577cd36 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -4,15 +4,25 @@ import logging from .run import Entrypoint from .secrets import Secrets +@click.group() +def cli(): + pass -@click.command(help='Run a job') +@cli.command(help='Run a dojo job') @click.argument('name') @click.option('--runner', default=None, help='specify a runner for the job') @click.option('--config', default='config', help='path to directory containing configuration files to be merged') @click.option('--env', default='development', help='environment used to select configuration and secrets') @click.pass_context -def cli(context, name, runner, config, env): +def run(context, name, runner, config, env): if context.obj is None: context.obj = {} logging.getLogger().setLevel(logging.INFO) - Secrets().encrypt(config, env) if name == 'encrypt' else Entrypoint().run(name, runner, config, env) + Entrypoint().run(name, runner, config, env) + +@cli.command(help='Encrypt secrets') +@click.option('--config', default='config', help='path to directory containing configuration files to be merged') +@click.option('--env', default='development', help='environment used to select configuration and secrets') +@click.pass_context +def encrypt(context, config, env): + Secrets().encrypt(config, env) \ No newline at end of file -- GitLab From 46e6bcade383774e1b632a4f0c8e85143ddca47a Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Wed, 13 Sep 2017 16:20:38 -0700 Subject: [PATCH 15/16] auto-lint --- dojo/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dojo/cli.py b/dojo/cli.py index 577cd36..07cb84c 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -4,10 +4,12 @@ import logging from .run import Entrypoint from .secrets import Secrets + @click.group() def cli(): pass + @cli.command(help='Run a dojo job') @click.argument('name') @click.option('--runner', default=None, help='specify a runner for the job') @@ -20,9 +22,10 @@ def run(context, name, runner, config, env): logging.getLogger().setLevel(logging.INFO) Entrypoint().run(name, runner, config, env) + @cli.command(help='Encrypt secrets') @click.option('--config', default='config', help='path to directory containing configuration files to be merged') @click.option('--env', default='development', help='environment used to select configuration and secrets') @click.pass_context def encrypt(context, config, env): - Secrets().encrypt(config, env) \ No newline at end of file + Secrets().encrypt(config, env) -- GitLab From a0372d3e65f5f0a5071249aaa651288eb9ae7e82 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Wed, 13 Sep 2017 17:09:57 -0700 Subject: [PATCH 16/16] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ead449a..a7b477b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages setup( name='dojo', - version='0.0.40', + version='0.0.41', description='A framework for building and running your data platform.', author='Data Up', author_email='dojo@dataup.me', -- GitLab