From a339d3379bb652db0a34cc5a96655e9b729a89ca Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:45:50 -0700 Subject: [PATCH 1/6] sentry integration --- dojo/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dojo/cli.py b/dojo/cli.py index 07cb84c..c2af877 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -1,6 +1,9 @@ import click import logging +from raven.handlers.logging import SentryHandler +from raven.conf import setup_logging + from .run import Entrypoint from .secrets import Secrets @@ -19,7 +22,11 @@ def cli(): def run(context, name, runner, config, env): if context.obj is None: context.obj = {} - logging.getLogger().setLevel(logging.INFO) + if os.environ.get('SENTRY_DSN'): + handler = SentryHandler(os.environ.get('SENTRY_DSN')) + setup_logging(handler) + else: + logging.getLogger().setLevel(logging.INFO) Entrypoint().run(name, runner, config, env) -- GitLab From 20ffc2f7bfb38f6c0045f94fd036700f3f750db3 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:48:09 -0700 Subject: [PATCH 2/6] autolinters --- dojo/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index c2af877..8b49f4f 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -23,10 +23,10 @@ def run(context, name, runner, config, env): if context.obj is None: context.obj = {} if os.environ.get('SENTRY_DSN'): - handler = SentryHandler(os.environ.get('SENTRY_DSN')) - setup_logging(handler) + handler = SentryHandler(os.environ.get('SENTRY_DSN')) + setup_logging(handler) else: - logging.getLogger().setLevel(logging.INFO) + logging.getLogger().setLevel(logging.INFO) Entrypoint().run(name, runner, config, env) -- GitLab From 68d32dafe89965debe4077f789eccd03627aa8ce Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 2 Sep 2017 01:58:00 -0700 Subject: [PATCH 3/6] import os --- dojo/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dojo/cli.py b/dojo/cli.py index 8b49f4f..d7e3d7d 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -1,3 +1,4 @@ +import os import click import logging -- GitLab From abfd8d4b5f8af197800bbfe8a1884e6472977bfe Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Wed, 13 Sep 2017 16:59:53 -0700 Subject: [PATCH 4/6] add expection and error-handler support, broken change for now --- dojo/run.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/dojo/run.py b/dojo/run.py index b784433..805ff77 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -46,7 +46,12 @@ class Entrypoint(object): raise ValueError('specified runner "%s" is not supported by job type %s, only %s' % (runner_name, job.__class__.__name__, job.RUNNERS.keys())) runner_class = job.RUNNERS[runner_name] - runner_class().run(job, config) + try: + runner_class().run(job, config) + except Exception as e: + if self._error_handler(config): + self._error_handler.capture(e, job, config) + raise def _build_job(self, job_name, config, secrets, runner): job_config = config.get('jobs', {}).get(job_name, {}) @@ -103,3 +108,11 @@ class Entrypoint(object): for filename in fnmatch.filter(filenames, '*.%s' % (extension, )): matches.append(os.path.join(root, filename)) return matches + + def _error_handler(self, dataset_name): + if 'errors' not in self.config: + return None + return self._adapter_class(self.config['errors']['adapter'])( + config=self.config['errors'], + secrets=self.secrets.get('errors', {}), + dataset=dataset_name) \ No newline at end of file -- GitLab From 0d98fa69f031704abe284f27a4ca6ad27f2d24d4 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 16 Sep 2017 18:24:16 -0700 Subject: [PATCH 5/6] adding sentry and checking for exceptions --- dojo/cli.py | 6 +----- dojo/run.py | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index d7e3d7d..58c7785 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -23,11 +23,7 @@ def cli(): def run(context, name, runner, config, env): if context.obj is None: context.obj = {} - if os.environ.get('SENTRY_DSN'): - handler = SentryHandler(os.environ.get('SENTRY_DSN')) - setup_logging(handler) - else: - logging.getLogger().setLevel(logging.INFO) + logging.getLogger().setLevel(logging.INFO) Entrypoint().run(name, runner, config, env) diff --git a/dojo/run.py b/dojo/run.py index 805ff77..0223dcd 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -45,13 +45,18 @@ class Entrypoint(object): if runner_name not in job.RUNNERS: raise ValueError('specified runner "%s" is not supported by job type %s, only %s' % (runner_name, job.__class__.__name__, job.RUNNERS.keys())) runner_class = job.RUNNERS[runner_name] - + try: runner_class().run(job, config) - except Exception as e: - if self._error_handler(config): - self._error_handler.capture(e, job, config) - raise + except Exception as error: + if job.config['errors'] is not None: + job.config['errors']( + job.config, + job.secrets['errors'], + env + ).capture(error) + raise error + def _build_job(self, job_name, config, secrets, runner): job_config = config.get('jobs', {}).get(job_name, {}) @@ -71,6 +76,11 @@ class Entrypoint(object): if 'store' in job_config and 'connection' in job_config['store']: job_secrets['store']['connection'] = secrets.get('connections', {}).get(job_config['store']['connection'],) + if 'errors' in config: + job_config['errors'] = self._get_module_class(config['errors']['adapter']) + if 'errors' in secrets: + job_secrets['errors'] = secrets.get('errors', {}) + job_connection = job_config.get('connection') if isinstance(job_connection, str): job_config['connection'] = config['connections'].get(job_connection, {}) @@ -108,11 +118,3 @@ class Entrypoint(object): for filename in fnmatch.filter(filenames, '*.%s' % (extension, )): matches.append(os.path.join(root, filename)) return matches - - def _error_handler(self, dataset_name): - if 'errors' not in self.config: - return None - return self._adapter_class(self.config['errors']['adapter'])( - config=self.config['errors'], - secrets=self.secrets.get('errors', {}), - dataset=dataset_name) \ No newline at end of file -- GitLab From 4b31e97a39a87abf554b087a3465daa612949354 Mon Sep 17 00:00:00 2001 From: Zak Wheaton Date: Sat, 16 Sep 2017 18:25:38 -0700 Subject: [PATCH 6/6] autolint and version bump --- dojo/cli.py | 3 --- dojo/run.py | 5 ++--- setup.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dojo/cli.py b/dojo/cli.py index 58c7785..9e5aa6f 100644 --- a/dojo/cli.py +++ b/dojo/cli.py @@ -1,9 +1,6 @@ -import os import click import logging -from raven.handlers.logging import SentryHandler -from raven.conf import setup_logging from .run import Entrypoint from .secrets import Secrets diff --git a/dojo/run.py b/dojo/run.py index 0223dcd..7aa56e8 100644 --- a/dojo/run.py +++ b/dojo/run.py @@ -45,7 +45,7 @@ class Entrypoint(object): if runner_name not in job.RUNNERS: raise ValueError('specified runner "%s" is not supported by job type %s, only %s' % (runner_name, job.__class__.__name__, job.RUNNERS.keys())) runner_class = job.RUNNERS[runner_name] - + try: runner_class().run(job, config) except Exception as error: @@ -54,9 +54,8 @@ class Entrypoint(object): job.config, job.secrets['errors'], env - ).capture(error) + ).capture(error) raise error - def _build_job(self, job_name, config, secrets, runner): job_config = config.get('jobs', {}).get(job_name, {}) diff --git a/setup.py b/setup.py index 56f9c6a..8600659 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages setup( name='dojo', - version='0.0.42', + version='0.0.44', description='A framework for building and running your data platform.', author='Data Up', author_email='dojo@dataup.me', -- GitLab