From 38423e3989369a1f3c118a21d15ee0e7d24ef012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 12:34:39 +0200 Subject: [PATCH 01/26] simple testing module test for contractme --- pyproject.toml | 1 + tests/test_hypothesis.py | 15 +++++++++++++++ uv.lock | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/test_hypothesis.py diff --git a/pyproject.toml b/pyproject.toml index d385b9c..7016f23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ authors = [ requires-python = ">=3.13" dependencies = [ "annotated-types>=0.7.0", + "hypothesis>=6.131.9", ] [project.urls] diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py new file mode 100644 index 0000000..7d7fee7 --- /dev/null +++ b/tests/test_hypothesis.py @@ -0,0 +1,15 @@ +from typing import TypeAlias, Annotated +from contractme import annotated +from annotated_types import Interval + +from contractme.testing import ContractTest + +# Test setup + +MyInt: TypeAlias = Annotated[int, Interval(0, 1000)] + +@annotated +def sub_and_div(a, b: MyInt) -> float: + return 1.0 / float(a - b + 100) + +test_sub_and_div = ContractTest(sub_and_div) diff --git a/uv.lock b/uv.lock index 6ac02b1..c867aef 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, ] +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -26,6 +35,7 @@ version = "1.2.0" source = { editable = "." } dependencies = [ { name = "annotated-types" }, + { name = "hypothesis" }, ] [package.dev-dependencies] @@ -37,7 +47,10 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "annotated-types", specifier = ">=0.7.0" }] +requires-dist = [ + { name = "annotated-types", specifier = ">=0.7.0" }, + { name = "hypothesis", specifier = ">=6.131.9" }, +] [package.metadata.requires-dev] dev = [ @@ -85,6 +98,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/01/aa81bef60e1aa2ce606ba7c86f7119ed6e1e3bf30a9e8b33a507681847c8/epycs-1.5.0-py3-none-any.whl", hash = "sha256:79a99b197eb251f80fd0a42bb54404e170c030e0df3698402f4037ac07ce9878", size = 10452 }, ] +[[package]] +name = "hypothesis" +version = "6.131.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/10/ff/217417d065aa8a4e6815ddc39acee1222f1b67bd0e4803b85de86a837873/hypothesis-6.131.9.tar.gz", hash = "sha256:ee9b0e1403e1121c91921dbdc79d7f509fdb96d457a0389222d2a68d6c8a8f8e", size = 435415 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/e5/41a6733bfe11997795669dec3b3d785c28918e06568a2540dcc29f0d3fa7/hypothesis-6.131.9-py3-none-any.whl", hash = "sha256:7c2d9d6382e98e5337b27bd34e5b223bac23956787a827e1d087e00d893561d6", size = 499853 }, +] + [[package]] name = "iniconfig" version = "2.1.0" @@ -162,6 +188,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde", size = 23841 }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + [[package]] name = "typing-extensions" version = "4.13.2" -- GitLab From 4f4bc99bc9b8bb47f7446b73c85bf2cbd366f614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 12:37:25 +0200 Subject: [PATCH 02/26] test passes but shouldnt --- src/contractme/testing.py | 2 ++ tests/test_hypothesis.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 src/contractme/testing.py diff --git a/src/contractme/testing.py b/src/contractme/testing.py new file mode 100644 index 0000000..da11d08 --- /dev/null +++ b/src/contractme/testing.py @@ -0,0 +1,2 @@ +def ContractTest(f): + return lambda: None diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 7d7fee7..ae06e5c 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -6,7 +6,7 @@ from contractme.testing import ContractTest # Test setup -MyInt: TypeAlias = Annotated[int, Interval(0, 1000)] +MyInt: TypeAlias = Annotated[int, Interval(le=0, ge=1000)] @annotated def sub_and_div(a, b: MyInt) -> float: -- GitLab From e94f1f2302fe44d985d7c11e7cb9ee1a8bc1bb5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 12:47:45 +0200 Subject: [PATCH 03/26] one succeeds one fails --- src/contractme/testing.py | 4 +++- tests/test_hypothesis.py | 24 ++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index da11d08..f895c8a 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,2 +1,4 @@ def ContractTest(f): - return lambda: None + def run(): + raise AssertionError() + return run diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index ae06e5c..27dc431 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -1,5 +1,7 @@ +from pytest import raises +import operator as op from typing import TypeAlias, Annotated -from contractme import annotated +from contractme import annotated, precondition from annotated_types import Interval from contractme.testing import ContractTest @@ -9,7 +11,25 @@ from contractme.testing import ContractTest MyInt: TypeAlias = Annotated[int, Interval(le=0, ge=1000)] @annotated +def sub_and_div_missing_contract(a, b: MyInt) -> float: + """ + This is incorrectly contracted: it fails for a == b + """ + return 1.0 / float(a - b + 100) + +def test_trivial_failure_missing_contract(): + test_sub_and_div = ContractTest(sub_and_div_missing_contract) + with raises(AssertionError): + test_sub_and_div() + +@annotated +@precondition(op.ne) def sub_and_div(a, b: MyInt) -> float: + """ + This is correctly contracted + """ return 1.0 / float(a - b + 100) -test_sub_and_div = ContractTest(sub_and_div) +def test_no_failure(): + test_sub_and_div = ContractTest(sub_and_div) + test_sub_and_div() -- GitLab From e7045abd39a2b46f2483c914a62f8f7c78403737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 12:51:17 +0200 Subject: [PATCH 04/26] stub impl --- src/contractme/testing.py | 5 ++++- tests/test_hypothesis.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index f895c8a..b771614 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,4 +1,7 @@ def ContractTest(f): def run(): - raise AssertionError() + if f.accepts(0, 0): + raise AssertionError() + else: + pass return run diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 27dc431..d7c1a38 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -8,7 +8,7 @@ from contractme.testing import ContractTest # Test setup -MyInt: TypeAlias = Annotated[int, Interval(le=0, ge=1000)] +MyInt: TypeAlias = Annotated[int, Interval(ge=0, le=1000)] @annotated def sub_and_div_missing_contract(a, b: MyInt) -> float: @@ -23,7 +23,7 @@ def test_trivial_failure_missing_contract(): test_sub_and_div() @annotated -@precondition(op.ne) +@precondition(lambda a, b: a != b) def sub_and_div(a, b: MyInt) -> float: """ This is correctly contracted -- GitLab From 17cb1afa9553674ec0c7f5510b6485b7d106b4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 12:59:02 +0200 Subject: [PATCH 05/26] black --- src/contractme/testing.py | 1 + tests/test_hypothesis.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index b771614..bc716f0 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -4,4 +4,5 @@ def ContractTest(f): raise AssertionError() else: pass + return run diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index d7c1a38..2a60105 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -10,6 +10,7 @@ from contractme.testing import ContractTest MyInt: TypeAlias = Annotated[int, Interval(ge=0, le=1000)] + @annotated def sub_and_div_missing_contract(a, b: MyInt) -> float: """ @@ -17,11 +18,13 @@ def sub_and_div_missing_contract(a, b: MyInt) -> float: """ return 1.0 / float(a - b + 100) + def test_trivial_failure_missing_contract(): test_sub_and_div = ContractTest(sub_and_div_missing_contract) with raises(AssertionError): test_sub_and_div() + @annotated @precondition(lambda a, b: a != b) def sub_and_div(a, b: MyInt) -> float: @@ -30,6 +33,7 @@ def sub_and_div(a, b: MyInt) -> float: """ return 1.0 / float(a - b + 100) + def test_no_failure(): test_sub_and_div = ContractTest(sub_and_div) test_sub_and_div() -- GitLab From 0720f1b7bc783a7da627805b6cd45b0e93dda8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:02:12 +0200 Subject: [PATCH 06/26] add failing test --- tests/test_hypothesis.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 2a60105..dc4f2a2 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -16,7 +16,7 @@ def sub_and_div_missing_contract(a, b: MyInt) -> float: """ This is incorrectly contracted: it fails for a == b """ - return 1.0 / float(a - b + 100) + return 1.0 / float(a - b) def test_trivial_failure_missing_contract(): @@ -31,9 +31,24 @@ def sub_and_div(a, b: MyInt) -> float: """ This is correctly contracted """ - return 1.0 / float(a - b + 100) + return 1.0 / float(a - b) def test_no_failure(): test_sub_and_div = ContractTest(sub_and_div) test_sub_and_div() + + +@annotated +@precondition(lambda a, b: a != b) +def sub_and_div_wrong_contract(a, b) -> float: + """ + The contract is incorrect: offset in divisor + """ + return 1.0 / float(a - b + 100) + + +def test_trivial_failure_wrong_contract(): + test_sub_and_div = ContractTest(sub_and_div_wrong_contract) + with raises(AssertionError): + test_sub_and_div() -- GitLab From e95a90b1f7b39885883ca367ccc4d404923b869b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:31:37 +0200 Subject: [PATCH 07/26] can specify counterexamples --- src/contractme/testing.py | 29 +++++++++++++++++++++++------ tests/test_hypothesis.py | 19 ++++++++++--------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index bc716f0..f6790c5 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,8 +1,25 @@ +from hypothesis import assume, given, strategies as st + def ContractTest(f): - def run(): - if f.accepts(0, 0): - raise AssertionError() - else: - pass + def generator(*examples): + for e in examples: + if e != Ellipsis: + f(*e) + + @st.composite + def correct_input(draw) -> tuple[int, int]: + a = draw(st.integers()) + b = draw(st.integers()) + assume(f.accepts(a, b)) + return a, b + + @given(correct_input()) + def run_hypothesis(value): + a, b = value + f(a, b) + + if not examples or Ellipsis in examples: + run_hypothesis() + - return run + return generator diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index dc4f2a2..f8a3625 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -20,9 +20,9 @@ def sub_and_div_missing_contract(a, b: MyInt) -> float: def test_trivial_failure_missing_contract(): - test_sub_and_div = ContractTest(sub_and_div_missing_contract) - with raises(AssertionError): - test_sub_and_div() + generator_sub_and_div = ContractTest(sub_and_div_missing_contract) + with raises(ZeroDivisionError): + generator_sub_and_div(...) @annotated @@ -35,8 +35,8 @@ def sub_and_div(a, b: MyInt) -> float: def test_no_failure(): - test_sub_and_div = ContractTest(sub_and_div) - test_sub_and_div() + generator_sub_and_div = ContractTest(sub_and_div) + generator_sub_and_div(...) @annotated @@ -48,7 +48,8 @@ def sub_and_div_wrong_contract(a, b) -> float: return 1.0 / float(a - b + 100) -def test_trivial_failure_wrong_contract(): - test_sub_and_div = ContractTest(sub_and_div_wrong_contract) - with raises(AssertionError): - test_sub_and_div() +def test_simple_failure_wrong_contract(): + generator_sub_and_div = ContractTest(sub_and_div_wrong_contract) + + with raises(ZeroDivisionError): + generator_sub_and_div((0, 100), ...) -- GitLab From a366b00bbb23339f93ac8927de372b0ee4346deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:32:33 +0200 Subject: [PATCH 08/26] can skip hypothesis --- tests/test_hypothesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index f8a3625..9271941 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -52,4 +52,4 @@ def test_simple_failure_wrong_contract(): generator_sub_and_div = ContractTest(sub_and_div_wrong_contract) with raises(ZeroDivisionError): - generator_sub_and_div((0, 100), ...) + generator_sub_and_div((0, 100)) -- GitLab From 10957d5184056d708bb745f38f891e090119f6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:32:56 +0200 Subject: [PATCH 09/26] black ; --- src/contractme/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index f6790c5..c0c0791 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,5 +1,6 @@ from hypothesis import assume, given, strategies as st + def ContractTest(f): def generator(*examples): for e in examples: @@ -21,5 +22,4 @@ def ContractTest(f): if not examples or Ellipsis in examples: run_hypothesis() - return generator -- GitLab From 090b5007ac54bd65475ee96684d383225887dde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:33:15 +0200 Subject: [PATCH 10/26] gitignore hypothesis --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f3d4dc9..7aa82dc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ wheels/ # Virtual environments .venv obj +.hypothesis -- GitLab From 7f1e1a4c6246ea77f9103d287d1417423adfbb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 13:38:56 +0200 Subject: [PATCH 11/26] refactor --- src/contractme/testing.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index c0c0791..62c3e19 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,6 +1,12 @@ from hypothesis import assume, given, strategies as st +def draw_accepted_input(draw, f): + v = draw(st.integers()), draw(st.integers()) + assume(f.accepts(*v)) + return v + + def ContractTest(f): def generator(*examples): for e in examples: @@ -9,10 +15,7 @@ def ContractTest(f): @st.composite def correct_input(draw) -> tuple[int, int]: - a = draw(st.integers()) - b = draw(st.integers()) - assume(f.accepts(a, b)) - return a, b + return draw_accepted_input(draw, f) @given(correct_input()) def run_hypothesis(value): -- GitLab From 3dcdd2202041e824bafc56e99a7cb239f2cafc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 18:18:49 +0200 Subject: [PATCH 12/26] automatically use type hints --- src/contractme/testing.py | 24 ++++++++++++++++++++---- tests/test_hypothesis.py | 10 +++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 62c3e19..a96b1b2 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,20 +1,36 @@ +import inspect +from contractme.contracting import ContractedFunction, normalize_constraint +from typing import Callable, get_type_hints from hypothesis import assume, given, strategies as st -def draw_accepted_input(draw, f): - v = draw(st.integers()), draw(st.integers()) +def get_strategies(f: Callable) -> list[st.SearchStrategy]: + th = get_type_hints(f, include_extras=True) + + sts = [] + + for n, t in th.items(): + if n == "return": + continue + sts.append(st.from_type(t)) + return sts + + +def draw_accepted_input(draw: Callable, f: ContractedFunction) -> tuple: + sts = get_strategies(f) + v = tuple(draw(e) for e in sts) assume(f.accepts(*v)) return v -def ContractTest(f): +def ContractTest(f: ContractedFunction): def generator(*examples): for e in examples: if e != Ellipsis: f(*e) @st.composite - def correct_input(draw) -> tuple[int, int]: + def correct_input(draw: Callable) -> tuple[int, int]: return draw_accepted_input(draw, f) @given(correct_input()) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 9271941..743da5d 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -2,17 +2,17 @@ from pytest import raises import operator as op from typing import TypeAlias, Annotated from contractme import annotated, precondition -from annotated_types import Interval +from annotated_types import Ge, Le from contractme.testing import ContractTest # Test setup -MyInt: TypeAlias = Annotated[int, Interval(ge=0, le=1000)] +MyInt: TypeAlias = Annotated[int, Ge(0), Le(1000)] @annotated -def sub_and_div_missing_contract(a, b: MyInt) -> float: +def sub_and_div_missing_contract(a: MyInt, b: MyInt) -> float: """ This is incorrectly contracted: it fails for a == b """ @@ -27,7 +27,7 @@ def test_trivial_failure_missing_contract(): @annotated @precondition(lambda a, b: a != b) -def sub_and_div(a, b: MyInt) -> float: +def sub_and_div(a: MyInt, b: MyInt) -> float: """ This is correctly contracted """ @@ -41,7 +41,7 @@ def test_no_failure(): @annotated @precondition(lambda a, b: a != b) -def sub_and_div_wrong_contract(a, b) -> float: +def sub_and_div_wrong_contract(a: MyInt, b: MyInt) -> float: """ The contract is incorrect: offset in divisor """ -- GitLab From aeb11978aaef3bbeec6129df76ac0a2af850eb7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 18:23:33 +0200 Subject: [PATCH 13/26] runtime error on mis-call of generator --- src/contractme/testing.py | 2 ++ tests/test_hypothesis.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index a96b1b2..1e32428 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -25,6 +25,8 @@ def draw_accepted_input(draw: Callable, f: ContractedFunction) -> tuple: def ContractTest(f: ContractedFunction): def generator(*examples): + if not examples: + raise RuntimeError("call function as `gen(...)` to fuzz search") for e in examples: if e != Ellipsis: f(*e) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 743da5d..6df18c4 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -34,6 +34,12 @@ def sub_and_div(a: MyInt, b: MyInt) -> float: return 1.0 / float(a - b) +def test_must_provides_examples_or_ellipsis(): + generator_sub_and_div = ContractTest(sub_and_div) + with raises(RuntimeError): + generator_sub_and_div() + + def test_no_failure(): generator_sub_and_div = ContractTest(sub_and_div) generator_sub_and_div(...) -- GitLab From f9bdb33c69ae17c59be7ebc6ebf8a495f4868aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sat, 3 May 2025 18:25:02 +0200 Subject: [PATCH 14/26] fix tuple splitting --- src/contractme/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 1e32428..61434b7 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -37,8 +37,7 @@ def ContractTest(f: ContractedFunction): @given(correct_input()) def run_hypothesis(value): - a, b = value - f(a, b) + f(*value) if not examples or Ellipsis in examples: run_hypothesis() -- GitLab From 82f1d8920c76aabeeb898f67f5c10db401a38417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 08:57:20 +0200 Subject: [PATCH 15/26] minor typo --- src/contractme/testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 61434b7..f9e8c85 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,5 +1,4 @@ -import inspect -from contractme.contracting import ContractedFunction, normalize_constraint +from contractme.contracting import ContractedFunction from typing import Callable, get_type_hints from hypothesis import assume, given, strategies as st -- GitLab From 8f347313548498008fbb67a02df1e6afec243834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 09:17:15 +0200 Subject: [PATCH 16/26] simplify test interface --- src/contractme/testing.py | 33 +++++++++++++-------------------- tests/test_hypothesis.py | 30 +++--------------------------- 2 files changed, 16 insertions(+), 47 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index f9e8c85..0ecda1e 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -22,23 +22,16 @@ def draw_accepted_input(draw: Callable, f: ContractedFunction) -> tuple: return v -def ContractTest(f: ContractedFunction): - def generator(*examples): - if not examples: - raise RuntimeError("call function as `gen(...)` to fuzz search") - for e in examples: - if e != Ellipsis: - f(*e) - - @st.composite - def correct_input(draw: Callable) -> tuple[int, int]: - return draw_accepted_input(draw, f) - - @given(correct_input()) - def run_hypothesis(value): - f(*value) - - if not examples or Ellipsis in examples: - run_hypothesis() - - return generator +def get_generator(f: ContractedFunction): + @st.composite + def correct_input(draw: Callable) -> tuple[int, int]: + return draw_accepted_input(draw, f) + + @given(correct_input()) + def search(value): + f(*value) + return search + + +def autotest(f: ContractedFunction): + get_generator(f)() diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 6df18c4..a31abb4 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -4,7 +4,7 @@ from typing import TypeAlias, Annotated from contractme import annotated, precondition from annotated_types import Ge, Le -from contractme.testing import ContractTest +from contractme.testing import autotest # Test setup @@ -20,9 +20,8 @@ def sub_and_div_missing_contract(a: MyInt, b: MyInt) -> float: def test_trivial_failure_missing_contract(): - generator_sub_and_div = ContractTest(sub_and_div_missing_contract) with raises(ZeroDivisionError): - generator_sub_and_div(...) + autotest(sub_and_div_missing_contract) @annotated @@ -34,28 +33,5 @@ def sub_and_div(a: MyInt, b: MyInt) -> float: return 1.0 / float(a - b) -def test_must_provides_examples_or_ellipsis(): - generator_sub_and_div = ContractTest(sub_and_div) - with raises(RuntimeError): - generator_sub_and_div() - - def test_no_failure(): - generator_sub_and_div = ContractTest(sub_and_div) - generator_sub_and_div(...) - - -@annotated -@precondition(lambda a, b: a != b) -def sub_and_div_wrong_contract(a: MyInt, b: MyInt) -> float: - """ - The contract is incorrect: offset in divisor - """ - return 1.0 / float(a - b + 100) - - -def test_simple_failure_wrong_contract(): - generator_sub_and_div = ContractTest(sub_and_div_wrong_contract) - - with raises(ZeroDivisionError): - generator_sub_and_div((0, 100)) + autotest(sub_and_div) -- GitLab From e55ef02ea893d5791af614e482657406b28d5e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 09:22:00 +0200 Subject: [PATCH 17/26] extend API to accept callable --- src/contractme/testing.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 0ecda1e..ac91247 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,5 +1,5 @@ from contractme.contracting import ContractedFunction -from typing import Callable, get_type_hints +from typing import Callable, get_type_hints, TypeAlias from hypothesis import assume, given, strategies as st @@ -15,14 +15,18 @@ def get_strategies(f: Callable) -> list[st.SearchStrategy]: return sts -def draw_accepted_input(draw: Callable, f: ContractedFunction) -> tuple: +TestableFunction: TypeAlias = ContractedFunction | Callable + + +def draw_accepted_input(draw: Callable, f: TestableFunction) -> tuple: sts = get_strategies(f) v = tuple(draw(e) for e in sts) - assume(f.accepts(*v)) + if isinstance(f, ContractedFunction): + assume(f.accepts(*v)) return v -def get_generator(f: ContractedFunction): +def get_generator(f: TestableFunction): @st.composite def correct_input(draw: Callable) -> tuple[int, int]: return draw_accepted_input(draw, f) @@ -33,5 +37,5 @@ def get_generator(f: ContractedFunction): return search -def autotest(f: ContractedFunction): +def autotest(f: TestableFunction): get_generator(f)() -- GitLab From bdd693264556afe57fd13783129e3c83621c1f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 09:23:49 +0200 Subject: [PATCH 18/26] black --- src/contractme/testing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index ac91247..14da342 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -34,6 +34,7 @@ def get_generator(f: TestableFunction): @given(correct_input()) def search(value): f(*value) + return search -- GitLab From 70ed9395f03ddc860e0b4059516b15f44642a7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 10:38:56 +0200 Subject: [PATCH 19/26] example for composite --- README.md | 31 +++++++++++++++++++++++++++++-- tests/test_hypothesis.py | 12 +++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fb5d2e2..0528fbf 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,35 @@ def square(v : Even) -> Even return v * v ``` +## Writing tests and having test generation + +The hypothesis plugin can be used easily through the `contractme.testing.autotest` +function. + +```python +Positive: TypeAlias = Annotated[int, Ge(1)] + +@annotated +def div(d: Positive) return Positive: + return 1000 // d + +def test_div(): + autotest(div) +``` + +You can access the underlying hypothesis generator with + +It's a pure hypothesis strategy generator, inferred from the annotated types and +contracts of the function. + +You can easily extend it with +[Hypothesis advanced features](https://hypothesis.readthedocs.io/en/latest/reference/api.html) + +```python +generator_function = contractme.testing.get_generator(div) +# kinda weird to have this double call, but that's decorators for you... +test_div_force_0 = example(0)(generator_function) +``` # Test `uv run pytest` @@ -85,5 +114,3 @@ Contracts can be disabled at runtime with `ignore_preconditions()` and `ignore_p Contracts are disabled from the start with python optimized (`-O`) flag. Fix a bug where contracts would hide an incorrect function call - - diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index a31abb4..704ff29 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -3,8 +3,9 @@ import operator as op from typing import TypeAlias, Annotated from contractme import annotated, precondition from annotated_types import Ge, Le +from hypothesis import example -from contractme.testing import autotest +from contractme.testing import autotest, get_generator # Test setup @@ -35,3 +36,12 @@ def sub_and_div(a: MyInt, b: MyInt) -> float: def test_no_failure(): autotest(sub_and_div) + + +def test_div_force_fail(): + gen_sub_and_div = get_generator(sub_and_div) + # kinda weird to have this double call, but that's decorators for you... + gen = example((0, 0))(gen_sub_and_div) + with raises(AssertionError): + # catched by contracts + gen() -- GitLab From 5f7700558e642b2cf62e1b4d46cfa6f2799ab210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 11:07:52 +0200 Subject: [PATCH 20/26] update doc --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0528fbf..b542c43 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,11 @@ def test_div(): autotest(div) ``` -You can access the underlying hypothesis generator with +You can access the underlying hypothesis generator with `contractme.testing.get_generator(div)`. It's a pure hypothesis strategy generator, inferred from the annotated types and -contracts of the function. +contracts of the function. The main weirdness is that it takes a tuple as parameter since +the parameters are all generated together so that the contracts can be checked. You can easily extend it with [Hypothesis advanced features](https://hypothesis.readthedocs.io/en/latest/reference/api.html) @@ -82,7 +83,7 @@ You can easily extend it with ```python generator_function = contractme.testing.get_generator(div) # kinda weird to have this double call, but that's decorators for you... -test_div_force_0 = example(0)(generator_function) +test_div_force_0 = example((0,))(generator_function) ``` # Test -- GitLab From eaa02b08e566c7d93a5d2c2c5d7c9f07b58c33fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:10:44 +0200 Subject: [PATCH 21/26] add test_with_examples --- README.md | 19 ++++++++++ src/contractme/testing.py | 21 ++++++++++- tests/test_hypothesis.py | 77 +++++++++++++++++++++++++++++++++++---- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b542c43..6e66ba5 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,25 @@ generator_function = contractme.testing.get_generator(div) # kinda weird to have this double call, but that's decorators for you... test_div_force_0 = example((0,))(generator_function) ``` + +The library provides its own `contractme.testing.test_with_examples` function which has three differences +with the one provided by hypothesis: + +* It checks the contracts when being called (at test construction): contracts should hold on all + examples. +* It takes a vararg of either tuple `*args` or dict `**kwarg` as examples, to avoid + function nesting. + +With pytest: +```python +test_div = contractme.testing.test_with_examples( + div, + (1,), + (2,), + (0,), # this causes a RuntimeError at test elaboration +) +``` + # Test `uv run pytest` diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 14da342..2dd921d 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -1,6 +1,6 @@ from contractme.contracting import ContractedFunction from typing import Callable, get_type_hints, TypeAlias -from hypothesis import assume, given, strategies as st +from hypothesis import assume, given, example, strategies as st def get_strategies(f: Callable) -> list[st.SearchStrategy]: @@ -16,6 +16,7 @@ def get_strategies(f: Callable) -> list[st.SearchStrategy]: TestableFunction: TypeAlias = ContractedFunction | Callable +TestGenerator: TypeAlias = Callable def draw_accepted_input(draw: Callable, f: TestableFunction) -> tuple: @@ -26,7 +27,7 @@ def draw_accepted_input(draw: Callable, f: TestableFunction) -> tuple: return v -def get_generator(f: TestableFunction): +def get_generator(f: TestableFunction) -> TestGenerator: @st.composite def correct_input(draw: Callable) -> tuple[int, int]: return draw_accepted_input(draw, f) @@ -40,3 +41,19 @@ def get_generator(f: TestableFunction): def autotest(f: TestableFunction): get_generator(f)() + +def call_with_composite(f: Callable, composite: tuple | dict): + if isinstance(composite, dict): + return f(**composite) + else: + return f(*composite) + +def test_with_examples(f: TestableFunction, *exs) -> TestGenerator: + gen = get_generator(f) + for e in exs: + if isinstance(f, ContractedFunction): + accepted = call_with_composite(f.accepts, e) + if not accepted: + raise RuntimeError(f"contracts rejected example {e}") + gen = example(e)(gen) + return gen diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 704ff29..296436c 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -1,19 +1,28 @@ from pytest import raises import operator as op from typing import TypeAlias, Annotated -from contractme import annotated, precondition +from contractme import annotated, precondition, postcondition from annotated_types import Ge, Le -from hypothesis import example +import hypothesis +import contractme.testing from contractme.testing import autotest, get_generator -# Test setup +""" +This contains tests for the testing extension. + +As this is testing the testing code, it is probably harder to understand that +it needs to be. + +Refer to the doc for simpler examples. +""" + MyInt: TypeAlias = Annotated[int, Ge(0), Le(1000)] @annotated -def sub_and_div_missing_contract(a: MyInt, b: MyInt) -> float: +def sub_and_div_no_contract(a: MyInt, b: MyInt) -> float: """ This is incorrectly contracted: it fails for a == b """ @@ -22,7 +31,7 @@ def sub_and_div_missing_contract(a: MyInt, b: MyInt) -> float: def test_trivial_failure_missing_contract(): with raises(ZeroDivisionError): - autotest(sub_and_div_missing_contract) + autotest(sub_and_div_no_contract) @annotated @@ -41,7 +50,61 @@ def test_no_failure(): def test_div_force_fail(): gen_sub_and_div = get_generator(sub_and_div) # kinda weird to have this double call, but that's decorators for you... - gen = example((0, 0))(gen_sub_and_div) + gen = hypothesis.example((0, 0))(gen_sub_and_div) with raises(AssertionError): - # catched by contracts + # catched by contracts at call + gen() + + +def test_test_with_examples(): + gen = contractme.testing.test_with_examples( + sub_and_div, + (1, 0), + (3, 2), + ) + gen() + + +def test_test_with_examples_contract_fails(): + with raises(RuntimeError): + # catched by contracts at test generation + _ = contractme.testing.test_with_examples( + sub_and_div, + (1, 0), + (0, 0), + ) + + +def test_test_with_examples_function_fails(): + # no contracts, no catch + gen = contractme.testing.test_with_examples( + sub_and_div_no_contract, + (1, 0), + (0, 0), + ) + + with raises(ZeroDivisionError): gen() + + +@annotated +@precondition(lambda a, b: a != b) +@postcondition(lambda result: result > 0.0) +def sub_and_div_wrong_impl(a: MyInt, b: MyInt) -> float: + """ + This is correctly contracted + """ + return 1.0 / float(a - b) + + +def test_test_with_examples_postcondition_checked_at_call(): + # no contracts, no catch + gen = contractme.testing.test_with_examples( + sub_and_div_wrong_impl, + (1, 10), + ) + + with raises(AssertionError): + gen() + + -- GitLab From e65f0e79bd92fcdeee51508f0a59aa16265b86e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:12:47 +0200 Subject: [PATCH 22/26] black --- src/contractme/testing.py | 2 ++ tests/test_hypothesis.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index 2dd921d..bfdb23c 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -42,12 +42,14 @@ def get_generator(f: TestableFunction) -> TestGenerator: def autotest(f: TestableFunction): get_generator(f)() + def call_with_composite(f: Callable, composite: tuple | dict): if isinstance(composite, dict): return f(**composite) else: return f(*composite) + def test_with_examples(f: TestableFunction, *exs) -> TestGenerator: gen = get_generator(f) for e in exs: diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 296436c..0241ff4 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -106,5 +106,3 @@ def test_test_with_examples_postcondition_checked_at_call(): with raises(AssertionError): gen() - - -- GitLab From 2989a2f22369d0fefeacd55c2f70cfb857e1f0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:16:17 +0200 Subject: [PATCH 23/26] gitignore: ignore coverage report --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7aa82dc..b23191f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ wheels/ .venv obj .hypothesis +.coverage -- GitLab From f022225165d45176edda38156c28ec594a4c1c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:16:28 +0200 Subject: [PATCH 24/26] add failing test with kwargs --- tests/test_hypothesis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 0241ff4..0f6f56c 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -61,6 +61,7 @@ def test_test_with_examples(): sub_and_div, (1, 0), (3, 2), + {"a": 4, "b": 10}, ) gen() -- GitLab From 5e1979421fd3648165a81fd828b3511e6038fe85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:19:32 +0200 Subject: [PATCH 25/26] add passing test --- tests/test_contracting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_contracting.py b/tests/test_contracting.py index 638c9f5..3fcf63a 100644 --- a/tests/test_contracting.py +++ b/tests/test_contracting.py @@ -136,6 +136,7 @@ def test_rejects_accepts(): assert f.rejects(0) assert f.accepts(1) + assert f.accepts(a=1) def test_with_global(): -- GitLab From 9c698e72b9bd0d671961f08b51a65bd77730c77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Germond?= Date: Sun, 4 May 2025 12:31:11 +0200 Subject: [PATCH 26/26] fix call of composite with kwarg --- src/contractme/testing.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/contractme/testing.py b/src/contractme/testing.py index bfdb23c..6e6bff7 100644 --- a/src/contractme/testing.py +++ b/src/contractme/testing.py @@ -19,6 +19,13 @@ TestableFunction: TypeAlias = ContractedFunction | Callable TestGenerator: TypeAlias = Callable +def call_with_composite(f: Callable, composite: tuple | dict): + if isinstance(composite, dict): + return f(**composite) + else: + return f(*composite) + + def draw_accepted_input(draw: Callable, f: TestableFunction) -> tuple: sts = get_strategies(f) v = tuple(draw(e) for e in sts) @@ -34,7 +41,7 @@ def get_generator(f: TestableFunction) -> TestGenerator: @given(correct_input()) def search(value): - f(*value) + call_with_composite(f, value) return search @@ -43,13 +50,6 @@ def autotest(f: TestableFunction): get_generator(f)() -def call_with_composite(f: Callable, composite: tuple | dict): - if isinstance(composite, dict): - return f(**composite) - else: - return f(*composite) - - def test_with_examples(f: TestableFunction, *exs) -> TestGenerator: gen = get_generator(f) for e in exs: -- GitLab