diff --git a/Makefile b/Makefile index 8b35e9c2..69bcbbc8 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,13 @@ ARGS := # Unit suites inherit that; special suites clear it with --override-ini. UNIT_FLAGS := --timeout=30 --timeout-method=thread SEQ_FLAGS := --override-ini="addopts=-v -x" -n logical --timeout=120 --timeout-method=thread -FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz +FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz \ + --ignore=tests/api/test_schemathesis.py \ + --ignore=tests/api/test_schemathesis_agent.py \ + --ignore=tests/api/test_schemathesis_swarm.py \ + --ignore=tests/api/test_schemathesis_ttp.py +SCHEMA_QUICK ?= 0 +SCHEMA_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz --timeout=600 --timeout-method=thread BENCH_FLAGS := --override-ini="addopts=-v" -p no:xdist --benchmark-only -m bench # ── Unit suites (xdist, 30s timeout) ───────────────────────────────────────── @@ -103,6 +109,15 @@ test-service: test-fuzz: $(PYTEST) $(FUZZ_FLAGS) $(ARGS) +.PHONY: test-schema +test-schema: + SCHEMA_QUICK=$(SCHEMA_QUICK) $(PYTEST) \ + tests/api/test_schemathesis.py \ + tests/api/test_schemathesis_agent.py \ + tests/api/test_schemathesis_swarm.py \ + tests/api/test_schemathesis_ttp.py \ + $(SCHEMA_FLAGS) $(ARGS) + .PHONY: test-bench test-bench: $(PYTEST) tests/perf $(BENCH_FLAGS) $(ARGS) @@ -133,7 +148,7 @@ test-pip-audit: _ALL_SUITES := core web db bus ttp intel analysis infra fleet cli features \ go react \ - live api stress service fuzz bench docker \ + live api schema stress service fuzz bench docker \ mypy bandit vulture pip-audit .PHONY: test-all test @@ -185,7 +200,9 @@ help: @echo " make test-api tests/api (schemathesis)" @echo " make test-stress tests/stress" @echo " make test-service tests/service_testing" - @echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz)" + @echo " make test-schema schemathesis contract tests (-m fuzz, xdist logical)" + @echo " make test-schema SCHEMA_QUICK=1 same, capped at 100 examples per test" + @echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz, skips schemathesis files)" @echo " make test-bench tests/perf" @echo " make test-docker tests/docker (needs DECNET_LIVE_DOCKER=1)" @echo "" @@ -196,7 +213,7 @@ help: @echo " make test-pip-audit pip-audit dependency vulnerability scan" @echo "" @echo "Composites:" - @echo " make test-all ALL suites (unit + go + react + live + api + fuzz + bench + stress + docker + static analysis)" + @echo " make test-all ALL suites (unit + go + react + live + api + schema + fuzz + bench + stress + docker + static analysis)" @echo " make test-all FAIL_FAST=0 same, report all failures instead of stopping" @echo "" @echo "Passthrough: make test-web ARGS='--lf -s'" diff --git a/tests/api/test_schemathesis.py b/tests/api/test_schemathesis.py index a18c430c..c9bcdb96 100644 --- a/tests/api/test_schemathesis.py +++ b/tests/api/test_schemathesis.py @@ -40,6 +40,7 @@ def _free_port() -> int: LIVE_PORT = _free_port() LIVE_SERVER_URL = f"http://127.0.0.1:{LIVE_PORT}" TEST_SECRET = "test-secret-for-automated-fuzzing" +_QUICK = os.getenv("SCHEMA_QUICK") == "1" import decnet.web.auth decnet.web.auth.SECRET_KEY = TEST_SECRET @@ -144,7 +145,7 @@ schema = st.openapi.from_url(f"{LIVE_SERVER_URL}/openapi.json") @pytest.mark.fuzz @st.pytest.parametrize(api=schema) @settings( - max_examples=3000, + max_examples=100 if _QUICK else 3000, deadline=None, verbosity=Verbosity.debug, suppress_health_check=[ @@ -161,7 +162,7 @@ def test_schema_compliance(case): @pytest.mark.fuzz @st.pytest.parametrize(api=schema) @settings( - max_examples=500, + max_examples=100 if _QUICK else 500, deadline=None, verbosity=Verbosity.normal, suppress_health_check=[ diff --git a/tests/api/test_schemathesis_agent.py b/tests/api/test_schemathesis_agent.py index ae3f0daa..daca995b 100644 --- a/tests/api/test_schemathesis_agent.py +++ b/tests/api/test_schemathesis_agent.py @@ -18,9 +18,12 @@ from schemathesis.specs.openapi.checks import ( response_headers_conformance, response_schema_conformance, ) +import os from hypothesis import settings, HealthCheck from decnet.agent import app as _agent_app_mod + +_QUICK = os.getenv("SCHEMA_QUICK") == "1" from decnet.agent import executor as _exec from decnet.agent import heartbeat as _heartbeat @@ -87,7 +90,7 @@ CHECKS = ( @pytest.mark.fuzz @SCHEMA.parametrize() @settings( - max_examples=300, + max_examples=100 if _QUICK else 300, deadline=None, suppress_health_check=[ HealthCheck.filter_too_much, diff --git a/tests/api/test_schemathesis_swarm.py b/tests/api/test_schemathesis_swarm.py index 0c5dfb38..3e306635 100644 --- a/tests/api/test_schemathesis_swarm.py +++ b/tests/api/test_schemathesis_swarm.py @@ -17,6 +17,8 @@ os.environ.setdefault("DECNET_JWT_SECRET", "schemathesis-swarm-secret-32chars-mi import pytest import schemathesis as st + +_QUICK = os.getenv("SCHEMA_QUICK") == "1" from schemathesis.checks import not_a_server_error from schemathesis.specs.openapi.checks import ( status_code_conformance, @@ -54,7 +56,7 @@ CHECKS = ( @pytest.mark.fuzz @SCHEMA.parametrize() @settings( - max_examples=200, + max_examples=100 if _QUICK else 200, deadline=None, suppress_health_check=[ HealthCheck.filter_too_much, diff --git a/tests/api/test_schemathesis_ttp.py b/tests/api/test_schemathesis_ttp.py index 1b789af8..17af3d45 100644 --- a/tests/api/test_schemathesis_ttp.py +++ b/tests/api/test_schemathesis_ttp.py @@ -24,7 +24,10 @@ Routes covered (all decorated ``tags=["TTP Tagging"]``): """ from __future__ import annotations +import os import pytest + +_QUICK = os.getenv("SCHEMA_QUICK") == "1" import schemathesis as st from hypothesis import HealthCheck, Verbosity, settings @@ -46,7 +49,7 @@ TTP_SCHEMA = st.openapi.from_url( @pytest.mark.fuzz @TTP_SCHEMA.parametrize() @settings( - max_examples=400, + max_examples=100 if _QUICK else 400, deadline=None, verbosity=Verbosity.normal, suppress_health_check=[ @@ -63,7 +66,7 @@ def test_ttp_schema_compliance(case): @pytest.mark.fuzz @TTP_SCHEMA.parametrize() @settings( - max_examples=120, + max_examples=100 if _QUICK else 120, deadline=None, verbosity=Verbosity.normal, suppress_health_check=[