feat(test): add test-schema target and SCHEMA_QUICK=1 mode for schemathesis

- Add dedicated test-schema Makefile target (xdist logical, 600s timeout,
  -m fuzz) so schemathesis runs separately from test-fuzz, which was
  spinning up competing uvicorn workers per xdist process
- Exclude all test_schemathesis*.py files from FUZZ_FLAGS via --ignore
- Add schema to _ALL_SUITES between api and fuzz
- Add SCHEMA_QUICK env var (default 0): caps every max_examples to 100
  across all four schemathesis files (4520 -> 600 total examples)
- Fix pre-push hook: use .311 venv and delegate to make test-all FAIL_FAST=0
  instead of hand-rolling five separate pytest invocations
This commit is contained in:
2026-05-16 18:25:40 -04:00
parent ac332a6ba9
commit 0fe9f895d0
5 changed files with 36 additions and 10 deletions

View File

@@ -6,7 +6,13 @@ ARGS :=
# Unit suites inherit that; special suites clear it with --override-ini. # Unit suites inherit that; special suites clear it with --override-ini.
UNIT_FLAGS := --timeout=30 --timeout-method=thread UNIT_FLAGS := --timeout=30 --timeout-method=thread
SEQ_FLAGS := --override-ini="addopts=-v -x" -n logical --timeout=120 --timeout-method=thread SEQ_FLAGS := --override-ini="addopts=-v -x" -n logical --timeout=120 --timeout-method=thread
FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz \
--ignore=tests/api/test_schemathesis.py \
--ignore=tests/api/test_schemathesis_agent.py \
--ignore=tests/api/test_schemathesis_swarm.py \
--ignore=tests/api/test_schemathesis_ttp.py
SCHEMA_QUICK ?= 0
SCHEMA_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz --timeout=600 --timeout-method=thread
BENCH_FLAGS := --override-ini="addopts=-v" -p no:xdist --benchmark-only -m bench BENCH_FLAGS := --override-ini="addopts=-v" -p no:xdist --benchmark-only -m bench
# ── Unit suites (xdist, 30s timeout) ───────────────────────────────────────── # ── Unit suites (xdist, 30s timeout) ─────────────────────────────────────────
@@ -103,6 +109,15 @@ test-service:
test-fuzz: test-fuzz:
$(PYTEST) $(FUZZ_FLAGS) $(ARGS) $(PYTEST) $(FUZZ_FLAGS) $(ARGS)
.PHONY: test-schema
test-schema:
SCHEMA_QUICK=$(SCHEMA_QUICK) $(PYTEST) \
tests/api/test_schemathesis.py \
tests/api/test_schemathesis_agent.py \
tests/api/test_schemathesis_swarm.py \
tests/api/test_schemathesis_ttp.py \
$(SCHEMA_FLAGS) $(ARGS)
.PHONY: test-bench .PHONY: test-bench
test-bench: test-bench:
$(PYTEST) tests/perf $(BENCH_FLAGS) $(ARGS) $(PYTEST) tests/perf $(BENCH_FLAGS) $(ARGS)
@@ -133,7 +148,7 @@ test-pip-audit:
_ALL_SUITES := core web db bus ttp intel analysis infra fleet cli features \ _ALL_SUITES := core web db bus ttp intel analysis infra fleet cli features \
go react \ go react \
live api stress service fuzz bench docker \ live api schema stress service fuzz bench docker \
mypy bandit vulture pip-audit mypy bandit vulture pip-audit
.PHONY: test-all test .PHONY: test-all test
@@ -185,7 +200,9 @@ help:
@echo " make test-api tests/api (schemathesis)" @echo " make test-api tests/api (schemathesis)"
@echo " make test-stress tests/stress" @echo " make test-stress tests/stress"
@echo " make test-service tests/service_testing" @echo " make test-service tests/service_testing"
@echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz)" @echo " make test-schema schemathesis contract tests (-m fuzz, xdist logical)"
@echo " make test-schema SCHEMA_QUICK=1 same, capped at 100 examples per test"
@echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz, skips schemathesis files)"
@echo " make test-bench tests/perf" @echo " make test-bench tests/perf"
@echo " make test-docker tests/docker (needs DECNET_LIVE_DOCKER=1)" @echo " make test-docker tests/docker (needs DECNET_LIVE_DOCKER=1)"
@echo "" @echo ""
@@ -196,7 +213,7 @@ help:
@echo " make test-pip-audit pip-audit dependency vulnerability scan" @echo " make test-pip-audit pip-audit dependency vulnerability scan"
@echo "" @echo ""
@echo "Composites:" @echo "Composites:"
@echo " make test-all ALL suites (unit + go + react + live + api + fuzz + bench + stress + docker + static analysis)" @echo " make test-all ALL suites (unit + go + react + live + api + schema + fuzz + bench + stress + docker + static analysis)"
@echo " make test-all FAIL_FAST=0 same, report all failures instead of stopping" @echo " make test-all FAIL_FAST=0 same, report all failures instead of stopping"
@echo "" @echo ""
@echo "Passthrough: make test-web ARGS='--lf -s'" @echo "Passthrough: make test-web ARGS='--lf -s'"

View File

@@ -40,6 +40,7 @@ def _free_port() -> int:
LIVE_PORT = _free_port() LIVE_PORT = _free_port()
LIVE_SERVER_URL = f"http://127.0.0.1:{LIVE_PORT}" LIVE_SERVER_URL = f"http://127.0.0.1:{LIVE_PORT}"
TEST_SECRET = "test-secret-for-automated-fuzzing" TEST_SECRET = "test-secret-for-automated-fuzzing"
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
import decnet.web.auth import decnet.web.auth
decnet.web.auth.SECRET_KEY = TEST_SECRET decnet.web.auth.SECRET_KEY = TEST_SECRET
@@ -144,7 +145,7 @@ schema = st.openapi.from_url(f"{LIVE_SERVER_URL}/openapi.json")
@pytest.mark.fuzz @pytest.mark.fuzz
@st.pytest.parametrize(api=schema) @st.pytest.parametrize(api=schema)
@settings( @settings(
max_examples=3000, max_examples=100 if _QUICK else 3000,
deadline=None, deadline=None,
verbosity=Verbosity.debug, verbosity=Verbosity.debug,
suppress_health_check=[ suppress_health_check=[
@@ -161,7 +162,7 @@ def test_schema_compliance(case):
@pytest.mark.fuzz @pytest.mark.fuzz
@st.pytest.parametrize(api=schema) @st.pytest.parametrize(api=schema)
@settings( @settings(
max_examples=500, max_examples=100 if _QUICK else 500,
deadline=None, deadline=None,
verbosity=Verbosity.normal, verbosity=Verbosity.normal,
suppress_health_check=[ suppress_health_check=[

View File

@@ -18,9 +18,12 @@ from schemathesis.specs.openapi.checks import (
response_headers_conformance, response_headers_conformance,
response_schema_conformance, response_schema_conformance,
) )
import os
from hypothesis import settings, HealthCheck from hypothesis import settings, HealthCheck
from decnet.agent import app as _agent_app_mod from decnet.agent import app as _agent_app_mod
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
from decnet.agent import executor as _exec from decnet.agent import executor as _exec
from decnet.agent import heartbeat as _heartbeat from decnet.agent import heartbeat as _heartbeat
@@ -87,7 +90,7 @@ CHECKS = (
@pytest.mark.fuzz @pytest.mark.fuzz
@SCHEMA.parametrize() @SCHEMA.parametrize()
@settings( @settings(
max_examples=300, max_examples=100 if _QUICK else 300,
deadline=None, deadline=None,
suppress_health_check=[ suppress_health_check=[
HealthCheck.filter_too_much, HealthCheck.filter_too_much,

View File

@@ -17,6 +17,8 @@ os.environ.setdefault("DECNET_JWT_SECRET", "schemathesis-swarm-secret-32chars-mi
import pytest import pytest
import schemathesis as st import schemathesis as st
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
from schemathesis.checks import not_a_server_error from schemathesis.checks import not_a_server_error
from schemathesis.specs.openapi.checks import ( from schemathesis.specs.openapi.checks import (
status_code_conformance, status_code_conformance,
@@ -54,7 +56,7 @@ CHECKS = (
@pytest.mark.fuzz @pytest.mark.fuzz
@SCHEMA.parametrize() @SCHEMA.parametrize()
@settings( @settings(
max_examples=200, max_examples=100 if _QUICK else 200,
deadline=None, deadline=None,
suppress_health_check=[ suppress_health_check=[
HealthCheck.filter_too_much, HealthCheck.filter_too_much,

View File

@@ -24,7 +24,10 @@ Routes covered (all decorated ``tags=["TTP Tagging"]``):
""" """
from __future__ import annotations from __future__ import annotations
import os
import pytest import pytest
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
import schemathesis as st import schemathesis as st
from hypothesis import HealthCheck, Verbosity, settings from hypothesis import HealthCheck, Verbosity, settings
@@ -46,7 +49,7 @@ TTP_SCHEMA = st.openapi.from_url(
@pytest.mark.fuzz @pytest.mark.fuzz
@TTP_SCHEMA.parametrize() @TTP_SCHEMA.parametrize()
@settings( @settings(
max_examples=400, max_examples=100 if _QUICK else 400,
deadline=None, deadline=None,
verbosity=Verbosity.normal, verbosity=Verbosity.normal,
suppress_health_check=[ suppress_health_check=[
@@ -63,7 +66,7 @@ def test_ttp_schema_compliance(case):
@pytest.mark.fuzz @pytest.mark.fuzz
@TTP_SCHEMA.parametrize() @TTP_SCHEMA.parametrize()
@settings( @settings(
max_examples=120, max_examples=100 if _QUICK else 120,
deadline=None, deadline=None,
verbosity=Verbosity.normal, verbosity=Verbosity.normal,
suppress_health_check=[ suppress_health_check=[