feat(test): add test-schema target and SCHEMA_QUICK=1 mode for schemathesis
- Add dedicated test-schema Makefile target (xdist logical, 600s timeout, -m fuzz) so schemathesis runs separately from test-fuzz, which was spinning up competing uvicorn workers per xdist process - Exclude all test_schemathesis*.py files from FUZZ_FLAGS via --ignore - Add schema to _ALL_SUITES between api and fuzz - Add SCHEMA_QUICK env var (default 0): caps every max_examples to 100 across all four schemathesis files (4520 -> 600 total examples) - Fix pre-push hook: use .311 venv and delegate to make test-all FAIL_FAST=0 instead of hand-rolling five separate pytest invocations
This commit is contained in:
25
Makefile
25
Makefile
@@ -6,7 +6,13 @@ ARGS :=
|
|||||||
# Unit suites inherit that; special suites clear it with --override-ini.
|
# Unit suites inherit that; special suites clear it with --override-ini.
|
||||||
UNIT_FLAGS := --timeout=30 --timeout-method=thread
|
UNIT_FLAGS := --timeout=30 --timeout-method=thread
|
||||||
SEQ_FLAGS := --override-ini="addopts=-v -x" -n logical --timeout=120 --timeout-method=thread
|
SEQ_FLAGS := --override-ini="addopts=-v -x" -n logical --timeout=120 --timeout-method=thread
|
||||||
FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz
|
FUZZ_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz \
|
||||||
|
--ignore=tests/api/test_schemathesis.py \
|
||||||
|
--ignore=tests/api/test_schemathesis_agent.py \
|
||||||
|
--ignore=tests/api/test_schemathesis_swarm.py \
|
||||||
|
--ignore=tests/api/test_schemathesis_ttp.py
|
||||||
|
SCHEMA_QUICK ?= 0
|
||||||
|
SCHEMA_FLAGS := --override-ini="addopts=-v -x" -n logical -m fuzz --timeout=600 --timeout-method=thread
|
||||||
BENCH_FLAGS := --override-ini="addopts=-v" -p no:xdist --benchmark-only -m bench
|
BENCH_FLAGS := --override-ini="addopts=-v" -p no:xdist --benchmark-only -m bench
|
||||||
|
|
||||||
# ── Unit suites (xdist, 30s timeout) ─────────────────────────────────────────
|
# ── Unit suites (xdist, 30s timeout) ─────────────────────────────────────────
|
||||||
@@ -103,6 +109,15 @@ test-service:
|
|||||||
test-fuzz:
|
test-fuzz:
|
||||||
$(PYTEST) $(FUZZ_FLAGS) $(ARGS)
|
$(PYTEST) $(FUZZ_FLAGS) $(ARGS)
|
||||||
|
|
||||||
|
.PHONY: test-schema
|
||||||
|
test-schema:
|
||||||
|
SCHEMA_QUICK=$(SCHEMA_QUICK) $(PYTEST) \
|
||||||
|
tests/api/test_schemathesis.py \
|
||||||
|
tests/api/test_schemathesis_agent.py \
|
||||||
|
tests/api/test_schemathesis_swarm.py \
|
||||||
|
tests/api/test_schemathesis_ttp.py \
|
||||||
|
$(SCHEMA_FLAGS) $(ARGS)
|
||||||
|
|
||||||
.PHONY: test-bench
|
.PHONY: test-bench
|
||||||
test-bench:
|
test-bench:
|
||||||
$(PYTEST) tests/perf $(BENCH_FLAGS) $(ARGS)
|
$(PYTEST) tests/perf $(BENCH_FLAGS) $(ARGS)
|
||||||
@@ -133,7 +148,7 @@ test-pip-audit:
|
|||||||
|
|
||||||
_ALL_SUITES := core web db bus ttp intel analysis infra fleet cli features \
|
_ALL_SUITES := core web db bus ttp intel analysis infra fleet cli features \
|
||||||
go react \
|
go react \
|
||||||
live api stress service fuzz bench docker \
|
live api schema stress service fuzz bench docker \
|
||||||
mypy bandit vulture pip-audit
|
mypy bandit vulture pip-audit
|
||||||
|
|
||||||
.PHONY: test-all test
|
.PHONY: test-all test
|
||||||
@@ -185,7 +200,9 @@ help:
|
|||||||
@echo " make test-api tests/api (schemathesis)"
|
@echo " make test-api tests/api (schemathesis)"
|
||||||
@echo " make test-stress tests/stress"
|
@echo " make test-stress tests/stress"
|
||||||
@echo " make test-service tests/service_testing"
|
@echo " make test-service tests/service_testing"
|
||||||
@echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz)"
|
@echo " make test-schema schemathesis contract tests (-m fuzz, xdist logical)"
|
||||||
|
@echo " make test-schema SCHEMA_QUICK=1 same, capped at 100 examples per test"
|
||||||
|
@echo " make test-fuzz hypothesis fuzz (all normal dirs, -m fuzz, skips schemathesis files)"
|
||||||
@echo " make test-bench tests/perf"
|
@echo " make test-bench tests/perf"
|
||||||
@echo " make test-docker tests/docker (needs DECNET_LIVE_DOCKER=1)"
|
@echo " make test-docker tests/docker (needs DECNET_LIVE_DOCKER=1)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@@ -196,7 +213,7 @@ help:
|
|||||||
@echo " make test-pip-audit pip-audit dependency vulnerability scan"
|
@echo " make test-pip-audit pip-audit dependency vulnerability scan"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Composites:"
|
@echo "Composites:"
|
||||||
@echo " make test-all ALL suites (unit + go + react + live + api + fuzz + bench + stress + docker + static analysis)"
|
@echo " make test-all ALL suites (unit + go + react + live + api + schema + fuzz + bench + stress + docker + static analysis)"
|
||||||
@echo " make test-all FAIL_FAST=0 same, report all failures instead of stopping"
|
@echo " make test-all FAIL_FAST=0 same, report all failures instead of stopping"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Passthrough: make test-web ARGS='--lf -s'"
|
@echo "Passthrough: make test-web ARGS='--lf -s'"
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ def _free_port() -> int:
|
|||||||
LIVE_PORT = _free_port()
|
LIVE_PORT = _free_port()
|
||||||
LIVE_SERVER_URL = f"http://127.0.0.1:{LIVE_PORT}"
|
LIVE_SERVER_URL = f"http://127.0.0.1:{LIVE_PORT}"
|
||||||
TEST_SECRET = "test-secret-for-automated-fuzzing"
|
TEST_SECRET = "test-secret-for-automated-fuzzing"
|
||||||
|
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
|
||||||
|
|
||||||
import decnet.web.auth
|
import decnet.web.auth
|
||||||
decnet.web.auth.SECRET_KEY = TEST_SECRET
|
decnet.web.auth.SECRET_KEY = TEST_SECRET
|
||||||
@@ -144,7 +145,7 @@ schema = st.openapi.from_url(f"{LIVE_SERVER_URL}/openapi.json")
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@st.pytest.parametrize(api=schema)
|
@st.pytest.parametrize(api=schema)
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=3000,
|
max_examples=100 if _QUICK else 3000,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
verbosity=Verbosity.debug,
|
verbosity=Verbosity.debug,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
@@ -161,7 +162,7 @@ def test_schema_compliance(case):
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@st.pytest.parametrize(api=schema)
|
@st.pytest.parametrize(api=schema)
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=500,
|
max_examples=100 if _QUICK else 500,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
verbosity=Verbosity.normal,
|
verbosity=Verbosity.normal,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
|
|||||||
@@ -18,9 +18,12 @@ from schemathesis.specs.openapi.checks import (
|
|||||||
response_headers_conformance,
|
response_headers_conformance,
|
||||||
response_schema_conformance,
|
response_schema_conformance,
|
||||||
)
|
)
|
||||||
|
import os
|
||||||
from hypothesis import settings, HealthCheck
|
from hypothesis import settings, HealthCheck
|
||||||
|
|
||||||
from decnet.agent import app as _agent_app_mod
|
from decnet.agent import app as _agent_app_mod
|
||||||
|
|
||||||
|
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
|
||||||
from decnet.agent import executor as _exec
|
from decnet.agent import executor as _exec
|
||||||
from decnet.agent import heartbeat as _heartbeat
|
from decnet.agent import heartbeat as _heartbeat
|
||||||
|
|
||||||
@@ -87,7 +90,7 @@ CHECKS = (
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@SCHEMA.parametrize()
|
@SCHEMA.parametrize()
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=300,
|
max_examples=100 if _QUICK else 300,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
HealthCheck.filter_too_much,
|
HealthCheck.filter_too_much,
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ os.environ.setdefault("DECNET_JWT_SECRET", "schemathesis-swarm-secret-32chars-mi
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import schemathesis as st
|
import schemathesis as st
|
||||||
|
|
||||||
|
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
|
||||||
from schemathesis.checks import not_a_server_error
|
from schemathesis.checks import not_a_server_error
|
||||||
from schemathesis.specs.openapi.checks import (
|
from schemathesis.specs.openapi.checks import (
|
||||||
status_code_conformance,
|
status_code_conformance,
|
||||||
@@ -54,7 +56,7 @@ CHECKS = (
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@SCHEMA.parametrize()
|
@SCHEMA.parametrize()
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=200,
|
max_examples=100 if _QUICK else 200,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
HealthCheck.filter_too_much,
|
HealthCheck.filter_too_much,
|
||||||
|
|||||||
@@ -24,7 +24,10 @@ Routes covered (all decorated ``tags=["TTP Tagging"]``):
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
_QUICK = os.getenv("SCHEMA_QUICK") == "1"
|
||||||
import schemathesis as st
|
import schemathesis as st
|
||||||
from hypothesis import HealthCheck, Verbosity, settings
|
from hypothesis import HealthCheck, Verbosity, settings
|
||||||
|
|
||||||
@@ -46,7 +49,7 @@ TTP_SCHEMA = st.openapi.from_url(
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@TTP_SCHEMA.parametrize()
|
@TTP_SCHEMA.parametrize()
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=400,
|
max_examples=100 if _QUICK else 400,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
verbosity=Verbosity.normal,
|
verbosity=Verbosity.normal,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
@@ -63,7 +66,7 @@ def test_ttp_schema_compliance(case):
|
|||||||
@pytest.mark.fuzz
|
@pytest.mark.fuzz
|
||||||
@TTP_SCHEMA.parametrize()
|
@TTP_SCHEMA.parametrize()
|
||||||
@settings(
|
@settings(
|
||||||
max_examples=120,
|
max_examples=100 if _QUICK else 120,
|
||||||
deadline=None,
|
deadline=None,
|
||||||
verbosity=Verbosity.normal,
|
verbosity=Verbosity.normal,
|
||||||
suppress_health_check=[
|
suppress_health_check=[
|
||||||
|
|||||||
Reference in New Issue
Block a user