perf(pytest): 194s → 4s collection — lazy heavy imports + norecursedirs
Four-part fix for the collection bottleneck that was blocking the dev loop: 1. Lazy mitreattack.stix20 import in attack_stix.py — deferred to first _load() call (TYPE_CHECKING guard at top level) 2. Lazy misp_stix_converter import in both MISP export routers — moved from module level into the route handler body 3. Lazy attack_catalog / attack_stix in ttp.py repo mixin — thin wrapper functions so the import chain never fires at module load time 4. tests/api/conftest.py — `from decnet.web.api import app` moved inside the `client()` fixture; `pytest_ignore_collect` broadened to skip all test_schemathesis*.py variants (not just test_schemathesis.py), which were launching a subprocess server at module-import time 5. pyproject.toml — `norecursedirs` for tests/live, tests/stress, tests/service_testing, tests/docker, tests/perf so these directories are never entered; `-m` filter removed from addopts (now redundant); `--dist loadscope` → `--dist load` to unblock workers immediately 6. behave_core / behave_shell rename — BEHAVE packages dropped the `decnet_` prefix; reinstalled editable installs and updated all 14 import sites across profiler, ttp, bus, and correlation modules
This commit is contained in:
@@ -427,7 +427,7 @@ def attacker_observation(primitive: str) -> str:
|
||||
``cognitive.feedback_loop_engagement``,
|
||||
``motor.shell_mastery.tab_completion``). Dotted primitives are
|
||||
permitted — this matches the format
|
||||
``decnet_behave_shell.spec.event_adapter.event_topic_for`` produces
|
||||
``behave_shell.spec.event_adapter.event_topic_for`` produces
|
||||
upstream, and DECNET's bus admits the dotted leaf the same way
|
||||
:func:`attacker` does for ``session.started``.
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ from decnet.logging import get_logger
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
try:
|
||||
from decnet_behave_shell.spec import (
|
||||
from behave_shell.spec import (
|
||||
PRIMITIVE_REGISTRY,
|
||||
ValueKind,
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features.cognitive import (
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation, Window
|
||||
from behave_core.spec.envelope import Observation, Window
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
import statistics
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -15,7 +15,7 @@ from __future__ import annotations
|
||||
import statistics
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -17,7 +17,7 @@ import collections
|
||||
import re
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -10,7 +10,7 @@ import statistics
|
||||
from itertools import chain
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -11,7 +11,7 @@ import collections
|
||||
import statistics
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -16,7 +16,7 @@ import math
|
||||
import statistics
|
||||
from typing import Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext
|
||||
from decnet.profiler.behave_shell._features._emit import make_observation
|
||||
|
||||
@@ -17,8 +17,8 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Optional
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from decnet_behave_shell.spec.event_adapter import event_topic_for, to_event_payload
|
||||
from behave_core.spec.envelope import Observation
|
||||
from behave_shell.spec.event_adapter import event_topic_for, to_event_payload
|
||||
|
||||
from decnet.logging import get_logger
|
||||
from decnet.profiler.behave_shell import extract_session
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Iterator
|
||||
|
||||
from decnet_behave_core.spec.envelope import Observation
|
||||
from behave_core.spec.envelope import Observation
|
||||
|
||||
from decnet.profiler.behave_shell._ctx import SessionContext, build_session_context
|
||||
from decnet.profiler.behave_shell._features import FEATURES
|
||||
|
||||
@@ -38,9 +38,10 @@ from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from mitreattack.stix20 import MitreAttackData
|
||||
if TYPE_CHECKING:
|
||||
from mitreattack.stix20 import MitreAttackData
|
||||
|
||||
from decnet.ttp.attack_version import (
|
||||
ATTACK_BUNDLE_SHA256,
|
||||
@@ -231,6 +232,7 @@ def loaded_license_path() -> Path | None:
|
||||
|
||||
|
||||
def _load() -> MitreAttackData:
|
||||
from mitreattack.stix20 import MitreAttackData # heavy — lazy on first call
|
||||
global _data, _loaded_path
|
||||
with _data_lock:
|
||||
if _data is not None:
|
||||
|
||||
@@ -77,7 +77,7 @@ class XDecnetBehaveProfile:
|
||||
evidence_ref, identity_ref (optional).
|
||||
|
||||
``schema_version`` matches ``OBSERVATION_SCHEMA_VERSION`` from
|
||||
decnet_behave_shell.spec.envelope — bump when the envelope schema changes.
|
||||
behave_shell.spec.envelope — bump when the envelope schema changes.
|
||||
|
||||
``kd_digraph_simhash`` is the 8-byte digraph SimHash from
|
||||
AttackerIdentity, hex-encoded. Null when identity has not been clustered.
|
||||
|
||||
@@ -209,7 +209,7 @@ def _threat_actor(
|
||||
|
||||
obs_list = observations or []
|
||||
if obs_list or kd_hash is not None:
|
||||
from decnet_behave_shell.spec.envelope import OBSERVATION_SCHEMA_VERSION
|
||||
from behave_shell.spec.envelope import OBSERVATION_SCHEMA_VERSION
|
||||
profile_id = (
|
||||
f"x-decnet-behave-profile--{_uuid.uuid5(_NS, attacker['uuid'])}"
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
emitted Observation envelope.
|
||||
|
||||
Mirrors the BEHAVE-SHELL ``Observation`` Pydantic envelope
|
||||
(``decnet_behave_core.spec.envelope.Observation``) field-for-field, plus
|
||||
(``behave_core.spec.envelope.Observation``) field-for-field, plus
|
||||
one DECNET-side denormalisation (``attacker_uuid``) for cheap joins.
|
||||
The class is named ``ObservationRow`` to avoid colliding with the
|
||||
BEHAVE Pydantic class when both are imported into the same module —
|
||||
|
||||
@@ -13,7 +13,7 @@ Composed onto :class:`SQLModelRepository`. Three public methods:
|
||||
drift charts.
|
||||
|
||||
PII discipline is the BEHAVE envelope's job
|
||||
(``core/decnet_behave_core/spec/envelope.py:3-19``); this mixin does
|
||||
(``core/behave_core/spec/envelope.py:3-19``); this mixin does
|
||||
not validate values — that happens at construction time by the BEHAVE
|
||||
``Observation`` subclass before the dict reaches us.
|
||||
"""
|
||||
|
||||
@@ -20,8 +20,6 @@ from typing import Any
|
||||
from sqlalchemy import func, select
|
||||
from sqlmodel import col
|
||||
|
||||
from decnet.ttp.attack_catalog import technique_name as _technique_name
|
||||
from decnet.ttp.attack_stix import mitre_url_for as _mitre_url_for
|
||||
from decnet.web.db.models import (
|
||||
Attacker,
|
||||
AttackerIdentity,
|
||||
@@ -34,6 +32,16 @@ from decnet.web.db.models.canary import CanaryTrigger
|
||||
from decnet.web.db.sqlmodel_repo._helpers import _MixinBase
|
||||
|
||||
|
||||
def _technique_name(tid: str | None) -> str | None:
|
||||
from decnet.ttp.attack_catalog import technique_name # heavy — lazy on first call
|
||||
return technique_name(tid)
|
||||
|
||||
|
||||
def _mitre_url_for(tid: str | None) -> str | None:
|
||||
from decnet.ttp.attack_stix import mitre_url_for # heavy — lazy on first call
|
||||
return mitre_url_for(tid)
|
||||
|
||||
|
||||
# Confidence floor: tags computed below this value are silently dropped
|
||||
# at insert time. Pinned by tests/ttp/test_confidence.py.
|
||||
_CONFIDENCE_FLOOR: float = 0.3
|
||||
|
||||
@@ -14,7 +14,6 @@ from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.responses import Response
|
||||
|
||||
from decnet.telemetry import traced as _traced
|
||||
from decnet.ttp.misp_export import build_attacker_misp_event
|
||||
from decnet.web.dependencies import require_viewer, repo
|
||||
|
||||
router = APIRouter()
|
||||
@@ -79,6 +78,7 @@ async def api_export_attacker_misp(
|
||||
observations = cast(list[dict[str, Any]], results[8])
|
||||
fingerprint_bounties = cast(list[dict[str, Any]], results[9])
|
||||
|
||||
from decnet.ttp.misp_export import build_attacker_misp_event # heavy — lazy on first call
|
||||
event = build_attacker_misp_event(
|
||||
attacker=attacker,
|
||||
behavior=behavior,
|
||||
|
||||
@@ -19,7 +19,6 @@ from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import Response
|
||||
|
||||
from decnet.telemetry import traced as _traced
|
||||
from decnet.ttp.misp_export import build_fleet_misp_collection
|
||||
from decnet.web.dependencies import require_viewer, repo
|
||||
|
||||
router = APIRouter()
|
||||
@@ -49,6 +48,7 @@ async def api_export_attackers_misp(
|
||||
repo.get_all_observations_for_export(),
|
||||
repo.get_all_fingerprint_bounties_for_export(),
|
||||
)
|
||||
from decnet.ttp.misp_export import build_fleet_misp_collection # heavy — lazy on first call
|
||||
collection = build_fleet_misp_collection(
|
||||
rows=rows,
|
||||
ttp_by_attacker=ttp_by_attacker,
|
||||
|
||||
Reference in New Issue
Block a user