feat(profiler/behave_shell): emit cognitive.exploration_style
Two-axis classification over the first_token_hash sequence: repetition_rate (drilling) vs backtrack_rate (jumping among prior tools). chaotic/targeted/methodical buckets. v0.1 thresholds; D.8 re-tunes.
This commit is contained in:
@@ -14,6 +14,7 @@ from decnet.profiler.behave_shell._ctx import SessionContext
|
|||||||
from decnet.profiler.behave_shell._features.cognitive import (
|
from decnet.profiler.behave_shell._features.cognitive import (
|
||||||
cognitive_load,
|
cognitive_load,
|
||||||
command_branch_diversity,
|
command_branch_diversity,
|
||||||
|
exploration_style,
|
||||||
feedback_loop_engagement,
|
feedback_loop_engagement,
|
||||||
inter_command_consistency,
|
inter_command_consistency,
|
||||||
inter_command_latency_class,
|
inter_command_latency_class,
|
||||||
@@ -47,4 +48,5 @@ FEATURES: tuple[FeatureFn, ...] = (
|
|||||||
feedback_loop_engagement,
|
feedback_loop_engagement,
|
||||||
inter_command_consistency,
|
inter_command_consistency,
|
||||||
cognitive_load,
|
cognitive_load,
|
||||||
|
exploration_style,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ from decnet.profiler.behave_shell._thresholds import (
|
|||||||
COGNITIVE_LOAD_LOW_MAX,
|
COGNITIVE_LOAD_LOW_MAX,
|
||||||
COGNITIVE_LOAD_MEDIUM_MAX,
|
COGNITIVE_LOAD_MEDIUM_MAX,
|
||||||
COGNITIVE_LOAD_PACE_REF_CV,
|
COGNITIVE_LOAD_PACE_REF_CV,
|
||||||
|
EXPLORATION_CHAOTIC_BACKTRACK_MIN,
|
||||||
|
EXPLORATION_TARGETED_REP_MIN,
|
||||||
FEEDBACK_CORRELATION_MIN,
|
FEEDBACK_CORRELATION_MIN,
|
||||||
FEEDBACK_MIN_PAIRS,
|
FEEDBACK_MIN_PAIRS,
|
||||||
INTER_CMD_DELIBERATE_MAX,
|
INTER_CMD_DELIBERATE_MAX,
|
||||||
@@ -179,6 +181,65 @@ def feedback_loop_engagement(ctx: SessionContext) -> Iterator[Observation]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def exploration_style(ctx: SessionContext) -> Iterator[Observation]:
|
||||||
|
"""Emit ``cognitive.exploration_style`` ∈ {methodical, chaotic, targeted}.
|
||||||
|
|
||||||
|
Two-axis classification over the first_token_hash sequence:
|
||||||
|
|
||||||
|
* **methodical** — low repetition, low backtracks. Operator marches
|
||||||
|
forward through new tools.
|
||||||
|
* **targeted** — high repetition (R ≥ EXPLORATION_TARGETED_REP_MIN).
|
||||||
|
Same tool re-invoked repeatedly; the operator is drilling.
|
||||||
|
* **chaotic** — high backtrack rate (J ≥ EXPLORATION_CHAOTIC_BACKTRACK_MIN).
|
||||||
|
Jumps among previously-used tools without a clear thread.
|
||||||
|
|
||||||
|
The registry doesn't permit ``unknown``; below the
|
||||||
|
MIN_COMMANDS_FOR_FULL_CONFIDENCE floor we emit at confidence 0.40
|
||||||
|
rather than skip — the engine has *some* signal, just less of it.
|
||||||
|
Skip emission only when there are no commands at all.
|
||||||
|
"""
|
||||||
|
n = len(ctx.commands)
|
||||||
|
if n == 0:
|
||||||
|
return
|
||||||
|
hashes = [c.first_token_hash for c in ctx.commands]
|
||||||
|
unique = len(set(hashes))
|
||||||
|
repetition_rate = 0.0 if n == 0 else 1.0 - (unique / n)
|
||||||
|
|
||||||
|
# Backtrack: at position i, hashes[i] previously seen at index < i-1
|
||||||
|
# and not equal to hashes[i-1]. (Repeating the immediate predecessor
|
||||||
|
# is "drilling", picked up by repetition_rate; backtrack is the
|
||||||
|
# non-local jump signal.)
|
||||||
|
seen_before: set[str] = set()
|
||||||
|
backtracks = 0
|
||||||
|
transitions = 0
|
||||||
|
if hashes:
|
||||||
|
seen_before.add(hashes[0])
|
||||||
|
for i in range(1, n):
|
||||||
|
transitions += 1
|
||||||
|
if hashes[i] != hashes[i - 1] and hashes[i] in seen_before:
|
||||||
|
backtracks += 1
|
||||||
|
seen_before.add(hashes[i])
|
||||||
|
backtrack_rate = (backtracks / transitions) if transitions else 0.0
|
||||||
|
|
||||||
|
if backtrack_rate >= EXPLORATION_CHAOTIC_BACKTRACK_MIN:
|
||||||
|
value = "chaotic"
|
||||||
|
elif repetition_rate >= EXPLORATION_TARGETED_REP_MIN:
|
||||||
|
value = "targeted"
|
||||||
|
else:
|
||||||
|
value = "methodical"
|
||||||
|
|
||||||
|
if n < MIN_COMMANDS_FOR_FULL_CONFIDENCE:
|
||||||
|
confidence = 0.40
|
||||||
|
else:
|
||||||
|
confidence = 0.60
|
||||||
|
yield make_observation(
|
||||||
|
ctx,
|
||||||
|
primitive="cognitive.exploration_style",
|
||||||
|
value=value,
|
||||||
|
confidence=confidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def cognitive_load(ctx: SessionContext) -> Iterator[Observation]:
|
def cognitive_load(ctx: SessionContext) -> Iterator[Observation]:
|
||||||
"""Emit ``cognitive.cognitive_load`` ∈ {low, medium, high}.
|
"""Emit ``cognitive.cognitive_load`` ∈ {low, medium, high}.
|
||||||
|
|
||||||
|
|||||||
@@ -108,6 +108,26 @@ COGNITIVE_LOAD_PACE_REF_CV: float = 1.50
|
|||||||
COGNITIVE_LOAD_LOW_MAX: float = 0.33
|
COGNITIVE_LOAD_LOW_MAX: float = 0.33
|
||||||
COGNITIVE_LOAD_MEDIUM_MAX: float = 0.67
|
COGNITIVE_LOAD_MEDIUM_MAX: float = 0.67
|
||||||
|
|
||||||
|
# ── cognitive.exploration_style (Step D.2) ─────────────────────────────────
|
||||||
|
# Two-axis classification over the first_token_hash sequence:
|
||||||
|
#
|
||||||
|
# repetition_rate (R) = 1 - (unique_first_tokens / total_commands)
|
||||||
|
# backtrack_rate (J) = transitions where commands[i+1].first_token_hash
|
||||||
|
# appeared anywhere in commands[0..i-1] but is NOT
|
||||||
|
# equal to commands[i].first_token_hash (jumping
|
||||||
|
# back to an older tool, not just repeating).
|
||||||
|
#
|
||||||
|
# J >= EXPLORATION_CHAOTIC_BACKTRACK_MIN → chaotic
|
||||||
|
# else if R >= EXPLORATION_TARGETED_REP_MIN → targeted
|
||||||
|
# else → methodical
|
||||||
|
#
|
||||||
|
# Methodical = low repetition, low backtracks (linear progression through
|
||||||
|
# novel tools). Targeted = high repetition (drilling the same tool).
|
||||||
|
# Chaotic = jumping between prior tools without a clear thread.
|
||||||
|
# v0.1; D.8 re-tunes.
|
||||||
|
EXPLORATION_TARGETED_REP_MIN: float = 0.50
|
||||||
|
EXPLORATION_CHAOTIC_BACKTRACK_MIN: float = 0.30
|
||||||
|
|
||||||
# ── motor.keystroke_cadence (Step B.1) ──────────────────────────────────────
|
# ── motor.keystroke_cadence (Step B.1) ──────────────────────────────────────
|
||||||
# Typing bursts split at gaps > IKI_THINK_MAX_S so think-pauses between
|
# Typing bursts split at gaps > IKI_THINK_MAX_S so think-pauses between
|
||||||
# commands don't inflate the within-burst CV. Mirrors the prototype's
|
# commands don't inflate the within-burst CV. Mirrors the prototype's
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Step D.2: ``cognitive.exploration_style``."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from decnet.profiler.behave_shell import extract_session
|
||||||
|
from decnet.profiler.behave_shell._parse import AsciinemaEvent
|
||||||
|
|
||||||
|
|
||||||
|
def _of(observations: list, primitive: str):
|
||||||
|
obs = [o for o in observations if o.primitive == primitive]
|
||||||
|
assert len(obs) == 1, f"expected exactly one {primitive}, got {len(obs)}"
|
||||||
|
return obs[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _cmds(tokens: list[str]) -> list[AsciinemaEvent]:
|
||||||
|
"""One command per token, evenly spaced one second apart."""
|
||||||
|
events: list[AsciinemaEvent] = []
|
||||||
|
for i, tok in enumerate(tokens):
|
||||||
|
t0 = i * 1.0
|
||||||
|
for j, c in enumerate(tok):
|
||||||
|
events.append((t0 + j * 0.05, "i", c))
|
||||||
|
events.append((t0 + len(tok) * 0.05, "i", "\r"))
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_commands_no_emission() -> None:
|
||||||
|
out = list(extract_session([(0.0, "i", "x")], sid="es-empty"))
|
||||||
|
assert [o for o in out if o.primitive == "cognitive.exploration_style"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_unique_tools_emits_methodical() -> None:
|
||||||
|
"""Linear progression through new tools: low R, low J → methodical."""
|
||||||
|
out = list(extract_session(
|
||||||
|
_cmds(["ls", "ps", "id", "uname", "whoami", "pwd", "env", "date"]),
|
||||||
|
sid="es-meth",
|
||||||
|
))
|
||||||
|
obs = _of(out, "cognitive.exploration_style")
|
||||||
|
assert obs.value == "methodical"
|
||||||
|
|
||||||
|
|
||||||
|
def test_drilling_one_tool_emits_targeted() -> None:
|
||||||
|
"""Same tool repeated → high R, low J → targeted."""
|
||||||
|
out = list(extract_session(
|
||||||
|
_cmds(["curl", "curl", "curl", "curl", "curl", "curl", "curl", "curl"]),
|
||||||
|
sid="es-tgt",
|
||||||
|
))
|
||||||
|
obs = _of(out, "cognitive.exploration_style")
|
||||||
|
assert obs.value == "targeted"
|
||||||
|
|
||||||
|
|
||||||
|
def test_jumping_among_old_tools_emits_chaotic() -> None:
|
||||||
|
"""Backtracking among prior tools → high J → chaotic."""
|
||||||
|
out = list(extract_session(
|
||||||
|
_cmds(["a", "b", "c", "a", "c", "b", "a", "b"]),
|
||||||
|
sid="es-chaos",
|
||||||
|
))
|
||||||
|
obs = _of(out, "cognitive.exploration_style")
|
||||||
|
assert obs.value == "chaotic"
|
||||||
|
|
||||||
|
|
||||||
|
def test_low_sample_count_reduces_confidence() -> None:
|
||||||
|
short = list(extract_session(_cmds(["a", "b", "c"]), sid="es-short"))
|
||||||
|
full = list(extract_session(_cmds(["a", "b", "c", "d", "e", "f"]), sid="es-full"))
|
||||||
|
s = _of(short, "cognitive.exploration_style")
|
||||||
|
f = _of(full, "cognitive.exploration_style")
|
||||||
|
assert s.confidence < f.confidence
|
||||||
|
|
||||||
|
|
||||||
|
def test_pii_no_command_bodies_in_observation() -> None:
|
||||||
|
out = list(extract_session(
|
||||||
|
_cmds(["secret_payload"] * 6),
|
||||||
|
sid="es-pii",
|
||||||
|
))
|
||||||
|
obs = _of(out, "cognitive.exploration_style")
|
||||||
|
assert "secret_payload" not in obs.model_dump_json()
|
||||||
Reference in New Issue
Block a user