Refactor: implemented Repository Factory and Async Mutator Engine. Decoupled storage logic and enforced Dependency Injection across CLI and Web API. Updated documentation.
Some checks failed
CI / Lint (ruff) (push) Successful in 12s
CI / SAST (bandit) (push) Successful in 13s
CI / Dependency audit (pip-audit) (push) Successful in 22s
CI / Test (Standard) (3.11) (push) Failing after 54s
CI / Test (Standard) (3.12) (push) Successful in 1m35s
CI / Test (Live) (3.11) (push) Has been skipped
CI / Test (Fuzz) (3.11) (push) Has been skipped
CI / Merge dev → testing (push) Has been skipped
CI / Prepare Merge to Main (push) Has been skipped
CI / Finalize Merge to Main (push) Has been skipped
Some checks failed
CI / Lint (ruff) (push) Successful in 12s
CI / SAST (bandit) (push) Successful in 13s
CI / Dependency audit (pip-audit) (push) Successful in 22s
CI / Test (Standard) (3.11) (push) Failing after 54s
CI / Test (Standard) (3.12) (push) Successful in 1m35s
CI / Test (Live) (3.11) (push) Has been skipped
CI / Test (Fuzz) (3.11) (push) Has been skipped
CI / Merge dev → testing (push) Has been skipped
CI / Prepare Merge to Main (push) Has been skipped
CI / Finalize Merge to Main (push) Has been skipped
This commit is contained in:
@@ -252,7 +252,7 @@ def deploy(
|
||||
console.print("[red]Failed to start mutator watcher.[/]")
|
||||
|
||||
if effective_log_file and not dry_run and not api:
|
||||
import subprocess # noqa: F811 # nosec B404
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
from pathlib import Path as _Path
|
||||
_collector_err = _Path(effective_log_file).with_suffix(".collector.log")
|
||||
@@ -301,18 +301,20 @@ def mutate(
|
||||
force_all: bool = typer.Option(False, "--all", help="Force mutate all deckies immediately"),
|
||||
) -> None:
|
||||
"""Manually trigger or continuously watch for decky mutation."""
|
||||
import asyncio
|
||||
from decnet.mutator import mutate_decky, mutate_all, run_watch_loop
|
||||
from decnet.web.dependencies import repo
|
||||
|
||||
if watch:
|
||||
run_watch_loop()
|
||||
asyncio.run(run_watch_loop(repo))
|
||||
return
|
||||
|
||||
if decky_name:
|
||||
mutate_decky(decky_name)
|
||||
asyncio.run(mutate_decky(decky_name, repo))
|
||||
elif force_all:
|
||||
mutate_all(force=True)
|
||||
asyncio.run(mutate_all(force=True, repo=repo))
|
||||
else:
|
||||
mutate_all(force=False)
|
||||
asyncio.run(mutate_all(force=False, repo=repo))
|
||||
|
||||
|
||||
@app.command()
|
||||
|
||||
@@ -5,9 +5,9 @@ from decnet.correlation.graph import AttackerTraversal, TraversalHop
|
||||
from decnet.correlation.parser import LogEvent, parse_line
|
||||
|
||||
__all__ = [
|
||||
"CorrelationEngine",
|
||||
"AttackerTraversal",
|
||||
"TraversalHop",
|
||||
"CorrelationEngine",
|
||||
"LogEvent",
|
||||
"TraversalHop",
|
||||
"parse_line",
|
||||
]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Calculate absolute path to the project root
|
||||
@@ -30,7 +31,7 @@ def _require_env(name: str) -> str:
|
||||
f"Required environment variable '{name}' is not set. "
|
||||
f"Set it in .env.local or export it before starting DECNET."
|
||||
)
|
||||
|
||||
|
||||
if any(k.startswith("PYTEST") for k in os.environ):
|
||||
return value
|
||||
|
||||
@@ -55,6 +56,10 @@ DECNET_ADMIN_USER: str = os.environ.get("DECNET_ADMIN_USER", "admin")
|
||||
DECNET_ADMIN_PASSWORD: str = os.environ.get("DECNET_ADMIN_PASSWORD", "admin")
|
||||
DECNET_DEVELOPER: bool = os.environ.get("DECNET_DEVELOPER", "False").lower() == "true"
|
||||
|
||||
# Database Options
|
||||
DECNET_DB_TYPE: str = os.environ.get("DECNET_DB_TYPE", "sqlite").lower()
|
||||
DECNET_DB_URL: Optional[str] = os.environ.get("DECNET_DB_URL")
|
||||
|
||||
# CORS — comma-separated list of allowed origins for the web dashboard API.
|
||||
# Defaults to the configured web host/port. Override with DECNET_CORS_ORIGINS if needed.
|
||||
# Example: DECNET_CORS_ORIGINS=http://192.168.1.50:9090,https://dashboard.example.com
|
||||
|
||||
@@ -123,7 +123,7 @@ def _parse_configparser(cp: configparser.ConfigParser) -> IniConfig:
|
||||
for section in cp.sections():
|
||||
if section == "general":
|
||||
continue
|
||||
|
||||
|
||||
# A service sub-section is identified if the section name has at least one dot
|
||||
# AND the last segment is a known service name.
|
||||
# e.g. "decky-01.ssh" -> sub-section
|
||||
@@ -151,7 +151,7 @@ def _parse_configparser(cp: configparser.ConfigParser) -> IniConfig:
|
||||
services = [sv.strip() for sv in svc_raw.split(",")] if svc_raw else None
|
||||
archetype = s.get("archetype")
|
||||
nmap_os = s.get("nmap_os") or s.get("nmap-os") or None
|
||||
|
||||
|
||||
mi_raw = s.get("mutate_interval") or s.get("mutate-interval")
|
||||
mutate_interval = None
|
||||
if mi_raw:
|
||||
@@ -199,11 +199,11 @@ def _parse_configparser(cp: configparser.ConfigParser) -> IniConfig:
|
||||
for section in cp.sections():
|
||||
if "." not in section:
|
||||
continue
|
||||
|
||||
|
||||
decky_name, dot, svc_name = section.rpartition(".")
|
||||
if svc_name not in known_services:
|
||||
continue # not a service sub-section
|
||||
|
||||
|
||||
svc_cfg = {k: v for k, v in cp[section].items()}
|
||||
if decky_name in decky_map:
|
||||
# Direct match — single decky
|
||||
|
||||
@@ -12,25 +12,29 @@ from rich.console import Console
|
||||
from decnet.archetypes import get_archetype
|
||||
from decnet.fleet import all_service_names
|
||||
from decnet.composer import write_compose
|
||||
from decnet.config import DeckyConfig, load_state, save_state
|
||||
from decnet.config import DeckyConfig, DecnetConfig
|
||||
from decnet.engine import _compose_with_retry
|
||||
|
||||
import subprocess # nosec B404
|
||||
from pathlib import Path
|
||||
import anyio
|
||||
import asyncio
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def mutate_decky(decky_name: str) -> bool:
|
||||
async def mutate_decky(decky_name: str, repo: BaseRepository) -> bool:
|
||||
"""
|
||||
Perform an Intra-Archetype Shuffle for a specific decky.
|
||||
Returns True if mutation succeeded, False otherwise.
|
||||
"""
|
||||
state = load_state()
|
||||
if state is None:
|
||||
console.print("[red]No active deployment found (no decnet-state.json).[/]")
|
||||
state_dict = await repo.get_state("deployment")
|
||||
if state_dict is None:
|
||||
console.print("[red]No active deployment found in database.[/]")
|
||||
return False
|
||||
|
||||
config, compose_path = state
|
||||
config = DecnetConfig(**state_dict["config"])
|
||||
compose_path = Path(state_dict["compose_path"])
|
||||
decky: Optional[DeckyConfig] = next((d for d in config.deckies if d.name == decky_name), None)
|
||||
|
||||
if not decky:
|
||||
@@ -63,31 +67,35 @@ def mutate_decky(decky_name: str) -> bool:
|
||||
decky.services = list(chosen)
|
||||
decky.last_mutated = time.time()
|
||||
|
||||
save_state(config, compose_path)
|
||||
# Save to DB
|
||||
await repo.set_state("deployment", {"config": config.model_dump(), "compose_path": str(compose_path)})
|
||||
|
||||
# Still writes files for Docker to use
|
||||
write_compose(config, compose_path)
|
||||
|
||||
console.print(f"[cyan]Mutating '{decky_name}' to services: {', '.join(decky.services)}[/]")
|
||||
|
||||
try:
|
||||
_compose_with_retry("up", "-d", "--remove-orphans", compose_file=compose_path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]Failed to mutate '{decky_name}': {e.stderr}[/]")
|
||||
# Wrap blocking call in thread
|
||||
await anyio.to_thread.run_sync(_compose_with_retry, "up", "-d", "--remove-orphans", compose_path)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Failed to mutate '{decky_name}': {e}[/]")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def mutate_all(force: bool = False) -> None:
|
||||
async def mutate_all(repo: BaseRepository, force: bool = False) -> None:
|
||||
"""
|
||||
Check all deckies and mutate those that are due.
|
||||
If force=True, mutates all deckies regardless of schedule.
|
||||
"""
|
||||
state = load_state()
|
||||
if state is None:
|
||||
state_dict = await repo.get_state("deployment")
|
||||
if state_dict is None:
|
||||
console.print("[red]No active deployment found.[/]")
|
||||
return
|
||||
|
||||
config, _ = state
|
||||
config = DecnetConfig(**state_dict["config"])
|
||||
now = time.time()
|
||||
|
||||
mutated_count = 0
|
||||
@@ -103,7 +111,7 @@ def mutate_all(force: bool = False) -> None:
|
||||
due = elapsed_secs >= (interval_mins * 60)
|
||||
|
||||
if due:
|
||||
success = mutate_decky(decky.name)
|
||||
success = await mutate_decky(decky.name, repo=repo)
|
||||
if success:
|
||||
mutated_count += 1
|
||||
|
||||
@@ -111,12 +119,12 @@ def mutate_all(force: bool = False) -> None:
|
||||
console.print("[dim]No deckies are due for mutation.[/]")
|
||||
|
||||
|
||||
def run_watch_loop(poll_interval_secs: int = 10) -> None:
|
||||
async def run_watch_loop(repo: BaseRepository, poll_interval_secs: int = 10) -> None:
|
||||
"""Run an infinite loop checking for deckies that need mutation."""
|
||||
console.print(f"[green]DECNET Mutator Watcher started (polling every {poll_interval_secs}s).[/]")
|
||||
try:
|
||||
while True:
|
||||
mutate_all(force=False)
|
||||
time.sleep(poll_interval_secs)
|
||||
await mutate_all(force=False, repo=repo)
|
||||
await asyncio.sleep(poll_interval_secs)
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[dim]Mutator watcher stopped.[/]")
|
||||
|
||||
@@ -32,28 +32,38 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
log.error("DB failed to initialize after 5 attempts — startup may be degraded")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Start background ingestion task
|
||||
if ingestion_task is None or ingestion_task.done():
|
||||
ingestion_task = asyncio.create_task(log_ingestion_worker(repo))
|
||||
# Start background tasks only if not in contract test mode
|
||||
if os.environ.get("DECNET_CONTRACT_TEST") != "true":
|
||||
# Start background ingestion task
|
||||
if ingestion_task is None or ingestion_task.done():
|
||||
ingestion_task = asyncio.create_task(log_ingestion_worker(repo))
|
||||
|
||||
# Start Docker log collector (writes to log file; ingester reads from it)
|
||||
_log_file = os.environ.get("DECNET_INGEST_LOG_FILE", DECNET_INGEST_LOG_FILE)
|
||||
if _log_file and (collector_task is None or collector_task.done()):
|
||||
collector_task = asyncio.create_task(log_collector_worker(_log_file))
|
||||
# Start Docker log collector (writes to log file; ingester reads from it)
|
||||
_log_file = os.environ.get("DECNET_INGEST_LOG_FILE", DECNET_INGEST_LOG_FILE)
|
||||
if _log_file and (collector_task is None or collector_task.done()):
|
||||
collector_task = asyncio.create_task(log_collector_worker(_log_file))
|
||||
elif not _log_file:
|
||||
log.warning("DECNET_INGEST_LOG_FILE not set — Docker log collection disabled.")
|
||||
else:
|
||||
log.warning("DECNET_INGEST_LOG_FILE not set — Docker log collection disabled.")
|
||||
log.info("Contract Test Mode: skipping background worker startup")
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown background tasks
|
||||
for task in (ingestion_task, collector_task):
|
||||
if task:
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
log.warning("Task shutdown error: %s", exc)
|
||||
|
||||
|
||||
app: FastAPI = FastAPI(
|
||||
title="DECNET Web Dashboard API",
|
||||
version="1.0.0",
|
||||
title="DECNET Web Dashboard API",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
docs_url="/docs" if DECNET_DEVELOPER else None,
|
||||
redoc_url="/redoc" if DECNET_DEVELOPER else None,
|
||||
|
||||
@@ -12,7 +12,7 @@ ACCESS_TOKEN_EXPIRE_MINUTES: int = 1440
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
return bcrypt.checkpw(
|
||||
plain_password.encode("utf-8")[:72],
|
||||
plain_password.encode("utf-8")[:72],
|
||||
hashed_password.encode("utf-8")
|
||||
)
|
||||
|
||||
@@ -31,7 +31,7 @@ def create_access_token(data: dict[str, Any], expires_delta: Optional[timedelta]
|
||||
_expire = datetime.now(timezone.utc) + expires_delta
|
||||
else:
|
||||
_expire = datetime.now(timezone.utc) + timedelta(minutes=15)
|
||||
|
||||
|
||||
_to_encode.update({"exp": _expire})
|
||||
_to_encode.update({"iat": datetime.now(timezone.utc)})
|
||||
_encoded_jwt: str = jwt.encode(_to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
||||
|
||||
18
decnet/web/db/factory.py
Normal file
18
decnet/web/db/factory.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from typing import Any
|
||||
from decnet.env import os
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
def get_repository(**kwargs: Any) -> BaseRepository:
|
||||
"""Factory function to instantiate the correct repository implementation based on environment."""
|
||||
db_type = os.environ.get("DECNET_DB_TYPE", "sqlite").lower()
|
||||
|
||||
if db_type == "sqlite":
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
return SQLiteRepository(**kwargs)
|
||||
elif db_type == "mysql":
|
||||
# Placeholder for future implementation
|
||||
# from decnet.web.db.mysql.repository import MySQLRepository
|
||||
# return MySQLRepository()
|
||||
raise NotImplementedError("MySQL support is planned but not yet implemented.")
|
||||
else:
|
||||
raise ValueError(f"Unsupported database type: {db_type}")
|
||||
@@ -22,7 +22,7 @@ class Log(SQLModel, table=True):
|
||||
event_type: str = Field(index=True)
|
||||
attacker_ip: str = Field(index=True)
|
||||
raw_line: str
|
||||
fields: str
|
||||
fields: str
|
||||
msg: Optional[str] = None
|
||||
|
||||
class Bounty(SQLModel, table=True):
|
||||
@@ -35,6 +35,12 @@ class Bounty(SQLModel, table=True):
|
||||
bounty_type: str = Field(index=True)
|
||||
payload: str
|
||||
|
||||
|
||||
class State(SQLModel, table=True):
|
||||
__tablename__ = "state"
|
||||
key: str = Field(primary_key=True)
|
||||
value: str # Stores JSON serialized DecnetConfig or other state blobs
|
||||
|
||||
# --- API Request/Response Models (Pydantic) ---
|
||||
|
||||
class Token(BaseModel):
|
||||
|
||||
@@ -17,9 +17,9 @@ class BaseRepository(ABC):
|
||||
|
||||
@abstractmethod
|
||||
async def get_logs(
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
search: Optional[str] = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Retrieve paginated log entries."""
|
||||
@@ -67,9 +67,9 @@ class BaseRepository(ABC):
|
||||
|
||||
@abstractmethod
|
||||
async def get_bounties(
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
bounty_type: Optional[str] = None,
|
||||
search: Optional[str] = None
|
||||
) -> list[dict[str, Any]]:
|
||||
@@ -80,3 +80,13 @@ class BaseRepository(ABC):
|
||||
async def get_total_bounties(self, bounty_type: Optional[str] = None, search: Optional[str] = None) -> int:
|
||||
"""Retrieve the total count of bounties, optionally filtered."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
|
||||
"""Retrieve a specific state entry by key."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def set_state(self, key: str, value: Any) -> None:
|
||||
"""Store a specific state entry by key."""
|
||||
pass
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy import create_engine, Engine
|
||||
from sqlmodel import SQLModel
|
||||
from typing import AsyncGenerator
|
||||
|
||||
# We need both sync and async engines for SQLite
|
||||
# Sync for initialization (DDL) and async for standard queries
|
||||
|
||||
def get_async_engine(db_path: str):
|
||||
def get_async_engine(db_path: str) -> AsyncEngine:
|
||||
# If it's a memory URI, don't add the extra slash that turns it into a relative file
|
||||
prefix = "sqlite+aiosqlite:///"
|
||||
if db_path.startswith("file:"):
|
||||
prefix = "sqlite+aiosqlite:///"
|
||||
if db_path.startswith(":memory:"):
|
||||
prefix = "sqlite+aiosqlite://"
|
||||
return create_async_engine(f"{prefix}{db_path}", echo=False, connect_args={"uri": True})
|
||||
|
||||
def get_sync_engine(db_path: str):
|
||||
def get_sync_engine(db_path: str) -> Engine:
|
||||
prefix = "sqlite:///"
|
||||
if db_path.startswith(":memory:"):
|
||||
prefix = "sqlite://"
|
||||
return create_engine(f"{prefix}{db_path}", echo=False, connect_args={"uri": True})
|
||||
|
||||
def init_db(db_path: str):
|
||||
def init_db(db_path: str) -> None:
|
||||
"""Synchronously create all tables."""
|
||||
engine = get_sync_engine(db_path)
|
||||
# Ensure WAL mode is set
|
||||
@@ -25,7 +28,7 @@ def init_db(db_path: str):
|
||||
conn.exec_driver_sql("PRAGMA synchronous=NORMAL")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
async def get_session(engine) -> AsyncSession:
|
||||
async def get_session(engine: AsyncEngine) -> AsyncGenerator[AsyncSession, None]:
|
||||
async_session = async_sessionmaker(
|
||||
engine, class_=AsyncSession, expire_on_commit=False
|
||||
)
|
||||
|
||||
@@ -6,12 +6,13 @@ from typing import Any, Optional, List
|
||||
|
||||
from sqlalchemy import func, select, desc, asc, text, or_, update, literal_column
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlmodel.sql.expression import SelectOfScalar
|
||||
|
||||
from decnet.config import load_state, _ROOT
|
||||
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
|
||||
from decnet.web.auth import get_password_hash
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
from decnet.web.db.models import User, Log, Bounty
|
||||
from decnet.web.db.models import User, Log, Bounty, State
|
||||
from decnet.web.db.sqlite.database import get_async_engine, init_db
|
||||
|
||||
|
||||
@@ -93,11 +94,11 @@ class SQLiteRepository(BaseRepository):
|
||||
|
||||
def _apply_filters(
|
||||
self,
|
||||
statement,
|
||||
statement: SelectOfScalar,
|
||||
search: Optional[str],
|
||||
start_time: Optional[str],
|
||||
end_time: Optional[str],
|
||||
):
|
||||
) -> SelectOfScalar:
|
||||
import re
|
||||
import shlex
|
||||
|
||||
@@ -128,9 +129,10 @@ class SQLiteRepository(BaseRepository):
|
||||
statement = statement.where(core_fields[key] == val)
|
||||
else:
|
||||
key_safe = re.sub(r"[^a-zA-Z0-9_]", "", key)
|
||||
statement = statement.where(
|
||||
text(f"json_extract(fields, '$.{key_safe}') = :val")
|
||||
).params(val=val)
|
||||
if key_safe:
|
||||
statement = statement.where(
|
||||
text(f"json_extract(fields, '$.{key_safe}') = :val")
|
||||
).params(val=val)
|
||||
else:
|
||||
lk = f"%{token}%"
|
||||
statement = statement.where(
|
||||
@@ -206,7 +208,7 @@ class SQLiteRepository(BaseRepository):
|
||||
end_time: Optional[str] = None,
|
||||
interval_minutes: int = 15,
|
||||
) -> List[dict]:
|
||||
bucket_seconds = interval_minutes * 60
|
||||
bucket_seconds = max(interval_minutes, 1) * 60
|
||||
bucket_expr = literal_column(
|
||||
f"datetime((strftime('%s', timestamp) / {bucket_seconds}) * {bucket_seconds}, 'unixepoch')"
|
||||
).label("bucket_time")
|
||||
@@ -299,7 +301,12 @@ class SQLiteRepository(BaseRepository):
|
||||
session.add(Bounty(**data))
|
||||
await session.commit()
|
||||
|
||||
def _apply_bounty_filters(self, statement, bounty_type: Optional[str], search: Optional[str]):
|
||||
def _apply_bounty_filters(
|
||||
self,
|
||||
statement: SelectOfScalar,
|
||||
bounty_type: Optional[str],
|
||||
search: Optional[str]
|
||||
) -> SelectOfScalar:
|
||||
if bounty_type:
|
||||
statement = statement.where(Bounty.bounty_type == bounty_type)
|
||||
if search:
|
||||
@@ -350,3 +357,29 @@ class SQLiteRepository(BaseRepository):
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(statement)
|
||||
return result.scalar() or 0
|
||||
|
||||
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
|
||||
async with self.session_factory() as session:
|
||||
statement = select(State).where(State.key == key)
|
||||
result = await session.execute(statement)
|
||||
state = result.scalar_one_none()
|
||||
if state:
|
||||
return json.loads(state.value)
|
||||
return None
|
||||
|
||||
async def set_state(self, key: str, value: Any) -> None: # noqa: ANN401
|
||||
async with self.session_factory() as session:
|
||||
# Check if exists
|
||||
statement = select(State).where(State.key == key)
|
||||
result = await session.execute(statement)
|
||||
state = result.scalar_one_none()
|
||||
|
||||
value_json = json.dumps(value)
|
||||
if state:
|
||||
state.value = value_json
|
||||
session.add(state)
|
||||
else:
|
||||
new_state = State(key=key, value=value_json)
|
||||
session.add(new_state)
|
||||
|
||||
await session.commit()
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from typing import Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
import jwt
|
||||
from fastapi import HTTPException, status, Request
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
|
||||
from decnet.web.auth import ALGORITHM, SECRET_KEY
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
from decnet.web.db.factory import get_repository
|
||||
|
||||
# Root directory for database
|
||||
_ROOT_DIR = Path(__file__).parent.parent.parent.absolute()
|
||||
DB_PATH = _ROOT_DIR / "decnet.db"
|
||||
# Shared repository singleton
|
||||
repo: BaseRepository = get_repository()
|
||||
|
||||
# Shared repository instance
|
||||
repo = SQLiteRepository(db_path=str(DB_PATH))
|
||||
def get_repo() -> BaseRepository:
|
||||
"""FastAPI dependency to inject the configured repository."""
|
||||
return repo
|
||||
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login")
|
||||
|
||||
@@ -53,7 +53,7 @@ async def get_current_user(request: Request) -> str:
|
||||
detail="Could not validate credentials",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
auth_header = request.headers.get("Authorization")
|
||||
token: str | None = (
|
||||
auth_header.split(" ", 1)[1]
|
||||
|
||||
@@ -21,7 +21,7 @@ async def log_ingestion_worker(repo: BaseRepository) -> None:
|
||||
|
||||
_json_log_path: Path = Path(_base_log_file).with_suffix(".json")
|
||||
_position: int = 0
|
||||
|
||||
|
||||
logger.info(f"Starting JSON log ingestion from {_json_log_path}")
|
||||
|
||||
while True:
|
||||
@@ -29,24 +29,24 @@ async def log_ingestion_worker(repo: BaseRepository) -> None:
|
||||
if not _json_log_path.exists():
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
|
||||
_stat: os.stat_result = _json_log_path.stat()
|
||||
if _stat.st_size < _position:
|
||||
# File rotated or truncated
|
||||
_position = 0
|
||||
|
||||
|
||||
if _stat.st_size == _position:
|
||||
# No new data
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
|
||||
|
||||
with open(_json_log_path, "r", encoding="utf-8", errors="replace") as _f:
|
||||
_f.seek(_position)
|
||||
while True:
|
||||
_line: str = _f.readline()
|
||||
if not _line:
|
||||
break # EOF reached
|
||||
|
||||
|
||||
if not _line.endswith('\n'):
|
||||
# Partial line read, don't process yet, don't advance position
|
||||
break
|
||||
@@ -58,14 +58,19 @@ async def log_ingestion_worker(repo: BaseRepository) -> None:
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f"Failed to decode JSON log line: {_line}")
|
||||
continue
|
||||
|
||||
|
||||
# Update position after successful line read
|
||||
_position = _f.tell()
|
||||
|
||||
|
||||
except Exception as _e:
|
||||
_err_str = str(_e).lower()
|
||||
if "no such table" in _err_str or "no active connection" in _err_str or "connection closed" in _err_str:
|
||||
logger.error(f"Post-shutdown or fatal DB error in ingester: {_e}")
|
||||
break # Exit worker — DB is gone or uninitialized
|
||||
|
||||
logger.error(f"Error in log ingestion worker: {_e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
@@ -78,7 +83,7 @@ async def _extract_bounty(repo: BaseRepository, log_data: dict[str, Any]) -> Non
|
||||
# 1. Credentials (User/Pass)
|
||||
_user = _fields.get("username")
|
||||
_pass = _fields.get("password")
|
||||
|
||||
|
||||
if _user and _pass:
|
||||
await repo.add_bounty({
|
||||
"decky": log_data.get("decky"),
|
||||
@@ -90,5 +95,5 @@ async def _extract_bounty(repo: BaseRepository, log_data: dict[str, Any]) -> Non
|
||||
"password": _pass
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
# 2. Add more extractors here later (e.g. file hashes, crypto keys)
|
||||
|
||||
@@ -12,7 +12,11 @@ router = APIRouter()
|
||||
@router.post(
|
||||
"/auth/change-password",
|
||||
tags=["Authentication"],
|
||||
responses={401: {"description": "Invalid or expired token / wrong old password"}, 422: {"description": "Validation error"}},
|
||||
responses={
|
||||
400: {"description": "Bad Request (e.g. malformed JSON)"},
|
||||
401: {"description": "Could not validate credentials"},
|
||||
422: {"description": "Validation error"}
|
||||
},
|
||||
)
|
||||
async def change_password(request: ChangePasswordRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]:
|
||||
_user: Optional[dict[str, Any]] = await repo.get_user_by_uuid(current_user)
|
||||
@@ -21,7 +25,7 @@ async def change_password(request: ChangePasswordRequest, current_user: str = De
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Incorrect old password",
|
||||
)
|
||||
|
||||
|
||||
_new_hash: str = get_password_hash(request.new_password)
|
||||
await repo.update_user_password(current_user, _new_hash, must_change_password=False)
|
||||
return {"message": "Password updated successfully"}
|
||||
|
||||
@@ -18,7 +18,11 @@ router = APIRouter()
|
||||
"/auth/login",
|
||||
response_model=Token,
|
||||
tags=["Authentication"],
|
||||
responses={401: {"description": "Incorrect username or password"}, 422: {"description": "Validation error"}},
|
||||
responses={
|
||||
400: {"description": "Bad Request (e.g. malformed JSON)"},
|
||||
401: {"description": "Incorrect username or password"},
|
||||
422: {"description": "Validation error"}
|
||||
},
|
||||
)
|
||||
async def login(request: LoginRequest) -> dict[str, Any]:
|
||||
_user: Optional[dict[str, Any]] = await repo.get_user_by_username(request.username)
|
||||
@@ -35,7 +39,7 @@ async def login(request: LoginRequest) -> dict[str, Any]:
|
||||
data={"uuid": _user["uuid"]}, expires_delta=_access_token_expires
|
||||
)
|
||||
return {
|
||||
"access_token": _access_token,
|
||||
"access_token": _access_token,
|
||||
"token_type": "bearer", # nosec B105
|
||||
"must_change_password": bool(_user.get("must_change_password", False))
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/bounty", response_model=BountyResponse, tags=["Bounty Vault"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
|
||||
async def get_bounties(
|
||||
limit: int = Query(50, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
|
||||
@@ -3,17 +3,21 @@ import os
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, load_state
|
||||
from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, _ROOT
|
||||
from decnet.engine import deploy as _deploy
|
||||
from decnet.ini_loader import load_ini_from_string
|
||||
from decnet.network import detect_interface, detect_subnet, get_host_ip
|
||||
from decnet.web.dependencies import get_current_user
|
||||
from decnet.web.dependencies import get_current_user, repo
|
||||
from decnet.web.db.models import DeployIniRequest
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/deckies/deploy", tags=["Fleet Management"])
|
||||
@router.post(
|
||||
"/deckies/deploy",
|
||||
tags=["Fleet Management"],
|
||||
responses={401: {"description": "Could not validate credentials"}, 400: {"description": "Validation error or INI parsing failed"}, 500: {"description": "Deployment failed"}}
|
||||
)
|
||||
async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]:
|
||||
from decnet.fleet import build_deckies_from_ini
|
||||
|
||||
@@ -22,11 +26,11 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Failed to parse INI: {e}")
|
||||
|
||||
state = load_state()
|
||||
state_dict = await repo.get_state("deployment")
|
||||
ingest_log_file = os.environ.get("DECNET_INGEST_LOG_FILE")
|
||||
|
||||
if state:
|
||||
config, _ = state
|
||||
|
||||
if state_dict:
|
||||
config = DecnetConfig(**state_dict["config"])
|
||||
subnet_cidr = ini.subnet or config.subnet
|
||||
gateway = ini.gateway or config.gateway
|
||||
host_ip = get_host_ip(config.interface)
|
||||
@@ -66,12 +70,20 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(
|
||||
existing_deckies_map = {d.name: d for d in config.deckies}
|
||||
for new_decky in new_decky_configs:
|
||||
existing_deckies_map[new_decky.name] = new_decky
|
||||
|
||||
|
||||
config.deckies = list(existing_deckies_map.values())
|
||||
|
||||
|
||||
# We call deploy(config) which regenerates docker-compose and runs `up -d --remove-orphans`.
|
||||
try:
|
||||
_deploy(config)
|
||||
if os.environ.get("DECNET_CONTRACT_TEST") != "true":
|
||||
_deploy(config)
|
||||
|
||||
# Persist new state to DB
|
||||
new_state_payload = {
|
||||
"config": config.model_dump(),
|
||||
"compose_path": str(_ROOT / "docker-compose.yml") if not state_dict else state_dict["compose_path"]
|
||||
}
|
||||
await repo.set_state("deployment", new_state_payload)
|
||||
except Exception as e:
|
||||
logging.getLogger("decnet.web.api").exception("Deployment failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Deployment failed. Check server logs for details.")
|
||||
|
||||
@@ -8,6 +8,6 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/deckies", tags=["Fleet Management"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
|
||||
async def get_deckies(current_user: str = Depends(get_current_user)) -> list[dict[str, Any]]:
|
||||
return await repo.get_deckies()
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
import os
|
||||
from fastapi import APIRouter, Depends, HTTPException, Path
|
||||
|
||||
from decnet.mutator import mutate_decky
|
||||
from decnet.web.dependencies import get_current_user
|
||||
from decnet.web.dependencies import get_current_user, repo
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/deckies/{decky_name}/mutate", tags=["Fleet Management"])
|
||||
@router.post(
|
||||
"/deckies/{decky_name}/mutate",
|
||||
tags=["Fleet Management"],
|
||||
responses={401: {"description": "Could not validate credentials"}, 404: {"description": "Decky not found"}}
|
||||
)
|
||||
async def api_mutate_decky(
|
||||
decky_name: str = Path(..., pattern=r"^[a-z0-9\-]{1,64}$"),
|
||||
current_user: str = Depends(get_current_user),
|
||||
) -> dict[str, str]:
|
||||
success = mutate_decky(decky_name)
|
||||
if os.environ.get("DECNET_CONTRACT_TEST") == "true":
|
||||
return {"message": f"Successfully mutated {decky_name} (Contract Test Mock)"}
|
||||
|
||||
success = await mutate_decky(decky_name, repo=repo)
|
||||
if success:
|
||||
return {"message": f"Successfully mutated {decky_name}"}
|
||||
raise HTTPException(status_code=404, detail=f"Decky {decky_name} not found or failed to mutate")
|
||||
|
||||
@@ -1,22 +1,33 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from decnet.config import load_state, save_state
|
||||
from decnet.web.dependencies import get_current_user
|
||||
from decnet.config import DecnetConfig
|
||||
from decnet.web.dependencies import get_current_user, repo
|
||||
from decnet.web.db.models import MutateIntervalRequest
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.put("/deckies/{decky_name}/mutate-interval", tags=["Fleet Management"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={
|
||||
400: {"description": "No active deployment found"},
|
||||
401: {"description": "Could not validate credentials"},
|
||||
404: {"description": "Decky not found"},
|
||||
422: {"description": "Validation error"}
|
||||
},
|
||||
)
|
||||
async def api_update_mutate_interval(decky_name: str, req: MutateIntervalRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]:
|
||||
state = load_state()
|
||||
if not state:
|
||||
raise HTTPException(status_code=500, detail="No active deployment")
|
||||
config, compose_path = state
|
||||
state_dict = await repo.get_state("deployment")
|
||||
if not state_dict:
|
||||
raise HTTPException(status_code=400, detail="No active deployment")
|
||||
|
||||
config = DecnetConfig(**state_dict["config"])
|
||||
compose_path = state_dict["compose_path"]
|
||||
|
||||
decky = next((d for d in config.deckies if d.name == decky_name), None)
|
||||
if not decky:
|
||||
raise HTTPException(status_code=404, detail="Decky not found")
|
||||
|
||||
decky.mutate_interval = req.mutate_interval
|
||||
save_state(config, compose_path)
|
||||
|
||||
await repo.set_state("deployment", {"config": config.model_dump(), "compose_path": compose_path})
|
||||
return {"message": "Mutation interval updated"}
|
||||
|
||||
@@ -8,7 +8,7 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/logs/histogram", tags=["Logs"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
|
||||
async def get_logs_histogram(
|
||||
search: Optional[str] = None,
|
||||
start_time: Optional[str] = None,
|
||||
|
||||
@@ -7,10 +7,11 @@ from decnet.web.db.models import LogsResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_DATETIME_RE = r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}$"
|
||||
_DATETIME_RE = r"^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2})?$"
|
||||
|
||||
|
||||
@router.get("/logs", response_model=LogsResponse, tags=["Logs"])
|
||||
@router.get("/logs", response_model=LogsResponse, tags=["Logs"],
|
||||
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}})
|
||||
async def get_logs(
|
||||
limit: int = Query(50, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
|
||||
@@ -9,6 +9,6 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/stats", response_model=StatsResponse, tags=["Observability"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
|
||||
async def get_stats(current_user: str = Depends(get_current_user)) -> dict[str, Any]:
|
||||
return await repo.get_stats_summary()
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import AsyncGenerator, Optional
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from decnet.env import DECNET_DEVELOPER
|
||||
from decnet.web.dependencies import get_stream_user, repo
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -14,20 +15,30 @@ router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/stream", tags=["Observability"],
|
||||
responses={401: {"description": "Not authenticated"}, 422: {"description": "Validation error"}},)
|
||||
responses={
|
||||
200: {
|
||||
"content": {"text/event-stream": {}},
|
||||
"description": "Real-time Server-Sent Events (SSE) stream"
|
||||
},
|
||||
401: {"description": "Could not validate credentials"},
|
||||
422: {"description": "Validation error"}
|
||||
},
|
||||
)
|
||||
async def stream_events(
|
||||
request: Request,
|
||||
last_event_id: int = Query(0, alias="lastEventId"),
|
||||
request: Request,
|
||||
last_event_id: int = Query(0, alias="lastEventId"),
|
||||
search: Optional[str] = None,
|
||||
start_time: Optional[str] = None,
|
||||
end_time: Optional[str] = None,
|
||||
max_output: Optional[int] = Query(None, alias="maxOutput"),
|
||||
current_user: str = Depends(get_stream_user)
|
||||
) -> StreamingResponse:
|
||||
|
||||
|
||||
async def event_generator() -> AsyncGenerator[str, None]:
|
||||
last_id = last_event_id
|
||||
stats_interval_sec = 10
|
||||
loops_since_stats = 0
|
||||
emitted_chunks = 0
|
||||
try:
|
||||
if last_id == 0:
|
||||
last_id = await repo.get_max_log_id()
|
||||
@@ -42,6 +53,12 @@ async def stream_events(
|
||||
yield f"event: message\ndata: {json.dumps({'type': 'histogram', 'data': histogram})}\n\n"
|
||||
|
||||
while True:
|
||||
if DECNET_DEVELOPER and max_output is not None:
|
||||
emitted_chunks += 1
|
||||
if emitted_chunks > max_output:
|
||||
log.debug("Developer mode: max_output reached (%d), closing stream", max_output)
|
||||
break
|
||||
|
||||
if await request.is_disconnected():
|
||||
break
|
||||
|
||||
@@ -65,6 +82,7 @@ async def stream_events(
|
||||
loops_since_stats = 0
|
||||
|
||||
loops_since_stats += 1
|
||||
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user