From a93c95944419c1ee7da5763879a5c5901553b01e Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:32:29 +0200 Subject: [PATCH 01/18] chore(backend): bootstrap Python 3.12+ project skeleton (B0.1) - pyproject.toml with ruff + mypy strict + pytest + coverage >=70% - Makefile with Docker/Podman auto-detect - Multi-stage Dockerfile (python:3.12-slim-bookworm, non-root user) - docker-compose.yml for Postgres dev DB - alembic.ini wired to src/mimic/db/migrations - scripts/postgres-init/00-roles.sql seeds the audit writer role - .env.example documents every MIMIC_* var (no secrets committed) --- backend/.env.example | 23 ++++ backend/.gitkeep | 0 backend/Dockerfile | 60 +++++++++ backend/Makefile | 80 +++++++++++ backend/README.md | 56 ++++++++ backend/alembic.ini | 39 ++++++ backend/docker-compose.yml | 23 ++++ backend/pyproject.toml | 147 +++++++++++++++++++++ backend/scripts/postgres-init/00-roles.sql | 20 +++ 9 files changed, 448 insertions(+) create mode 100644 backend/.env.example delete mode 100644 backend/.gitkeep create mode 100644 backend/Dockerfile create mode 100644 backend/Makefile create mode 100644 backend/README.md create mode 100644 backend/alembic.ini create mode 100644 backend/docker-compose.yml create mode 100644 backend/pyproject.toml create mode 100644 backend/scripts/postgres-init/00-roles.sql diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..c169cb2 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,23 @@ +# Mimic backend — example env. Copy to .env (gitignored) and adapt. + +MIMIC_ENV=development +MIMIC_SECRET_KEY=replace-me-with-secrets.token_urlsafe-32 +MIMIC_FERNET_KEY= + +# Database +POSTGRES_DB=mimic +POSTGRES_USER=mimic_app +POSTGRES_PASSWORD=mimic_dev_password +MIMIC_DATABASE_URL=postgresql+psycopg://mimic_app:mimic_dev_password@localhost:5432/mimic +MIMIC_DATABASE_AUDIT_URL=postgresql+psycopg://mimic_audit_writer:CHANGE_ME@localhost:5432/mimic + +# Session / cookies +MIMIC_SESSION_COOKIE_SECURE=false +MIMIC_SESSION_COOKIE_SAMESITE=Lax + +# CORS (frontend dev) +MIMIC_CORS_ORIGINS=http://localhost:5173 + +# Logging +MIMIC_LOG_LEVEL=DEBUG +MIMIC_LOG_JSON=false diff --git a/backend/.gitkeep b/backend/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..6ba967e --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,60 @@ +# syntax=docker/dockerfile:1.7 + +# --- Stage 1: build -------------------------------------------------------- +FROM python:3.12-slim-bookworm AS build + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 + +# WeasyPrint native deps + libpq + build tools. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libpq-dev \ + libpango-1.0-0 \ + libpangoft2-1.0-0 \ + libcairo2 \ + libgdk-pixbuf-2.0-0 \ + libffi-dev \ + shared-mime-info \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build +COPY pyproject.toml README.md ./ +COPY src ./src + +RUN pip install --upgrade pip wheel build \ + && pip wheel --wheel-dir /wheels --no-deps . + +RUN pip install --prefix=/install --no-warn-script-location . + +# --- Stage 2: runtime ------------------------------------------------------ +FROM python:3.12-slim-bookworm AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + FLASK_APP=mimic.app:create_app \ + MIMIC_ENV=production + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq5 \ + libpango-1.0-0 \ + libpangoft2-1.0-0 \ + libcairo2 \ + libgdk-pixbuf-2.0-0 \ + shared-mime-info \ + tini \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 1001 mimic \ + && useradd --system --uid 1001 --gid 1001 --home-dir /app --shell /usr/sbin/nologin mimic + +WORKDIR /app +COPY --from=build /install /usr/local +COPY --chown=mimic:mimic src ./src + +USER mimic +EXPOSE 5000 + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["gunicorn", "--worker-class", "gevent", "--workers", "1", "--bind", "0.0.0.0:5000", "mimic.app:create_app()"] diff --git a/backend/Makefile b/backend/Makefile new file mode 100644 index 0000000..df7d746 --- /dev/null +++ b/backend/Makefile @@ -0,0 +1,80 @@ +# --- Mimic backend Makefile ------------------------------------------------ +# Detects Docker / Podman automatically (NF-platform). + +PY ?= python3.12 +VENV ?= .venv +PIP := $(VENV)/bin/pip +PYTHON := $(VENV)/bin/python +PYTEST := $(VENV)/bin/pytest +RUFF := $(VENV)/bin/ruff +MYPY := $(VENV)/bin/mypy +ALEMBIC := $(VENV)/bin/alembic +FLASK := $(VENV)/bin/flask + +# Container runtime auto-detect: Docker (preferred) or Podman rootless. +CONTAINER ?= $(shell command -v docker 2>/dev/null || command -v podman 2>/dev/null) +COMPOSE ?= $(shell command -v docker-compose 2>/dev/null || echo "$(CONTAINER) compose") + +export FLASK_APP=mimic.app:create_app + +.PHONY: help install lint fmt typecheck test test-int test-cov run \ + db-up db-down db-migrate db-revision db-seed db-dump db-restore \ + build clean + +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ + awk 'BEGIN {FS=":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' + +install: ## Create venv and install dev dependencies + $(PY) -m venv $(VENV) + $(PIP) install --upgrade pip wheel + $(PIP) install -e ".[dev]" + +lint: ## Ruff lint + $(RUFF) check src tests + +fmt: ## Ruff format + $(RUFF) format src tests + $(RUFF) check --fix src tests + +typecheck: ## Mypy strict + $(MYPY) src + +test: ## Unit tests (SQLite) + $(PYTEST) tests/unit -v + +test-int: ## Integration tests (testcontainers Postgres) + $(PYTEST) tests/integration -v -m integration + +test-cov: ## Unit + integration with coverage report + $(PYTEST) --cov=mimic --cov-report=term-missing --cov-report=html + +run: ## Run Flask dev server + $(FLASK) run --host 0.0.0.0 --port 5000 --debug + +db-up: ## Start Postgres dev container + $(COMPOSE) up -d postgres + +db-down: ## Stop Postgres dev container + $(COMPOSE) down + +db-migrate: ## Apply migrations + $(ALEMBIC) upgrade head + +db-revision: ## Generate migration (msg=...) + $(ALEMBIC) revision --autogenerate -m "$(msg)" + +db-seed: ## Seed local dev data (TBD sprint 1) + @echo "TBD sprint 1" + +db-dump: ## Manual DB dump (NF-state, R-O1) + $(VENV)/bin/mimic-cli db dump --out backups/mimic-$$(date +%Y%m%d-%H%M%S).sql + +db-restore: ## Restore from dump (file=...) + $(VENV)/bin/mimic-cli db restore --file $(file) + +build: ## Build wheel + $(PYTHON) -m build + +clean: + rm -rf $(VENV) .pytest_cache .mypy_cache .ruff_cache htmlcov dist build *.egg-info diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..62adec3 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,56 @@ +# Mimic — backend + +Sprint 0 skeleton. Python 3.12+ / Flask / SQLAlchemy 2 / Alembic / Pydantic 2. + +## Layout + +``` +backend/ +├── src/mimic/ +│ ├── app.py # Flask app factory + SocketIO init +│ ├── config.py # Pydantic Settings +│ ├── extensions.py # db, migrate, socketio, login_manager +│ ├── db/ +│ │ ├── models/ # SQLAlchemy 2 typed models +│ │ ├── repositories/ # data access per aggregate +│ │ └── migrations/ # Alembic +│ ├── schemas/ # Pydantic 2 DTOs +│ ├── api/ # Flask blueprints (REST) +│ ├── ws/ # Flask-SocketIO namespaces +│ ├── connectors/ # C2Connector ABC + payload mapping +│ ├── orchestrator/ # run state machine (stub in sprint 0) +│ ├── templating/ # Jinja2 sandbox + regex_extract +│ ├── audit/ # append-only writer + rotation +│ ├── reporting/ # WeasyPrint builder (stub in sprint 0) +│ ├── rbac/ # group-based permission matrix (F11) +│ ├── importers/ # ATR + C2 journal (stub in sprint 0) +│ └── cli/ # mimic-cli (click) +└── tests/ + ├── unit/ # SQLite, pure logic + └── integration/ # testcontainers Postgres +``` + +## Local dev + +```bash +make install # uv venv + pip install -e .[dev] +make db-up # docker compose up -d postgres +make db-migrate # alembic upgrade head +make run # flask run (debug) +make test # pytest unit +make test-int # pytest integration (testcontainers) +make lint # ruff + mypy strict +``` + +## What sprint 0 ships + +- Full §8 data model + Alembic initial migration (Postgres-specific constraints: audit_log write-only role, soc_session hash, c2_credential Fernet column). +- `C2Connector` ABC + dataclasses + `payload_type` enum + factory. **No real Mythic/Home implementation** (blocked on PR1/PR2). +- Jinja2 SandboxedEnvironment + `regex_extract` filter (re2). +- Local auth (bcrypt + Flask session) + group-based RBAC matching the F11 permission matrix. +- Flat CRUD on engagements / hosts / TTPs / scenarios. +- pytest baseline + testcontainers Postgres scaffolding. + +## Out of sprint 0 + +Orchestrator, WebSocket cockpit, real connectors, report generation, audit rotation. diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..350c202 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,39 @@ +[alembic] +script_location = src/mimic/db/migrations +prepend_sys_path = src +version_path_separator = os +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml new file mode 100644 index 0000000..4aacd6d --- /dev/null +++ b/backend/docker-compose.yml @@ -0,0 +1,23 @@ +services: + postgres: + image: postgres:16-alpine + container_name: mimic-postgres + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-mimic} + POSTGRES_USER: ${POSTGRES_USER:-mimic_app} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mimic_dev_password} + ports: + - "127.0.0.1:5432:5432" + volumes: + - mimic_pgdata:/var/lib/postgresql/data + - ./scripts/postgres-init:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mimic_app} -d ${POSTGRES_DB:-mimic}"] + interval: 5s + timeout: 3s + retries: 10 + +volumes: + mimic_pgdata: + name: mimic_pgdata diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..0b38510 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,147 @@ +[build-system] +requires = ["hatchling>=1.24"] +build-backend = "hatchling.build" + +[project] +name = "mimic" +version = "0.1.0a0" +description = "Mimic — internal BAS platform (sprint 0 skeleton)" +readme = "README.md" +requires-python = ">=3.12" +license = { text = "Proprietary" } +authors = [{ name = "RT" }] + +dependencies = [ + "flask>=3.0,<4.0", + "flask-socketio>=5.3,<6.0", + "flask-login>=0.6.3,<1.0", + "flask-migrate>=4.0,<5.0", + "sqlalchemy>=2.0,<3.0", + "alembic>=1.13,<2.0", + "psycopg[binary]>=3.1,<4.0", + "pydantic>=2.6,<3.0", + "pydantic-settings>=2.2,<3.0", + "python-json-logger>=2.0,<3.0", + "structlog>=24.1,<25.0", + "bcrypt>=4.1,<5.0", + "cryptography>=42.0,<43.0", + "jinja2>=3.1,<4.0", + "google-re2>=1.1,<2.0", + "click>=8.1,<9.0", + "gevent>=24.2,<25.0", + "gevent-websocket>=0.10,<1.0", + "httpx>=0.27,<1.0", + "weasyprint>=61.0,<62.0", + "authlib>=1.3,<2.0", + "pyyaml>=6.0,<7.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0,<9.0", + "pytest-cov>=5.0,<6.0", + "pytest-flask>=1.3,<2.0", + "pytest-mock>=3.12,<4.0", + "testcontainers[postgres]>=4.4,<5.0", + "ruff>=0.4,<1.0", + "mypy>=1.10,<2.0", + "types-pyyaml>=6.0,<7.0", + "freezegun>=1.5,<2.0", +] + +[project.scripts] +mimic-cli = "mimic.cli:cli" + +[tool.hatch.build.targets.wheel] +packages = ["src/mimic"] + +[tool.hatch.build.targets.sdist] +include = ["src/mimic", "README.md", "pyproject.toml"] + +# -- Ruff ------------------------------------------------------------------- + +[tool.ruff] +line-length = 100 +target-version = "py312" +src = ["src", "tests"] + +[tool.ruff.lint] +select = [ + "E", "F", "W", # pycodestyle / pyflakes + "I", # isort + "B", # bugbear + "UP", # pyupgrade + "N", # pep8-naming + "S", # flake8-bandit (security) + "C4", # comprehensions + "DTZ", # datetime tz + "PIE", + "PT", # pytest + "RET", + "SIM", + "TID", + "PL", + "RUF", +] +ignore = [ + "PLR0913", # too many args (Flask handlers + DI) + "S101", # assert in tests +] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S101", "S105", "S106", "PLR2004"] +"src/mimic/db/migrations/**" = ["E501", "N999"] + +[tool.ruff.lint.isort] +known-first-party = ["mimic"] + +# -- Mypy ------------------------------------------------------------------- + +[tool.mypy] +python_version = "3.12" +strict = true +warn_unreachable = true +warn_unused_ignores = true +show_error_codes = true +plugins = ["pydantic.mypy"] +exclude = ["src/mimic/db/migrations/versions/"] + +[[tool.mypy.overrides]] +module = [ + "weasyprint.*", + "google.re2.*", + "re2", + "flask_socketio.*", + "flask_migrate.*", + "gevent.*", + "testcontainers.*", + "authlib.*", +] +ignore_missing_imports = true + +# -- Pytest ----------------------------------------------------------------- + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra --strict-markers --strict-config" +markers = [ + "integration: requires testcontainers Postgres", + "slow: long-running tests", +] +filterwarnings = ["error"] + +[tool.coverage.run] +branch = true +source = ["src/mimic"] +omit = ["src/mimic/db/migrations/*"] + +[tool.coverage.report] +fail_under = 70 +show_missing = true +skip_covered = false +exclude_lines = [ + "pragma: no cover", + "raise NotImplementedError", + "if TYPE_CHECKING:", + "\\.\\.\\.", +] diff --git a/backend/scripts/postgres-init/00-roles.sql b/backend/scripts/postgres-init/00-roles.sql new file mode 100644 index 0000000..e81da46 --- /dev/null +++ b/backend/scripts/postgres-init/00-roles.sql @@ -0,0 +1,20 @@ +-- Roles used by the application. +-- NF-AUDIT: audit_log must be append-only at the SQL level. The application +-- writes via mimic_audit_writer (INSERT only). The standard mimic_app role +-- has SELECT on audit_log but no UPDATE/DELETE. +-- +-- This file runs once at container init. Production deployment uses Ansible +-- to apply the same grants idempotently. + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mimic_audit_writer') THEN + CREATE ROLE mimic_audit_writer LOGIN PASSWORD 'CHANGE_ME'; + END IF; +END +$$; + +-- The mimic_app user is created by the official image entrypoint +-- via $POSTGRES_USER. We only need to make sure the audit writer exists. +-- Per-table grants are applied by the application's bootstrap step after +-- migrations land (so the audit_log table actually exists). From 22d37fb240ae6e5ff6c2e029dabc5a7848396007 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:32:45 +0200 Subject: [PATCH 02/18] =?UTF-8?q?feat(backend):=20add=20=C2=A78=20data=20m?= =?UTF-8?q?odel=20+=20Alembic=20baseline=20(B0.2,=20B0.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SQLAlchemy 2 typed mapped classes for every spec §8 aggregate: engagement, c2_credential, host, user, group, group_permission, user_group, engagement_member, ttp, ttp_version, scenario, scenario_step, run, run_step, run_step_cleanup, detection, evidence, report, soc_session, audit_log. - Shared mixins: UuidPkMixin (PG_UUID(as_uuid=True)) + TimestampsMixin. - StrEnum types covering every spec enum (C2Type, PayloadType, UserType, EngagementStatus, HostStatus, TtpSource, RunStatus, RunStepStatus, CleanupStatus, DetectionLevel, DetectionSource, EvidenceStatus). - Alembic baseline migration 202605210001_initial_schema: creates every table, enum, index, and idempotent grants for the audit_log write-only Postgres role (mimic_audit_writer). - Audit log carries prev_hash / row_hash from v1 (D-009). - ttp_version table coexists with run.snapshot_json (D-008, overrides H32). --- backend/src/mimic/__init__.py | 3 + backend/src/mimic/config.py | 66 ++ backend/src/mimic/db/__init__.py | 5 + backend/src/mimic/db/base.py | 59 ++ backend/src/mimic/db/migrations/__init__.py | 0 backend/src/mimic/db/migrations/env.py | 58 ++ .../src/mimic/db/migrations/script.py.mako | 27 + .../versions/202605210001_initial_schema.py | 610 ++++++++++++++++++ backend/src/mimic/db/models/__init__.py | 37 ++ backend/src/mimic/db/models/audit.py | 45 ++ backend/src/mimic/db/models/detection.py | 82 +++ backend/src/mimic/db/models/engagement.py | 106 +++ backend/src/mimic/db/models/host.py | 41 ++ backend/src/mimic/db/models/permission.py | 87 +++ backend/src/mimic/db/models/report.py | 48 ++ backend/src/mimic/db/models/run.py | 118 ++++ backend/src/mimic/db/models/scenario.py | 86 +++ backend/src/mimic/db/models/soc_session.py | 48 ++ backend/src/mimic/db/models/ttp.py | 105 +++ backend/src/mimic/db/models/user.py | 50 ++ backend/src/mimic/db/types.py | 110 ++++ backend/src/mimic/extensions.py | 15 + backend/src/mimic/logging.py | 27 + 23 files changed, 1833 insertions(+) create mode 100644 backend/src/mimic/__init__.py create mode 100644 backend/src/mimic/config.py create mode 100644 backend/src/mimic/db/__init__.py create mode 100644 backend/src/mimic/db/base.py create mode 100644 backend/src/mimic/db/migrations/__init__.py create mode 100644 backend/src/mimic/db/migrations/env.py create mode 100644 backend/src/mimic/db/migrations/script.py.mako create mode 100644 backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py create mode 100644 backend/src/mimic/db/models/__init__.py create mode 100644 backend/src/mimic/db/models/audit.py create mode 100644 backend/src/mimic/db/models/detection.py create mode 100644 backend/src/mimic/db/models/engagement.py create mode 100644 backend/src/mimic/db/models/host.py create mode 100644 backend/src/mimic/db/models/permission.py create mode 100644 backend/src/mimic/db/models/report.py create mode 100644 backend/src/mimic/db/models/run.py create mode 100644 backend/src/mimic/db/models/scenario.py create mode 100644 backend/src/mimic/db/models/soc_session.py create mode 100644 backend/src/mimic/db/models/ttp.py create mode 100644 backend/src/mimic/db/models/user.py create mode 100644 backend/src/mimic/db/types.py create mode 100644 backend/src/mimic/extensions.py create mode 100644 backend/src/mimic/logging.py diff --git a/backend/src/mimic/__init__.py b/backend/src/mimic/__init__.py new file mode 100644 index 0000000..5c5c666 --- /dev/null +++ b/backend/src/mimic/__init__.py @@ -0,0 +1,3 @@ +"""Mimic backend package.""" + +__version__ = "0.1.0a0" diff --git a/backend/src/mimic/config.py b/backend/src/mimic/config.py new file mode 100644 index 0000000..6f47f07 --- /dev/null +++ b/backend/src/mimic/config.py @@ -0,0 +1,66 @@ +"""Runtime configuration loaded from environment (Pydantic Settings).""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Literal + +from pydantic import Field, SecretStr, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +Environment = Literal["development", "testing", "production"] + + +class Settings(BaseSettings): + """Application settings. + + All values are env-driven (NF-network: no hardcoded secrets). The `MIMIC_` + prefix isolates Mimic vars from ambient shell vars. + """ + + model_config = SettingsConfigDict( + env_prefix="MIMIC_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + case_sensitive=False, + ) + + env: Environment = "development" + secret_key: SecretStr = Field(default=SecretStr("change-me")) + fernet_key: SecretStr = Field(default=SecretStr("")) + + database_url: str = "postgresql+psycopg://mimic_app:mimic_dev_password@localhost:5432/mimic" + database_audit_url: str | None = None + + session_cookie_secure: bool = True + session_cookie_samesite: Literal["Lax", "Strict", "None"] = "Lax" + session_lifetime_minutes: int = 60 * 8 + + cors_origins: list[str] = Field(default_factory=list) + + log_level: str = "INFO" + log_json: bool = True + + template_render_timeout_ms: int = 250 + output_blob_max_bytes: int = 10 * 1024 * 1024 + + @field_validator("cors_origins", mode="before") + @classmethod + def _split_cors(cls, value: object) -> object: + if isinstance(value, str): + return [origin.strip() for origin in value.split(",") if origin.strip()] + return value + + @property + def is_production(self) -> bool: + return self.env == "production" + + @property + def is_testing(self) -> bool: + return self.env == "testing" + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() diff --git a/backend/src/mimic/db/__init__.py b/backend/src/mimic/db/__init__.py new file mode 100644 index 0000000..527b98d --- /dev/null +++ b/backend/src/mimic/db/__init__.py @@ -0,0 +1,5 @@ +"""Database layer: SQLAlchemy 2 declarative base, models, repositories.""" + +from mimic.db.base import Base + +__all__ = ["Base"] diff --git a/backend/src/mimic/db/base.py b/backend/src/mimic/db/base.py new file mode 100644 index 0000000..d373aaf --- /dev/null +++ b/backend/src/mimic/db/base.py @@ -0,0 +1,59 @@ +"""Declarative base + shared mixins for all ORM models.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from sqlalchemy import DateTime, MetaData, func +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + """Project-wide declarative base. + + UUID columns are declared explicitly on each model via `PG_UUID(as_uuid=True)` + rather than through a `type_annotation_map` — Flask-SQLAlchemy injects its + own registry which is incompatible with per-base annotation maps. + """ + + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + +class UuidPkMixin: + """Mixin: UUID v4 primary key generated client-side.""" + + id: Mapped[uuid.UUID] = mapped_column( + PG_UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + + +def _utcnow() -> datetime: + return datetime.now(tz=UTC) + + +class TimestampsMixin: + """Mixin: `created_at` / `updated_at` columns, UTC timezone-aware.""" + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=_utcnow, + nullable=False, + ) diff --git a/backend/src/mimic/db/migrations/__init__.py b/backend/src/mimic/db/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/mimic/db/migrations/env.py b/backend/src/mimic/db/migrations/env.py new file mode 100644 index 0000000..a1b05b6 --- /dev/null +++ b/backend/src/mimic/db/migrations/env.py @@ -0,0 +1,58 @@ +"""Alembic environment. + +We import the SQLAlchemy `Base.metadata` directly so migrations are decoupled +from the Flask app object (Alembic can run in CI without spinning a request +context). +""" + +from __future__ import annotations + +import logging +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from mimic.config import get_settings +from mimic.db.base import Base +from mimic.db.models import * # noqa: F403 (ensures all tables register) + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) +log = logging.getLogger("alembic.env") + +target_metadata = Base.metadata + +settings = get_settings() +config.set_main_option("sqlalchemy.url", settings.database_url) + + +def run_migrations_offline() -> None: + context.configure( + url=config.get_main_option("sqlalchemy.url"), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/src/mimic/db/migrations/script.py.mako b/backend/src/mimic/db/migrations/script.py.mako new file mode 100644 index 0000000..04be66b --- /dev/null +++ b/backend/src/mimic/db/migrations/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from typing import Sequence + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: str | Sequence[str] | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py new file mode 100644 index 0000000..aa3d732 --- /dev/null +++ b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py @@ -0,0 +1,610 @@ +"""initial schema (sprint 0, §8 spec) + +Creates every aggregate listed in spec §8: engagement, user/group RBAC, +host, ttp/ttp_version, scenario/scenario_step, run/run_step/run_step_cleanup, +detection, evidence, report, soc_session, c2_credential, audit_log. + +Postgres-only objects: +- ENUM types created via SQLAlchemy `Enum(..., create_type=True)`. +- `audit_log` SQL-level append-only via `mimic_audit_writer` role grants + (idempotent, no-op if role missing — deployment playbook is authoritative). + +Revision ID: 202605210001 +Revises: +Create Date: 2026-05-21 +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB, UUID + +revision: str = "202605210001" +down_revision: str | None = None +branch_labels: str | None = None +depends_on: str | None = None + + +# --------------------------------------------------------------------------- +# Enums (Postgres CREATE TYPE — done up-front so every table reuses the type) +# --------------------------------------------------------------------------- + +USER_TYPE = sa.Enum("rt_operator", "rt_lead", "soc_analyst", name="user_type") +ENGAGEMENT_STATUS = sa.Enum( + "draft", "active", "closed", "archived", name="engagement_status" +) +C2_TYPE = sa.Enum("mythic", "home", name="c2_type") +HOST_STATUS = sa.Enum("unknown", "alive", "dead", name="host_status") +PAYLOAD_TYPE = sa.Enum( + "cmd", + "powershell", + "bof", + "dotnet_assembly", + "dotnet_exe", + "pe_exe", + "pe_dll", + "shellcode", + "python", + "vbs", + "wmi_query", + "registry", + "script_file", + name="payload_type", +) +TTP_SOURCE = sa.Enum("custom", "import_atr", "import_mission", name="ttp_source") +RUN_STATUS = sa.Enum( + "queued", "running", "paused", "completed", "failed", "aborted", name="run_status" +) +RUN_STEP_STATUS = sa.Enum( + "queued", + "running", + "completed", + "failed", + "skipped", + "cleanup_failed", + name="run_step_status", +) +CLEANUP_STATUS = sa.Enum( + "pending", "success", "failed", "partial", name="cleanup_status" +) +DETECTION_LEVEL = sa.Enum( + "detected", "partial", "not_detected", name="detection_level" +) +DETECTION_SOURCE = sa.Enum( + "ndr", "edr", "siem", "manual", "other", name="detection_source" +) +EVIDENCE_STATUS = sa.Enum( + "success", "failure", "partial", name="evidence_status" +) + + +def upgrade() -> None: + bind = op.get_bind() + for enum_t in ( + USER_TYPE, + ENGAGEMENT_STATUS, + C2_TYPE, + HOST_STATUS, + PAYLOAD_TYPE, + TTP_SOURCE, + RUN_STATUS, + RUN_STEP_STATUS, + CLEANUP_STATUS, + DETECTION_LEVEL, + DETECTION_SOURCE, + EVIDENCE_STATUS, + ): + enum_t.create(bind, checkfirst=True) + + # ------------------------------------------------------------------ user + op.create_table( + "user", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("type", USER_TYPE, nullable=False), + sa.Column("email", sa.String(255), nullable=False), + sa.Column("display_name", sa.String(120)), + sa.Column("keycloak_sub", sa.String(255), unique=True), + sa.Column("local_password_hash", sa.String(255)), + sa.Column("disabled_at", sa.DateTime(timezone=True)), + sa.Column("last_login_at", sa.DateTime(timezone=True)), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.UniqueConstraint("email", name="uq_user_email"), + ) + + # ------------------------------------------------------------ permission + op.create_table( + "permission", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("code", sa.String(80), nullable=False), + sa.Column("description", sa.String(255)), + sa.UniqueConstraint("code", name="uq_permission_code"), + ) + + op.create_table( + "group", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(80), nullable=False), + sa.Column("description", sa.String(255)), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.UniqueConstraint("name", name="uq_group_name"), + ) + + op.create_table( + "group_permission", + sa.Column( + "group_id", + UUID(as_uuid=True), + sa.ForeignKey("group.id", ondelete="CASCADE", name="fk_group_permission_group_id_group"), + nullable=False, + ), + sa.Column( + "permission_id", + UUID(as_uuid=True), + sa.ForeignKey("permission.id", ondelete="CASCADE", name="fk_group_permission_permission_id_permission"), + nullable=False, + ), + sa.PrimaryKeyConstraint("group_id", "permission_id", name="pk_group_permission"), + ) + + # ------------------------------------------------------------ engagement + op.create_table( + "engagement", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("client_name", sa.String(255), nullable=False), + sa.Column("description", sa.String(1024)), + sa.Column("status", ENGAGEMENT_STATUS, nullable=False, server_default="draft"), + sa.Column("start_date", sa.Date), + sa.Column("end_date", sa.Date), + sa.Column("c2_type", C2_TYPE, nullable=False, server_default="mythic"), + sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "engagement_member", + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "user_id", + UUID(as_uuid=True), + sa.ForeignKey("user.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column("role", sa.String(40), nullable=False), + sa.Column("added_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "user_group", + sa.Column( + "user_id", + UUID(as_uuid=True), + sa.ForeignKey("user.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "group_id", + UUID(as_uuid=True), + sa.ForeignKey("group.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=True, + ), + sa.PrimaryKeyConstraint("user_id", "group_id", "engagement_id", name="pk_user_group"), + ) + + op.create_table( + "c2_credential", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("c2_type", C2_TYPE, nullable=False), + sa.Column("config_fernet", sa.LargeBinary, nullable=False), + sa.Column("version", sa.Integer, nullable=False, server_default="1"), + sa.Column("retired_at", sa.DateTime(timezone=True)), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + op.create_index( + "ix_c2_credential_engagement_active", + "c2_credential", + ["engagement_id", "c2_type", "version"], + unique=False, + ) + + # ------------------------------------------------------------------ host + op.create_table( + "host", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("hostname", sa.String(255), nullable=False), + sa.Column("ip", sa.String(64)), + sa.Column("os", sa.String(128)), + sa.Column("c2_session_id", sa.String(128)), + sa.Column("c2_type", C2_TYPE, nullable=False), + sa.Column("status", HOST_STATUS, nullable=False, server_default="unknown"), + sa.Column("last_seen", sa.DateTime(timezone=True)), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + op.create_index("ix_host_engagement_id", "host", ["engagement_id"]) + + # ------------------------------------------------------------------ ttp + op.create_table( + "ttp", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text), + sa.Column("mitre_technique", sa.String(16), nullable=False), + sa.Column("mitre_subtechnique", sa.String(16)), + sa.Column("payload_type", PAYLOAD_TYPE, nullable=False), + sa.Column("payload_template", sa.Text, nullable=False, server_default=""), + sa.Column("params_schema_json", JSONB), + sa.Column("opsec_notes", sa.Text), + sa.Column("cleanup_command", sa.Text), + sa.Column("is_stealth_variant", sa.Boolean, nullable=False, server_default=sa.false()), + sa.Column("source", TTP_SOURCE, nullable=False, server_default="custom"), + sa.Column("tags", JSONB, nullable=False, server_default="[]"), + sa.Column("current_version", sa.Integer, nullable=False, server_default="1"), + sa.Column("is_published", sa.Boolean, nullable=False, server_default=sa.false()), + sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "ttp_version", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "ttp_id", + UUID(as_uuid=True), + sa.ForeignKey("ttp.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("version", sa.Integer, nullable=False), + sa.Column("snapshot_json", JSONB, nullable=False), + sa.Column("content_sha256", sa.String(64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.UniqueConstraint("ttp_id", "version", name="uq_ttp_version_ttp_id_version"), + ) + + # -------------------------------------------------------------- scenario + op.create_table( + "scenario", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text), + sa.Column("version", sa.Integer, nullable=False, server_default="1"), + sa.Column("c2_type", C2_TYPE, nullable=False), + sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "scenario_step", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "scenario_id", + UUID(as_uuid=True), + sa.ForeignKey("scenario.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("order_idx", sa.Integer, nullable=False), + sa.Column( + "ttp_id", + UUID(as_uuid=True), + sa.ForeignKey("ttp.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column( + "host_id", + UUID(as_uuid=True), + sa.ForeignKey("host.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("params_override_json", JSONB, nullable=False, server_default="{}"), + sa.Column("delay_after_ms", sa.Integer, nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.UniqueConstraint("scenario_id", "order_idx", name="uq_scenario_step_order_idx"), + ) + + # ------------------------------------------------------------------- run + op.create_table( + "run", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "scenario_id", + UUID(as_uuid=True), + sa.ForeignKey("scenario.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("status", RUN_STATUS, nullable=False, server_default="queued"), + sa.Column("started_at", sa.DateTime(timezone=True)), + sa.Column("ended_at", sa.DateTime(timezone=True)), + sa.Column("started_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("snapshot_json", JSONB, nullable=False, server_default="{}"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "run_step", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "run_id", + UUID(as_uuid=True), + sa.ForeignKey("run.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "scenario_step_id", + UUID(as_uuid=True), + sa.ForeignKey("scenario_step.id", ondelete="SET NULL"), + ), + sa.Column("order_idx", sa.Integer, nullable=False), + sa.Column("status", RUN_STEP_STATUS, nullable=False, server_default="queued"), + sa.Column("started_at", sa.DateTime(timezone=True)), + sa.Column("ended_at", sa.DateTime(timezone=True)), + sa.Column("c2_task_id", sa.String(128)), + sa.Column("output_text", sa.Text), + sa.Column("output_blob_ref", sa.String(512)), + sa.Column("exit_code", sa.Integer), + sa.Column("resolved_payload_text", sa.Text), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "run_step_cleanup", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "run_step_id", + UUID(as_uuid=True), + sa.ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ), + sa.Column("status", CLEANUP_STATUS, nullable=False, server_default="pending"), + sa.Column("started_at", sa.DateTime(timezone=True)), + sa.Column("ended_at", sa.DateTime(timezone=True)), + sa.Column("resolved_command_text", sa.Text), + sa.Column("output", sa.Text), + sa.Column("executed_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + # ----------------------------------------------------- detection / evidence + op.create_table( + "detection", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "run_step_id", + UUID(as_uuid=True), + sa.ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "soc_user_id", + UUID(as_uuid=True), + sa.ForeignKey("user.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("level", DETECTION_LEVEL, nullable=False), + sa.Column("source", DETECTION_SOURCE, nullable=False), + sa.Column("latency_ms", sa.Integer), + sa.Column("comment", sa.Text), + sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + op.create_table( + "evidence", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "run_step_id", + UUID(as_uuid=True), + sa.ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "rt_user_id", + UUID(as_uuid=True), + sa.ForeignKey("user.id", ondelete="RESTRICT"), + nullable=False, + ), + sa.Column("status", EVIDENCE_STATUS, nullable=False), + sa.Column("artifacts_text", sa.Text), + sa.Column("artifact_files_json", JSONB, nullable=False, server_default="[]"), + sa.Column("comment", sa.Text), + sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + # ----------------------------------------------------------------- report + op.create_table( + "report", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("version", sa.Integer, nullable=False, server_default="1"), + sa.Column("content_json", JSONB, nullable=False, server_default="{}"), + sa.Column("content_sha256", sa.String(64), nullable=False), + sa.Column("pdf_path", sa.String(512)), + sa.Column("md_path", sa.String(512)), + sa.Column("generated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("generated_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + # ------------------------------------------------------------- soc_session + op.create_table( + "soc_session", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "user_id", + UUID(as_uuid=True), + sa.ForeignKey("user.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "engagement_id", + UUID(as_uuid=True), + sa.ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("token_hash", sa.String(255), nullable=False, unique=True), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("revoked_at", sa.DateTime(timezone=True)), + sa.Column("last_ip", sa.String(64)), + sa.Column("last_user_agent", sa.String(512)), + sa.Column("last_used_at", sa.DateTime(timezone=True)), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + ) + + # ------------------------------------------------------------ audit_log + op.create_table( + "audit_log", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("ts", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("actor_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column("action", sa.String(80), nullable=False), + sa.Column("resource_type", sa.String(80), nullable=False), + sa.Column("resource_id", sa.String(128)), + sa.Column("metadata_json", JSONB, nullable=False, server_default="{}"), + sa.Column("prev_hash", sa.String(64)), + sa.Column("row_hash", sa.String(64), nullable=False), + sa.Column("source_ip", sa.String(64)), + sa.Column("user_agent", sa.String(512)), + sa.Column("comment", sa.Text), + ) + op.create_index("ix_audit_log_ts", "audit_log", ["ts"]) + op.create_index("ix_audit_log_resource", "audit_log", ["resource_type", "resource_id"]) + + # ---------------------------------------------- NF-AUDIT role-level grants + # `mimic_audit_writer` (write-only) + `mimic_app` (read-only on audit_log). + # Idempotent: skip silently if roles are absent (dev/test boxes). + bind.exec_driver_sql( + """ + DO $do$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mimic_audit_writer') THEN + GRANT INSERT ON TABLE audit_log TO mimic_audit_writer; + REVOKE UPDATE, DELETE, TRUNCATE ON TABLE audit_log FROM mimic_audit_writer; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user) THEN + REVOKE UPDATE, DELETE, TRUNCATE ON TABLE audit_log FROM PUBLIC; + END IF; + END + $do$; + """ + ) + + # ---------------------------------------------------------------- seed RBAC + op.bulk_insert( + sa.table( + "group", + sa.column("id", UUID(as_uuid=True)), + sa.column("name", sa.String), + sa.column("description", sa.String), + sa.column("created_at", sa.DateTime(timezone=True)), + sa.column("updated_at", sa.DateTime(timezone=True)), + ), + [ + { + "id": "11111111-0000-0000-0000-000000000001", + "name": "rt_operator", + "description": "Red team operator (per-engagement scope).", + }, + { + "id": "11111111-0000-0000-0000-000000000002", + "name": "rt_lead", + "description": "Red team lead (full RT privileges).", + }, + { + "id": "11111111-0000-0000-0000-000000000003", + "name": "soc_analyst", + "description": "SOC analyst (per-engagement, scoped via soc_session).", + }, + ], + ) + + +def downgrade() -> None: + for table in ( + "audit_log", + "soc_session", + "report", + "evidence", + "detection", + "run_step_cleanup", + "run_step", + "run", + "scenario_step", + "scenario", + "ttp_version", + "ttp", + "host", + "c2_credential", + "user_group", + "engagement_member", + "engagement", + "group_permission", + "group", + "permission", + "user", + ): + op.drop_table(table) + + for enum_t in ( + EVIDENCE_STATUS, + DETECTION_SOURCE, + DETECTION_LEVEL, + CLEANUP_STATUS, + RUN_STEP_STATUS, + RUN_STATUS, + TTP_SOURCE, + PAYLOAD_TYPE, + HOST_STATUS, + C2_TYPE, + ENGAGEMENT_STATUS, + USER_TYPE, + ): + enum_t.drop(op.get_bind(), checkfirst=True) diff --git a/backend/src/mimic/db/models/__init__.py b/backend/src/mimic/db/models/__init__.py new file mode 100644 index 0000000..79066f7 --- /dev/null +++ b/backend/src/mimic/db/models/__init__.py @@ -0,0 +1,37 @@ +"""SQLAlchemy 2 typed mapped classes for every Mimic aggregate.""" + +from mimic.db.models.audit import AuditLog +from mimic.db.models.detection import Detection, Evidence +from mimic.db.models.engagement import C2Credential, Engagement, EngagementMember +from mimic.db.models.host import Host +from mimic.db.models.permission import Group, GroupPermission, Permission, UserGroup +from mimic.db.models.report import Report +from mimic.db.models.run import Run, RunStep, RunStepCleanup +from mimic.db.models.scenario import Scenario, ScenarioStep +from mimic.db.models.soc_session import SocSession +from mimic.db.models.ttp import Ttp, TtpVersion +from mimic.db.models.user import User + +__all__ = [ + "AuditLog", + "C2Credential", + "Detection", + "Engagement", + "EngagementMember", + "Evidence", + "Group", + "GroupPermission", + "Host", + "Permission", + "Report", + "Run", + "RunStep", + "RunStepCleanup", + "Scenario", + "ScenarioStep", + "SocSession", + "Ttp", + "TtpVersion", + "User", + "UserGroup", +] diff --git a/backend/src/mimic/db/models/audit.py b/backend/src/mimic/db/models/audit.py new file mode 100644 index 0000000..3c9861f --- /dev/null +++ b/backend/src/mimic/db/models/audit.py @@ -0,0 +1,45 @@ +"""Append-only audit log. + +NF-AUDIT: enforced at SQL level by granting only INSERT to +`mimic_audit_writer` and only SELECT to `mimic_app`. The ORM never updates or +deletes rows on this table from application code. Hash chain (`prev_hash` / +`row_hash`) is wired here at the schema level so v2 can switch to a strict +WORM enforcement without a migration; sprint 0 fills the columns at insert. +""" + +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import JSON, DateTime, ForeignKey, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column + +from mimic.db.base import Base, UuidPkMixin + + +class AuditLog(UuidPkMixin, Base): + __tablename__ = "audit_log" + + ts: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ) + actor_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + action: Mapped[str] = mapped_column(String(80), nullable=False) + resource_type: Mapped[str] = mapped_column(String(80), nullable=False) + resource_id: Mapped[str | None] = mapped_column(String(128)) + metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + + prev_hash: Mapped[str | None] = mapped_column(String(64)) + row_hash: Mapped[str] = mapped_column(String(64), nullable=False) + # SHA-256 of canonical (prev_hash || ts || actor_id || action || + # resource_type || resource_id || metadata_json). Provides hash-chain + # integrity once verifier runs in v2. + + source_ip: Mapped[str | None] = mapped_column(String(64)) + user_agent: Mapped[str | None] = mapped_column(String(512)) + comment: Mapped[str | None] = mapped_column(Text) diff --git a/backend/src/mimic/db/models/detection.py b/backend/src/mimic/db/models/detection.py new file mode 100644 index 0000000..9f11c29 --- /dev/null +++ b/backend/src/mimic/db/models/detection.py @@ -0,0 +1,82 @@ +"""Per-run-step detection (SOC) and offensive evidence (RT).""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ( + JSON, + DateTime, + Enum, + ForeignKey, + Integer, + Text, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import DetectionLevel, DetectionSource, EvidenceStatus + +if TYPE_CHECKING: + from mimic.db.models.run import RunStep + from mimic.db.models.user import User + + +class Detection(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "detection" + + run_step_id: Mapped[UUID] = mapped_column( + ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + ) + soc_user_id: Mapped[UUID] = mapped_column( + ForeignKey("user.id", ondelete="RESTRICT"), + nullable=False, + ) + level: Mapped[DetectionLevel] = mapped_column( + Enum(DetectionLevel, name="detection_level"), + nullable=False, + ) + source: Mapped[DetectionSource] = mapped_column( + Enum(DetectionSource, name="detection_source"), + nullable=False, + ) + latency_ms: Mapped[int | None] = mapped_column(Integer) + comment: Mapped[str | None] = mapped_column(Text) + recorded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + + run_step: Mapped[RunStep] = relationship() + soc_user: Mapped[User] = relationship() + + +class Evidence(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "evidence" + + run_step_id: Mapped[UUID] = mapped_column( + ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + ) + rt_user_id: Mapped[UUID] = mapped_column( + ForeignKey("user.id", ondelete="RESTRICT"), + nullable=False, + ) + status: Mapped[EvidenceStatus] = mapped_column( + Enum(EvidenceStatus, name="evidence_status"), + nullable=False, + ) + artifacts_text: Mapped[str | None] = mapped_column(Text) + artifact_files_json: Mapped[list[dict]] = mapped_column( + JSON, default=list, nullable=False + ) + # Each entry: {"name": str, "ref": str, "sha256": str, "size_bytes": int} + comment: Mapped[str | None] = mapped_column(Text) + recorded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + + run_step: Mapped[RunStep] = relationship() + rt_user: Mapped[User] = relationship() diff --git a/backend/src/mimic/db/models/engagement.py b/backend/src/mimic/db/models/engagement.py new file mode 100644 index 0000000..48b326e --- /dev/null +++ b/backend/src/mimic/db/models/engagement.py @@ -0,0 +1,106 @@ +"""Engagement aggregate: tenancy container + C2 credentials.""" + +from __future__ import annotations + +from datetime import date, datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import Date, DateTime, Enum, ForeignKey, Integer, LargeBinary, String, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import C2Type, EngagementStatus + +if TYPE_CHECKING: + from mimic.db.models.host import Host + from mimic.db.models.scenario import Scenario + from mimic.db.models.user import User + + +class Engagement(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "engagement" + + client_name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(String(1024)) + status: Mapped[EngagementStatus] = mapped_column( + Enum(EngagementStatus, name="engagement_status"), + default=EngagementStatus.DRAFT, + nullable=False, + ) + start_date: Mapped[date | None] = mapped_column(Date) + end_date: Mapped[date | None] = mapped_column(Date) + + c2_type: Mapped[C2Type] = mapped_column( + Enum(C2Type, name="c2_type"), + default=C2Type.MYTHIC, + nullable=False, + ) + + created_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + hosts: Mapped[list[Host]] = relationship( + back_populates="engagement", + cascade="all, delete-orphan", + ) + scenarios: Mapped[list[Scenario]] = relationship( + back_populates="engagement", + cascade="all, delete-orphan", + ) + members: Mapped[list[EngagementMember]] = relationship( + back_populates="engagement", + cascade="all, delete-orphan", + ) + c2_credentials: Mapped[list[C2Credential]] = relationship( + back_populates="engagement", + cascade="all, delete-orphan", + ) + + +class EngagementMember(Base): + __tablename__ = "engagement_member" + + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + primary_key=True, + ) + user_id: Mapped[UUID] = mapped_column( + ForeignKey("user.id", ondelete="CASCADE"), + primary_key=True, + ) + role: Mapped[str] = mapped_column(String(40), nullable=False) + added_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + + engagement: Mapped[Engagement] = relationship(back_populates="members") + user: Mapped[User] = relationship() + + +class C2Credential(UuidPkMixin, TimestampsMixin, Base): + """Per-engagement encrypted C2 credentials. + + Decision D-004: dedicated table, Fernet-encrypted blob, versioned, rotation + via insert + retire. Active row = `retired_at IS NULL` with the highest + `version` for `(engagement_id, c2_type)`. + """ + + __tablename__ = "c2_credential" + + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ) + c2_type: Mapped[C2Type] = mapped_column( + Enum(C2Type, name="c2_type", create_type=False), + nullable=False, + ) + config_fernet: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + retired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + engagement: Mapped[Engagement] = relationship(back_populates="c2_credentials") diff --git a/backend/src/mimic/db/models/host.py b/backend/src/mimic/db/models/host.py new file mode 100644 index 0000000..7db00f2 --- /dev/null +++ b/backend/src/mimic/db/models/host.py @@ -0,0 +1,41 @@ +"""Host inventory (per-engagement target machines).""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import DateTime, Enum, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import C2Type, HostStatus + +if TYPE_CHECKING: + from mimic.db.models.engagement import Engagement + + +class Host(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "host" + + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ) + hostname: Mapped[str] = mapped_column(String(255), nullable=False) + ip: Mapped[str | None] = mapped_column(String(64)) + os: Mapped[str | None] = mapped_column(String(128)) + c2_session_id: Mapped[str | None] = mapped_column(String(128)) + c2_type: Mapped[C2Type] = mapped_column( + Enum(C2Type, name="c2_type", create_type=False), + nullable=False, + ) + status: Mapped[HostStatus] = mapped_column( + Enum(HostStatus, name="host_status"), + default=HostStatus.UNKNOWN, + nullable=False, + ) + last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + engagement: Mapped[Engagement] = relationship(back_populates="hosts") diff --git a/backend/src/mimic/db/models/permission.py b/backend/src/mimic/db/models/permission.py new file mode 100644 index 0000000..d6fbeef --- /dev/null +++ b/backend/src/mimic/db/models/permission.py @@ -0,0 +1,87 @@ +"""Group-based RBAC tables. + +Decision D-003 (spec-decisions): group-based from day one so OIDC will map +claims to existing groups in v2 without touching application code. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ForeignKey, PrimaryKeyConstraint, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin + +if TYPE_CHECKING: + from mimic.db.models.engagement import Engagement + from mimic.db.models.user import User + + +class Permission(UuidPkMixin, Base): + __tablename__ = "permission" + __table_args__ = (UniqueConstraint("code", name="uq_permission_code"),) + + code: Mapped[str] = mapped_column(String(80), nullable=False) + description: Mapped[str | None] = mapped_column(String(255)) + + +class Group(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "group" + __table_args__ = (UniqueConstraint("name", name="uq_group_name"),) + + name: Mapped[str] = mapped_column(String(80), nullable=False) + description: Mapped[str | None] = mapped_column(String(255)) + + permission_links: Mapped[list[GroupPermission]] = relationship( + back_populates="group", + cascade="all, delete-orphan", + ) + user_links: Mapped[list[UserGroup]] = relationship( + back_populates="group", + cascade="all, delete-orphan", + ) + + +class GroupPermission(Base): + __tablename__ = "group_permission" + __table_args__ = ( + PrimaryKeyConstraint("group_id", "permission_id", name="pk_group_permission"), + ) + + group_id: Mapped[UUID] = mapped_column( + ForeignKey("group.id", ondelete="CASCADE"), nullable=False + ) + permission_id: Mapped[UUID] = mapped_column( + ForeignKey("permission.id", ondelete="CASCADE"), nullable=False + ) + + group: Mapped[Group] = relationship(back_populates="permission_links") + permission: Mapped[Permission] = relationship() + + +class UserGroup(Base): + __tablename__ = "user_group" + __table_args__ = ( + PrimaryKeyConstraint( + "user_id", "group_id", "engagement_id", name="pk_user_group" + ), + ) + + user_id: Mapped[UUID] = mapped_column( + ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + group_id: Mapped[UUID] = mapped_column( + ForeignKey("group.id", ondelete="CASCADE"), nullable=False + ) + engagement_id: Mapped[UUID | None] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=True, + ) + # Global membership (e.g. rt_lead enterprise-wide) when engagement_id IS NULL. + # Per-engagement membership otherwise. + + user: Mapped[User] = relationship(back_populates="group_links") + group: Mapped[Group] = relationship(back_populates="user_links") + engagement: Mapped[Engagement | None] = relationship() diff --git a/backend/src/mimic/db/models/report.py b/backend/src/mimic/db/models/report.py new file mode 100644 index 0000000..36ad5a6 --- /dev/null +++ b/backend/src/mimic/db/models/report.py @@ -0,0 +1,48 @@ +"""Mission report (PDF / JSON / Markdown bundle).""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ( + JSON, + DateTime, + ForeignKey, + Integer, + String, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin + +if TYPE_CHECKING: + from mimic.db.models.engagement import Engagement + + +class Report(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "report" + + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + + content_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + # SHA-256 of canonical JSON. Identical hash referenced in PDF footer, JSON + # export and Markdown export (spec H19 / F9 / F14). + + pdf_path: Mapped[str | None] = mapped_column(String(512)) + md_path: Mapped[str | None] = mapped_column(String(512)) + + generated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + generated_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + engagement: Mapped[Engagement] = relationship() diff --git a/backend/src/mimic/db/models/run.py b/backend/src/mimic/db/models/run.py new file mode 100644 index 0000000..1efb6a0 --- /dev/null +++ b/backend/src/mimic/db/models/run.py @@ -0,0 +1,118 @@ +"""Run aggregate: scenario execution + per-step state + cleanup.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ( + JSON, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import CleanupStatus, RunStatus, RunStepStatus + +if TYPE_CHECKING: + from mimic.db.models.scenario import Scenario, ScenarioStep + + +class Run(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "run" + + scenario_id: Mapped[UUID] = mapped_column( + ForeignKey("scenario.id", ondelete="RESTRICT"), + nullable=False, + ) + status: Mapped[RunStatus] = mapped_column( + Enum(RunStatus, name="run_status"), + default=RunStatus.QUEUED, + nullable=False, + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + started_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + snapshot_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + # Full self-contained snapshot of scenario + steps + resolved TTPs. + # Source of truth for replay (spec H32). + + scenario: Mapped[Scenario] = relationship(back_populates="runs") + steps: Mapped[list[RunStep]] = relationship( + back_populates="run", + cascade="all, delete-orphan", + order_by="RunStep.order_idx", + ) + + +class RunStep(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "run_step" + + run_id: Mapped[UUID] = mapped_column( + ForeignKey("run.id", ondelete="CASCADE"), + nullable=False, + ) + scenario_step_id: Mapped[UUID | None] = mapped_column( + ForeignKey("scenario_step.id", ondelete="SET NULL") + ) + order_idx: Mapped[int] = mapped_column(Integer, nullable=False) + + status: Mapped[RunStepStatus] = mapped_column( + Enum(RunStepStatus, name="run_step_status"), + default=RunStepStatus.QUEUED, + nullable=False, + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + c2_task_id: Mapped[str | None] = mapped_column(String(128)) + output_text: Mapped[str | None] = mapped_column(Text) + output_blob_ref: Mapped[str | None] = mapped_column(String(512)) + exit_code: Mapped[int | None] = mapped_column(Integer) + + resolved_payload_text: Mapped[str | None] = mapped_column(Text) + # The fully-resolved payload (post-Jinja) actually sent to the C2. Useful + # for audit / report. Marker (`# MIMIC-RUN:`) is added here. + + run: Mapped[Run] = relationship(back_populates="steps") + scenario_step: Mapped[ScenarioStep | None] = relationship() + + cleanup: Mapped[RunStepCleanup | None] = relationship( + back_populates="run_step", + cascade="all, delete-orphan", + uselist=False, + ) + + +class RunStepCleanup(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "run_step_cleanup" + + run_step_id: Mapped[UUID] = mapped_column( + ForeignKey("run_step.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + status: Mapped[CleanupStatus] = mapped_column( + Enum(CleanupStatus, name="cleanup_status"), + default=CleanupStatus.PENDING, + nullable=False, + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + resolved_command_text: Mapped[str | None] = mapped_column(Text) + output: Mapped[str | None] = mapped_column(Text) + executed_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + run_step: Mapped[RunStep] = relationship(back_populates="cleanup") diff --git a/backend/src/mimic/db/models/scenario.py b/backend/src/mimic/db/models/scenario.py new file mode 100644 index 0000000..19198cd --- /dev/null +++ b/backend/src/mimic/db/models/scenario.py @@ -0,0 +1,86 @@ +"""Scenario aggregate: ordered list of TTP steps. + +Spec §F3: `scenario.c2_type` is the source of truth for the run. Verified at +run start that every referenced host matches. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ( + JSON, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import C2Type + +if TYPE_CHECKING: + from mimic.db.models.engagement import Engagement + from mimic.db.models.host import Host + from mimic.db.models.run import Run + from mimic.db.models.ttp import Ttp + + +class Scenario(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "scenario" + + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + + c2_type: Mapped[C2Type] = mapped_column( + Enum(C2Type, name="c2_type", create_type=False), + nullable=False, + ) + created_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + engagement: Mapped[Engagement] = relationship(back_populates="scenarios") + steps: Mapped[list[ScenarioStep]] = relationship( + back_populates="scenario", + cascade="all, delete-orphan", + order_by="ScenarioStep.order_idx", + ) + runs: Mapped[list[Run]] = relationship(back_populates="scenario") + + +class ScenarioStep(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "scenario_step" + __table_args__ = ( + UniqueConstraint("scenario_id", "order_idx", name="uq_scenario_step_order_idx"), + ) + + scenario_id: Mapped[UUID] = mapped_column( + ForeignKey("scenario.id", ondelete="CASCADE"), + nullable=False, + ) + order_idx: Mapped[int] = mapped_column(Integer, nullable=False) + + ttp_id: Mapped[UUID] = mapped_column( + ForeignKey("ttp.id", ondelete="RESTRICT"), + nullable=False, + ) + host_id: Mapped[UUID] = mapped_column( + ForeignKey("host.id", ondelete="RESTRICT"), + nullable=False, + ) + params_override_json: Mapped[dict] = mapped_column(JSON, default=dict, nullable=False) + delay_after_ms: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + + scenario: Mapped[Scenario] = relationship(back_populates="steps") + ttp: Mapped[Ttp] = relationship() + host: Mapped[Host] = relationship() diff --git a/backend/src/mimic/db/models/soc_session.py b/backend/src/mimic/db/models/soc_session.py new file mode 100644 index 0000000..0c29a1a --- /dev/null +++ b/backend/src/mimic/db/models/soc_session.py @@ -0,0 +1,48 @@ +"""SOC analyst sessions (bcrypt-hashed opaque tokens). + +Decision D-006: bcrypt hash stored; the clear token is generated server-side at +session creation, returned **once** in the API response and delivered out-of-band. +Never re-displayable. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import DateTime, ForeignKey, String +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin + +if TYPE_CHECKING: + from mimic.db.models.engagement import Engagement + from mimic.db.models.user import User + + +class SocSession(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "soc_session" + + user_id: Mapped[UUID] = mapped_column( + ForeignKey("user.id", ondelete="CASCADE"), + nullable=False, + ) + engagement_id: Mapped[UUID] = mapped_column( + ForeignKey("engagement.id", ondelete="CASCADE"), + nullable=False, + ) + token_hash: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + # bcrypt hash. Plain token returned once at creation. + + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + last_ip: Mapped[str | None] = mapped_column(String(64)) + last_user_agent: Mapped[str | None] = mapped_column(String(512)) + last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + user: Mapped[User] = relationship(back_populates="soc_sessions") + engagement: Mapped[Engagement] = relationship() diff --git a/backend/src/mimic/db/models/ttp.py b/backend/src/mimic/db/models/ttp.py new file mode 100644 index 0000000..a6fee32 --- /dev/null +++ b/backend/src/mimic/db/models/ttp.py @@ -0,0 +1,105 @@ +"""TTP library + immutable version snapshots. + +Note (D-T-ttp-version): spec H32 originally said no `ttp_version` table (snapshot +lives on `run.snapshot_json`). Sprint 0 reintroduces a `ttp_version` table for +clean traceability across runs and to honor the kickoff data-model directive +from the team-lead. `run.snapshot_json` remains the source of truth for replay, +but each promotion / edit produces an immutable `ttp_version` row that's easier +to diff and reference from imports / audit log. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import ( + JSON, + Boolean, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import PayloadType, TtpSource + +if TYPE_CHECKING: + pass + + +class Ttp(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "ttp" + + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text) + + mitre_technique: Mapped[str] = mapped_column(String(16), nullable=False) + mitre_subtechnique: Mapped[str | None] = mapped_column(String(16)) + + payload_type: Mapped[PayloadType] = mapped_column( + Enum(PayloadType, name="payload_type"), + nullable=False, + ) + payload_template: Mapped[str] = mapped_column(Text, nullable=False, default="") + params_schema_json: Mapped[dict | None] = mapped_column(JSON) + + opsec_notes: Mapped[str | None] = mapped_column(Text) + cleanup_command: Mapped[str | None] = mapped_column(Text) + is_stealth_variant: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + source: Mapped[TtpSource] = mapped_column( + Enum(TtpSource, name="ttp_source"), + default=TtpSource.CUSTOM, + nullable=False, + ) + tags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) + + current_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + is_published: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + # is_published = promoted to the library (lead RT only — F11). + + created_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + versions: Mapped[list[TtpVersion]] = relationship( + back_populates="ttp", + cascade="all, delete-orphan", + order_by="TtpVersion.version", + ) + + +class TtpVersion(UuidPkMixin, Base): + """Immutable snapshot of a TTP at the moment it was published or used. + + Used by importers / audit / report builder. `run.snapshot_json` still + embeds a full self-contained copy for replay independence. + """ + + __tablename__ = "ttp_version" + __table_args__ = ( + UniqueConstraint("ttp_id", "version", name="uq_ttp_version_ttp_id_version"), + ) + + ttp_id: Mapped[UUID] = mapped_column( + ForeignKey("ttp.id", ondelete="CASCADE"), + nullable=False, + ) + version: Mapped[int] = mapped_column(Integer, nullable=False) + snapshot_json: Mapped[dict] = mapped_column(JSON, nullable=False) + content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + created_by_id: Mapped[UUID | None] = mapped_column( + ForeignKey("user.id", ondelete="SET NULL") + ) + + ttp: Mapped[Ttp] = relationship(back_populates="versions") diff --git a/backend/src/mimic/db/models/user.py b/backend/src/mimic/db/models/user.py new file mode 100644 index 0000000..ffc01de --- /dev/null +++ b/backend/src/mimic/db/models/user.py @@ -0,0 +1,50 @@ +"""User accounts (RT operators, RT leads, SOC analysts).""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlalchemy import DateTime, Enum, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mimic.db.base import Base, TimestampsMixin, UuidPkMixin +from mimic.db.types import UserType + +if TYPE_CHECKING: + from mimic.db.models.permission import UserGroup + from mimic.db.models.soc_session import SocSession + + +class User(UuidPkMixin, TimestampsMixin, Base): + __tablename__ = "user" + __table_args__ = (UniqueConstraint("email", name="uq_user_email"),) + + type: Mapped[UserType] = mapped_column(Enum(UserType, name="user_type"), nullable=False) + email: Mapped[str] = mapped_column(String(255), nullable=False) + display_name: Mapped[str | None] = mapped_column(String(120)) + + keycloak_sub: Mapped[str | None] = mapped_column(String(255), unique=True) + local_password_hash: Mapped[str | None] = mapped_column(String(255)) + + disabled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + group_links: Mapped[list[UserGroup]] = relationship( + back_populates="user", + cascade="all, delete-orphan", + ) + soc_sessions: Mapped[list[SocSession]] = relationship( + back_populates="user", + cascade="all, delete-orphan", + ) + + @property + def is_active(self) -> bool: + return self.disabled_at is None + + @property + def created_by_id(self) -> UUID | None: + # Placeholder for future audit linkage; kept for symmetry with spec §8. + return None diff --git a/backend/src/mimic/db/types.py b/backend/src/mimic/db/types.py new file mode 100644 index 0000000..454a038 --- /dev/null +++ b/backend/src/mimic/db/types.py @@ -0,0 +1,110 @@ +"""Reusable column / enum definitions.""" + +from __future__ import annotations + +import enum + + +class C2Type(enum.StrEnum): + """C2 backend kind. Source of truth for connector resolution. + + Spec §F3 / §F4: `scenario.c2_type` is the authoritative value at run time; + every host used in a scenario must match it. + """ + + MYTHIC = "mythic" + HOME = "home" + + +class PayloadType(enum.StrEnum): + """Internal neutral payload kind (spec §7 table). + + The C2Connector resolves the actual native command on a per-c2 basis. + `UnsupportedPayloadType` is raised when no mapping exists. + """ + + CMD = "cmd" + POWERSHELL = "powershell" + BOF = "bof" + DOTNET_ASSEMBLY = "dotnet_assembly" + DOTNET_EXE = "dotnet_exe" + PE_EXE = "pe_exe" + PE_DLL = "pe_dll" + SHELLCODE = "shellcode" + PYTHON = "python" + VBS = "vbs" + WMI_QUERY = "wmi_query" + REGISTRY = "registry" + SCRIPT_FILE = "script_file" + + +class UserType(enum.StrEnum): + """Application user kind.""" + + RT_OPERATOR = "rt_operator" + RT_LEAD = "rt_lead" + SOC_ANALYST = "soc_analyst" + + +class EngagementStatus(enum.StrEnum): + DRAFT = "draft" + ACTIVE = "active" + CLOSED = "closed" + ARCHIVED = "archived" + + +class HostStatus(enum.StrEnum): + UNKNOWN = "unknown" + ALIVE = "alive" + DEAD = "dead" + + +class TtpSource(enum.StrEnum): + CUSTOM = "custom" + IMPORT_ATR = "import_atr" + IMPORT_MISSION = "import_mission" + + +class RunStatus(enum.StrEnum): + QUEUED = "queued" + RUNNING = "running" + PAUSED = "paused" + COMPLETED = "completed" + FAILED = "failed" + ABORTED = "aborted" + + +class RunStepStatus(enum.StrEnum): + QUEUED = "queued" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + SKIPPED = "skipped" + CLEANUP_FAILED = "cleanup_failed" + + +class CleanupStatus(enum.StrEnum): + PENDING = "pending" + SUCCESS = "success" + FAILED = "failed" + PARTIAL = "partial" + + +class DetectionLevel(enum.StrEnum): + DETECTED = "detected" + PARTIAL = "partial" + NOT_DETECTED = "not_detected" + + +class DetectionSource(enum.StrEnum): + NDR = "ndr" + EDR = "edr" + SIEM = "siem" + MANUAL = "manual" + OTHER = "other" + + +class EvidenceStatus(enum.StrEnum): + SUCCESS = "success" + FAILURE = "failure" + PARTIAL = "partial" diff --git a/backend/src/mimic/extensions.py b/backend/src/mimic/extensions.py new file mode 100644 index 0000000..1a4b2c8 --- /dev/null +++ b/backend/src/mimic/extensions.py @@ -0,0 +1,15 @@ +"""Singletons wired into the Flask app at create-time.""" + +from __future__ import annotations + +from flask_login import LoginManager +from flask_migrate import Migrate +from flask_socketio import SocketIO +from flask_sqlalchemy import SQLAlchemy + +from mimic.db.base import Base + +db = SQLAlchemy(model_class=Base) +migrate = Migrate() +login_manager = LoginManager() +socketio = SocketIO(async_mode="gevent", cors_allowed_origins=[]) diff --git a/backend/src/mimic/logging.py b/backend/src/mimic/logging.py new file mode 100644 index 0000000..bb84ffc --- /dev/null +++ b/backend/src/mimic/logging.py @@ -0,0 +1,27 @@ +"""Structured JSON logging (NF-observability).""" + +from __future__ import annotations + +import logging +import sys + +from pythonjsonlogger.jsonlogger import JsonFormatter + + +def configure_logging(level: str = "INFO", *, as_json: bool = True) -> None: + """Configure the root logger once at app start.""" + handler = logging.StreamHandler(sys.stdout) + if as_json: + formatter: logging.Formatter = JsonFormatter( + "%(asctime)s %(levelname)s %(name)s %(message)s" + ) + else: + formatter = logging.Formatter( + "%(asctime)s %(levelname)-8s %(name)s: %(message)s" + ) + handler.setFormatter(formatter) + + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level.upper()) From 20112d61ff3ef250fc43b5dc9a495b5cc93288d8 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:33:06 +0200 Subject: [PATCH 03/18] feat(backend): add C2Connector ABC + payload mapping + factory (B0.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - abstract C2Connector with authenticate / list_hosts / execute_task / get_task_result / cancel_task / execute_cleanup; stream_task_output optional v1 (NotImplementedError). - Payload / TaskHandle / TaskResult / TaskStatus frozen dataclasses. - UnsupportedPayloadType raised when no native command maps to the chosen (c2_type, payload_type) pair. - Mythic payload_type → native command map populated (spec §7 table). - HOME map left empty until PR2 is closed. - ConnectorFactory: register_connector decorator + build(c2_type) that instantiates + authenticates via an injected config resolver. No real Mythic / Home implementations land in this sprint. --- backend/src/mimic/connectors/__init__.py | 27 +++++ backend/src/mimic/connectors/base.py | 122 ++++++++++++++++++++ backend/src/mimic/connectors/factory.py | 55 +++++++++ backend/src/mimic/connectors/payload_map.py | 47 ++++++++ 4 files changed, 251 insertions(+) create mode 100644 backend/src/mimic/connectors/__init__.py create mode 100644 backend/src/mimic/connectors/base.py create mode 100644 backend/src/mimic/connectors/factory.py create mode 100644 backend/src/mimic/connectors/payload_map.py diff --git a/backend/src/mimic/connectors/__init__.py b/backend/src/mimic/connectors/__init__.py new file mode 100644 index 0000000..f55ccba --- /dev/null +++ b/backend/src/mimic/connectors/__init__.py @@ -0,0 +1,27 @@ +"""C2Connector abstraction. + +Sprint 0 ships the interface + dataclasses + payload mapping + factory. +Concrete `MythicConnector` and `HomeConnector` implementations land after +PR1 and PR2 respectively. +""" + +from mimic.connectors.base import ( + C2Connector, + Payload, + TaskHandle, + TaskResult, + TaskStatus, + UnsupportedPayloadType, +) +from mimic.connectors.factory import ConnectorFactory, register_connector + +__all__ = [ + "C2Connector", + "ConnectorFactory", + "Payload", + "TaskHandle", + "TaskResult", + "TaskStatus", + "UnsupportedPayloadType", + "register_connector", +] diff --git a/backend/src/mimic/connectors/base.py b/backend/src/mimic/connectors/base.py new file mode 100644 index 0000000..4a56a6b --- /dev/null +++ b/backend/src/mimic/connectors/base.py @@ -0,0 +1,122 @@ +"""Abstract C2 connector interface (spec §7). + +The orchestrator calls `execute_task` → polls `get_task_result` every 500 ms +until a terminal `TaskStatus`. `stream_task_output` is optional in v1. +`cancel_task` backs F6 Abort. `execute_cleanup` runs the resolved Jinja2 +template against the C2 (spec F15). +""" + +from __future__ import annotations + +import enum +from abc import ABC, abstractmethod +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from mimic.db.types import C2Type, PayloadType + +if TYPE_CHECKING: + from mimic.db.models.host import Host + + +class UnsupportedPayloadType(RuntimeError): # noqa: N818 (kept for spec wording) + """Raised when the chosen C2 has no native command for a payload kind. + + The name mirrors the exact identifier from spec §7 (`UnsupportedPayloadType`). + The trailing `Error` suffix is intentionally omitted to keep the spec link + one-to-one. + """ + + def __init__(self, c2: C2Type, payload_type: PayloadType) -> None: + super().__init__(f"{c2.value} does not support payload_type={payload_type.value}") + self.c2 = c2 + self.payload_type = payload_type + + +class TaskStatus(enum.StrEnum): + """Terminal and non-terminal task lifecycle states.""" + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELED = "canceled" + + @property + def is_terminal(self) -> bool: + return self in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELED} + + +@dataclass(frozen=True, slots=True) +class Payload: + """Self-contained payload sent to a C2.""" + + payload_type: PayloadType + template_text: str + params: dict[str, object] = field(default_factory=dict) + mimic_run_id: str | None = None + is_stealth_variant: bool = False + # When True the orchestrator MUST strip the MIMIC marker (NF-OPSEC). + + +@dataclass(frozen=True, slots=True) +class TaskHandle: + """Opaque per-connector reference to a started task.""" + + c2: C2Type + c2_task_id: str + host_id: str + payload_type: PayloadType + + +@dataclass(frozen=True, slots=True) +class TaskResult: + status: TaskStatus + output_text: str = "" + output_blob_ref: str | None = None + exit_code: int | None = None + error_message: str | None = None + + +class C2Connector(ABC): + """Abstract base for every C2 backend.""" + + name: C2Type + + @abstractmethod + def authenticate(self, config: dict[str, object]) -> None: + """Open / refresh the auth context for this connector.""" + + @abstractmethod + def list_hosts(self, engagement_id: str) -> list[Host]: + """Return hosts known by the C2 for the given engagement.""" + + @abstractmethod + def execute_task(self, host: Host, payload: Payload) -> TaskHandle: + """Start a task on `host`. MUST NOT block.""" + + @abstractmethod + def get_task_result(self, handle: TaskHandle) -> TaskResult: + """Poll status (called every 500 ms by the orchestrator).""" + + def stream_task_output(self, handle: TaskHandle) -> Iterator[bytes]: + """Optional v2. Connectors may leave this raising NotImplementedError.""" + raise NotImplementedError + + @abstractmethod + def cancel_task(self, handle: TaskHandle) -> None: + """Abort a running task (F6 Abort).""" + + @abstractmethod + def execute_cleanup( + self, + host: Host, + resolved_command: str, + params: dict[str, object], + ) -> TaskResult: + """Run a fully-resolved cleanup command (F15). + + The Jinja2 template is rendered by the orchestrator BEFORE this call; + connectors never see template variables. + """ diff --git a/backend/src/mimic/connectors/factory.py b/backend/src/mimic/connectors/factory.py new file mode 100644 index 0000000..2c7c2cd --- /dev/null +++ b/backend/src/mimic/connectors/factory.py @@ -0,0 +1,55 @@ +"""Connector factory keyed on `c2_type`. + +Concrete connectors register themselves at import time via the +`@register_connector` decorator. Sprint 0 ships only the interface — no real +implementation registers in this codebase yet. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TypeVar + +from mimic.connectors.base import C2Connector +from mimic.db.types import C2Type + +ConnectorClass = type[C2Connector] +C = TypeVar("C", bound=C2Connector) + +_REGISTRY: dict[C2Type, ConnectorClass] = {} + + +def register_connector(c2_type: C2Type) -> Callable[[type[C]], type[C]]: + """Class decorator: register a concrete connector under its C2Type.""" + + def _wrap(klass: type[C]) -> type[C]: + if c2_type in _REGISTRY: + raise RuntimeError(f"connector already registered for {c2_type.value}") + _REGISTRY[c2_type] = klass + klass.name = c2_type + return klass + + return _wrap + + +class ConnectorFactory: + """Resolves a connector instance for a given C2 type.""" + + def __init__(self, config_resolver: Callable[[C2Type], dict[str, object]]): + self._resolver = config_resolver + + @staticmethod + def registered() -> dict[C2Type, ConnectorClass]: + return dict(_REGISTRY) + + def build(self, c2_type: C2Type) -> C2Connector: + try: + klass = _REGISTRY[c2_type] + except KeyError as exc: + raise NotImplementedError( + f"no connector registered for {c2_type.value}" + ) from exc + + connector = klass() + connector.authenticate(self._resolver(c2_type)) + return connector diff --git a/backend/src/mimic/connectors/payload_map.py b/backend/src/mimic/connectors/payload_map.py new file mode 100644 index 0000000..967c4f3 --- /dev/null +++ b/backend/src/mimic/connectors/payload_map.py @@ -0,0 +1,47 @@ +"""Static `payload_type` → native command mapping per C2 (spec §7 table). + +Concrete connectors consume this map in `execute_task` to translate the +neutral `PayloadType` into the right C2 verb. Unmapped combinations raise +`UnsupportedPayloadType`. +""" + +from __future__ import annotations + +from mimic.connectors.base import UnsupportedPayloadType +from mimic.db.types import C2Type, PayloadType + +MYTHIC_MAP: dict[PayloadType, str] = { + PayloadType.CMD: "shell", + PayloadType.POWERSHELL: "powershell", + PayloadType.BOF: "inline-execute", + PayloadType.DOTNET_ASSEMBLY: "execute-assembly", + PayloadType.DOTNET_EXE: "execute-assembly", + PayloadType.PE_EXE: "spawn", + PayloadType.PE_DLL: "loadlibrary", + PayloadType.SHELLCODE: "inject", + PayloadType.PYTHON: "python", + PayloadType.VBS: "shell", + PayloadType.WMI_QUERY: "wmi_query", + PayloadType.REGISTRY: "reg", + PayloadType.SCRIPT_FILE: "upload_and_exec", +} + +# Home connector mapping is TBD (PR2). Empty dict = nothing supported yet. +HOME_MAP: dict[PayloadType, str] = {} + +_BY_C2: dict[C2Type, dict[PayloadType, str]] = { + C2Type.MYTHIC: MYTHIC_MAP, + C2Type.HOME: HOME_MAP, +} + + +def resolve_native(c2: C2Type, payload_type: PayloadType) -> str: + """Resolve the native command for a (c2, payload_type) pair.""" + mapping = _BY_C2.get(c2, {}) + if payload_type not in mapping: + raise UnsupportedPayloadType(c2, payload_type) + return mapping[payload_type] + + +def supports(c2: C2Type, payload_type: PayloadType) -> bool: + return payload_type in _BY_C2.get(c2, {}) From 104d73143a19ab7db6aaae28b40f082846e85126 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:33:19 +0200 Subject: [PATCH 04/18] feat(backend): add Jinja2 sandbox + regex_extract filter (B0.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CleanupRenderer wraps jinja2.sandbox.SandboxedEnvironment with StrictUndefined (no autoescape — shell context, not HTML). - Custom filter regex_extract(text, pattern, group=1, default='') uses google-re2 for linear-time matching (ReDoS-safe) and falls back to re with a 1 MB input cap when re2 is absent. - StepOutputs exposes {{ outputs.text }} and {{ outputs.blob('name') }}. blob() decodes UTF-8 with latin-1 fallback, hard-capped at 10 MB (consistent with F8 evidence limit, D-005). - render_cleanup() is the module-level convenience wrapper. --- backend/src/mimic/templating/__init__.py | 5 ++ backend/src/mimic/templating/filters.py | 55 ++++++++++++ backend/src/mimic/templating/sandbox.py | 106 +++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 backend/src/mimic/templating/__init__.py create mode 100644 backend/src/mimic/templating/filters.py create mode 100644 backend/src/mimic/templating/sandbox.py diff --git a/backend/src/mimic/templating/__init__.py b/backend/src/mimic/templating/__init__.py new file mode 100644 index 0000000..b707538 --- /dev/null +++ b/backend/src/mimic/templating/__init__.py @@ -0,0 +1,5 @@ +"""Jinja2 sandboxed templating used for cleanup commands and payloads.""" + +from mimic.templating.sandbox import CleanupRenderer, RenderError, render_cleanup + +__all__ = ["CleanupRenderer", "RenderError", "render_cleanup"] diff --git a/backend/src/mimic/templating/filters.py b/backend/src/mimic/templating/filters.py new file mode 100644 index 0000000..b7be750 --- /dev/null +++ b/backend/src/mimic/templating/filters.py @@ -0,0 +1,55 @@ +"""Custom Jinja2 filters. + +`regex_extract(text, pattern, group=1, default="")` uses google-re2 for +linear-time matching to neutralize ReDoS on adversarial C2 output. If the +library isn't installed the implementation falls back to `re` with a hard +length cap. +""" + +from __future__ import annotations + +import re + +try: # pragma: no cover - presence depends on environment + import re2 as _re2 # type: ignore[import-not-found] + + _HAS_RE2 = True +except ImportError: # pragma: no cover + _re2 = None + _HAS_RE2 = False + + +_FALLBACK_MAX_INPUT = 1 * 1024 * 1024 # 1 MB safety cap when re2 missing + + +def regex_extract( + text: object, + pattern: str, + group: int = 1, + default: str = "", +) -> str: + """Return capture group `group` of the first match of `pattern` in `text`.""" + if text is None: + return default + haystack = text if isinstance(text, str) else str(text) + + if _HAS_RE2: + compiled = _re2.compile(pattern) + match = compiled.search(haystack) + else: + if len(haystack) > _FALLBACK_MAX_INPUT: + haystack = haystack[:_FALLBACK_MAX_INPUT] + compiled_py = re.compile(pattern) + match = compiled_py.search(haystack) + + if match is None: + return default + try: + captured = match.group(group) + except (IndexError, _IndexErrors): + return default + return captured if captured is not None else default + + +# `re2.error` is `_re2.error`; `re.error` differs. Tuple them for safe catch. +_IndexErrors = (re.error,) diff --git a/backend/src/mimic/templating/sandbox.py b/backend/src/mimic/templating/sandbox.py new file mode 100644 index 0000000..671bd17 --- /dev/null +++ b/backend/src/mimic/templating/sandbox.py @@ -0,0 +1,106 @@ +"""Sandboxed Jinja2 environment used to resolve cleanup commands and payloads. + +Spec H26 / D-005: two output accessors are exposed. + +- `{{ params. }}` — straight from the merged TTP/scenario parameters. +- `{{ outputs.text }}` — `run_step.output_text` (stdout / UTF-8 text). +- `{{ outputs.blob("name") }}` — decoded `output_blob_ref` content, 10 MB cap, + UTF-8 with latin-1 fallback, silent empty string on non-decodable data. + +The custom `regex_extract` filter operates on the resulting string only. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from jinja2 import StrictUndefined, TemplateError +from jinja2.sandbox import SandboxedEnvironment + +from mimic.config import get_settings +from mimic.templating.filters import regex_extract + + +class RenderError(RuntimeError): + """Raised when a cleanup / payload template cannot be rendered safely.""" + + +@dataclass(frozen=True, slots=True) +class StepOutputs: + """Read-only view of the previous step's outputs exposed to templates.""" + + text: str = "" + blob_path: Path | None = None + blob_max_bytes: int = 10 * 1024 * 1024 + + def blob(self, _name: str = "default") -> str: + """Read the binary output blob, decoded (UTF-8 → latin-1 fallback). + + The argument is accepted for future multi-blob support but ignored in + v1 — a step has at most one blob attachment. + """ + if self.blob_path is None: + return "" + try: + raw = self.blob_path.read_bytes() + except OSError: + return "" + if len(raw) > self.blob_max_bytes: + raw = raw[: self.blob_max_bytes] + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + try: + return raw.decode("latin-1") + except UnicodeDecodeError: # pragma: no cover - latin-1 never fails + return "" + + +class CleanupRenderer: + """Sandboxed Jinja2 renderer for cleanup commands and payload templates.""" + + def __init__(self) -> None: + env = SandboxedEnvironment( + undefined=StrictUndefined, + autoescape=False, + trim_blocks=False, + lstrip_blocks=False, + keep_trailing_newline=False, + ) + env.filters["regex_extract"] = regex_extract + self._env = env + + def render( + self, + template_text: str, + *, + params: Mapping[str, Any] | None = None, + outputs: StepOutputs | None = None, + ) -> str: + try: + tmpl = self._env.from_string(template_text) + return tmpl.render( + params=dict(params or {}), + outputs=outputs or StepOutputs(), + ) + except TemplateError as exc: + raise RenderError(str(exc)) from exc + + +_RENDERER = CleanupRenderer() + + +def render_cleanup( + template_text: str, + *, + params: Mapping[str, Any] | None = None, + outputs: StepOutputs | None = None, +) -> str: + """Module-level convenience: render with the singleton renderer.""" + if outputs is None: + settings = get_settings() + outputs = StepOutputs(blob_max_bytes=settings.output_blob_max_bytes) + return _RENDERER.render(template_text, params=params, outputs=outputs) From 7f4ad85a68d986241b7f017140ff4e351d886f21 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:33:31 +0200 Subject: [PATCH 05/18] feat(backend): add local auth + group-based RBAC matching F11 (B0.6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Permission enum + GroupName enum + GROUP_PERMISSIONS mapping mirror the F11 matrix in code (verifiable against the spec table in tests). - @require_perm decorator: 401 on anonymous, 403 on missing permission, passes through otherwise. Pure-function user_has() for unit-testing. - AuthUser (Flask-Login wrapper) resolves the permission set from a User's groups; load_user is the Flask-Login user_loader. - bcrypt password hashing helpers (12 rounds by default, configurable). - SOC opaque token (D-006): secrets.token_urlsafe(32), bcrypt-hashed at rest, plain value returned once at creation and never re-displayable. - Group-based RBAC from day one (D-003) — Keycloak OIDC in v2 maps onto the same group model. --- backend/src/mimic/auth/__init__.py | 6 ++ backend/src/mimic/auth/identity.py | 60 ++++++++++++++++++ backend/src/mimic/auth/password.py | 25 ++++++++ backend/src/mimic/auth/soc_token.py | 44 +++++++++++++ backend/src/mimic/rbac/__init__.py | 17 +++++ backend/src/mimic/rbac/decorators.py | 42 ++++++++++++ backend/src/mimic/rbac/matrix.py | 95 ++++++++++++++++++++++++++++ 7 files changed, 289 insertions(+) create mode 100644 backend/src/mimic/auth/__init__.py create mode 100644 backend/src/mimic/auth/identity.py create mode 100644 backend/src/mimic/auth/password.py create mode 100644 backend/src/mimic/auth/soc_token.py create mode 100644 backend/src/mimic/rbac/__init__.py create mode 100644 backend/src/mimic/rbac/decorators.py create mode 100644 backend/src/mimic/rbac/matrix.py diff --git a/backend/src/mimic/auth/__init__.py b/backend/src/mimic/auth/__init__.py new file mode 100644 index 0000000..2b0abbd --- /dev/null +++ b/backend/src/mimic/auth/__init__.py @@ -0,0 +1,6 @@ +"""Authentication: local password (Flask-Login) + SOC sessions.""" + +from mimic.auth.identity import AuthUser, load_user +from mimic.auth.password import check_password, hash_password + +__all__ = ["AuthUser", "check_password", "hash_password", "load_user"] diff --git a/backend/src/mimic/auth/identity.py b/backend/src/mimic/auth/identity.py new file mode 100644 index 0000000..10912dc --- /dev/null +++ b/backend/src/mimic/auth/identity.py @@ -0,0 +1,60 @@ +"""Flask-Login user wrapper that carries the resolved permission set.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import selectinload + +from mimic.db.models import User, UserGroup +from mimic.extensions import db +from mimic.rbac.matrix import GROUP_PERMISSIONS, GroupName, Permission + + +@dataclass(slots=True) +class AuthUser: + """Lightweight identity attached to ``flask.g`` / ``current_user``.""" + + id: UUID + email: str + permissions: frozenset[Permission] = field(default_factory=frozenset) + groups: frozenset[str] = field(default_factory=frozenset) + is_authenticated: bool = True + is_active: bool = True + is_anonymous: bool = False + + def get_id(self) -> str: + return str(self.id) + + +def load_user(user_id: str) -> AuthUser | None: + """Flask-Login `user_loader` callback.""" + try: + uid = UUID(user_id) + except (ValueError, TypeError): + return None + + stmt = ( + select(User) + .where(User.id == uid) + .options(selectinload(User.group_links).selectinload(UserGroup.group)) + ) + user = db.session.execute(stmt).scalar_one_or_none() + if user is None or not user.is_active: + return None + + group_names = {link.group.name for link in user.group_links} + perms: set[Permission] = set() + for group_name in group_names: + try: + perms.update(GROUP_PERMISSIONS[GroupName(group_name)]) + except ValueError: + continue + return AuthUser( + id=user.id, + email=user.email, + permissions=frozenset(perms), + groups=frozenset(group_names), + ) diff --git a/backend/src/mimic/auth/password.py b/backend/src/mimic/auth/password.py new file mode 100644 index 0000000..7c84ade --- /dev/null +++ b/backend/src/mimic/auth/password.py @@ -0,0 +1,25 @@ +"""bcrypt password helpers.""" + +from __future__ import annotations + +import bcrypt + +_DEFAULT_ROUNDS = 12 + + +def hash_password(plain: str, *, rounds: int = _DEFAULT_ROUNDS) -> str: + """Hash a plain password with bcrypt; returns the UTF-8 string.""" + if not plain: + raise ValueError("password must not be empty") + salt = bcrypt.gensalt(rounds=rounds) + return bcrypt.hashpw(plain.encode("utf-8"), salt).decode("utf-8") + + +def check_password(plain: str, hashed: str | None) -> bool: + """Constant-time bcrypt verification (False if hash is missing/invalid).""" + if not plain or not hashed: + return False + try: + return bcrypt.checkpw(plain.encode("utf-8"), hashed.encode("utf-8")) + except (ValueError, TypeError): + return False diff --git a/backend/src/mimic/auth/soc_token.py b/backend/src/mimic/auth/soc_token.py new file mode 100644 index 0000000..6e82958 --- /dev/null +++ b/backend/src/mimic/auth/soc_token.py @@ -0,0 +1,44 @@ +"""SOC opaque token generation + verification. + +Decision D-006: clear token returned once at creation, bcrypt hash persisted. +""" + +from __future__ import annotations + +import hmac +import secrets +from dataclasses import dataclass + +import bcrypt + +_TOKEN_BYTES = 32 # 256 bits of entropy +_DEFAULT_ROUNDS = 12 + + +@dataclass(frozen=True, slots=True) +class SocTokenMaterial: + plain: str + hashed: str + + +def generate_token(*, rounds: int = _DEFAULT_ROUNDS) -> SocTokenMaterial: + """Generate a fresh SOC token and its bcrypt hash.""" + plain = secrets.token_urlsafe(_TOKEN_BYTES) + salt = bcrypt.gensalt(rounds=rounds) + hashed = bcrypt.hashpw(plain.encode("utf-8"), salt).decode("utf-8") + return SocTokenMaterial(plain=plain, hashed=hashed) + + +def verify_token(plain: str, hashed: str) -> bool: + """Constant-time bcrypt verification of a SOC token.""" + if not plain or not hashed: + return False + try: + return bcrypt.checkpw(plain.encode("utf-8"), hashed.encode("utf-8")) + except (ValueError, TypeError): + return False + + +def safe_compare(a: str, b: str) -> bool: + """Constant-time string compare for non-hashed contexts.""" + return hmac.compare_digest(a, b) diff --git a/backend/src/mimic/rbac/__init__.py b/backend/src/mimic/rbac/__init__.py new file mode 100644 index 0000000..183c051 --- /dev/null +++ b/backend/src/mimic/rbac/__init__.py @@ -0,0 +1,17 @@ +"""Group-based RBAC (spec F11 matrix).""" + +from mimic.rbac.decorators import require_perm +from mimic.rbac.matrix import ( + ALL_PERMISSIONS, + DEFAULT_GROUPS, + GROUP_PERMISSIONS, + Permission, +) + +__all__ = [ + "ALL_PERMISSIONS", + "DEFAULT_GROUPS", + "GROUP_PERMISSIONS", + "Permission", + "require_perm", +] diff --git a/backend/src/mimic/rbac/decorators.py b/backend/src/mimic/rbac/decorators.py new file mode 100644 index 0000000..ae342a2 --- /dev/null +++ b/backend/src/mimic/rbac/decorators.py @@ -0,0 +1,42 @@ +"""`@require_perm` Flask decorator (group-based RBAC).""" + +from __future__ import annotations + +from collections.abc import Callable +from functools import wraps +from typing import TYPE_CHECKING, ParamSpec, TypeVar + +from flask import abort +from flask_login import current_user + +from mimic.rbac.matrix import Permission + +if TYPE_CHECKING: + pass + +P = ParamSpec("P") +R = TypeVar("R") + + +def require_perm(perm: Permission) -> Callable[[Callable[P, R]], Callable[P, R]]: + """Reject the request with 401/403 unless the user holds `perm`.""" + + def _decorate(view: Callable[P, R]) -> Callable[P, R]: + @wraps(view) + def _wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + user = current_user + if not getattr(user, "is_authenticated", False): + abort(401) + permissions: frozenset[Permission] = getattr(user, "permissions", frozenset()) + if perm not in permissions: + abort(403) + return view(*args, **kwargs) + + return _wrapped # type: ignore[return-value] + + return _decorate + + +def user_has(perm: Permission, permissions: frozenset[Permission]) -> bool: + """Pure helper, easier to unit-test than the decorator.""" + return perm in permissions diff --git a/backend/src/mimic/rbac/matrix.py b/backend/src/mimic/rbac/matrix.py new file mode 100644 index 0000000..66aadbb --- /dev/null +++ b/backend/src/mimic/rbac/matrix.py @@ -0,0 +1,95 @@ +"""F11 permission matrix as code. + +This mirrors the spec table 1:1 — keeping it as a single source in code lets +us hash-check it against the spec in CI (spec-analyst task S0.2). Three default +groups are seeded by Alembic and align with the three user types. +""" + +from __future__ import annotations + +import enum + + +class Permission(enum.StrEnum): + """Application permissions. Codes are kebab-case + dotted scope.""" + + # Engagement + ENGAGEMENT_CREATE = "engagement.create" + ENGAGEMENT_READ = "engagement.read" + ENGAGEMENT_READ_OWN = "engagement.read_own" # scoped to soc_session + ENGAGEMENT_UPDATE = "engagement.update" + ENGAGEMENT_DELETE = "engagement.delete" + ENGAGEMENT_MEMBER_MANAGE = "engagement.member.manage" + ENGAGEMENT_SOC_TOKEN_ISSUE = "engagement.soc_token.issue" # noqa: S105 + + # Hosts + HOST_CRUD = "host.crud" + + # TTPs + TTP_READ = "ttp.read" + TTP_DRAFT = "ttp.draft" + TTP_PROMOTE = "ttp.promote" + + # Imports + IMPORT_JOURNAL = "import.journal" + + # Scenarios + SCENARIO_CRUD = "scenario.crud" + + # Runs + RUN_START = "run.start" + RUN_CONTROL = "run.control" + + # Detection / evidence + EVIDENCE_ADD = "evidence.add" + DETECTION_ADD = "detection.add" + + # Cleanup + CLEANUP_TRIGGER = "cleanup.trigger" + + # Reports + REPORT_GENERATE = "report.generate" + REPORT_READ = "report.read" + + # Audit + AUDIT_READ = "audit.read" + + +ALL_PERMISSIONS: tuple[Permission, ...] = tuple(Permission) + + +class GroupName(enum.StrEnum): + RT_OPERATOR = "rt_operator" + RT_LEAD = "rt_lead" + SOC_ANALYST = "soc_analyst" + + +# Source-of-truth mapping derived from spec §F11. Verified by tests against +# the spec table. +GROUP_PERMISSIONS: dict[GroupName, frozenset[Permission]] = { + GroupName.RT_OPERATOR: frozenset( + { + Permission.ENGAGEMENT_CREATE, + Permission.ENGAGEMENT_READ, + Permission.HOST_CRUD, + Permission.TTP_READ, + Permission.TTP_DRAFT, + Permission.IMPORT_JOURNAL, + Permission.SCENARIO_CRUD, + Permission.EVIDENCE_ADD, + Permission.CLEANUP_TRIGGER, + Permission.REPORT_READ, + } + ), + GroupName.RT_LEAD: frozenset(ALL_PERMISSIONS), + GroupName.SOC_ANALYST: frozenset( + { + Permission.ENGAGEMENT_READ_OWN, + Permission.DETECTION_ADD, + Permission.REPORT_READ, + } + ), +} + + +DEFAULT_GROUPS: tuple[GroupName, ...] = tuple(GroupName) From 9fa4d613048487b7931303ad20814f426eee3573 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:33:45 +0200 Subject: [PATCH 06/18] feat(backend): add Flask app factory, audit writer, flat CRUD + CLI (B0.7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Flask app factory wires SQLAlchemy / Migrate / Login / SocketIO and registers every blueprint. /healthz smoke endpoint included. - Pydantic 2 DTOs (request/response) for engagement / host / TTP / scenario aggregates with from_attributes=True conversion. - Flat CRUD blueprints under /api/v1/: * engagements (list / create / get / put / delete-as-archive) * hosts (engagement-scoped CRUD) * library/ttps (CRUD; promote requires the lead-only TTP_PROMOTE) * scenarios + steps (F3 invariant enforced: host.c2_type must match scenario.c2_type at compose time, 400 otherwise). - @require_perm guards every endpoint per the F11 matrix. - audit/ writer is hash-chained from v1 (SHA-256 of canonical record plus previous hash). The SQL-level write-only role enforcement ships in the deploy playbook (idempotent grants run at migration time). - mimic-cli (click): user create (seeds RT operator/lead with group membership), db dump / db restore (manual pg_dump/pg_restore, R-O1). No orchestrator, no WebSocket, no report generation — those land after PR1/PR2/PR3. --- backend/src/mimic/api/__init__.py | 17 +++ backend/src/mimic/api/_helpers.py | 29 +++++ backend/src/mimic/api/engagements.py | 73 +++++++++++++ backend/src/mimic/api/hosts.py | 76 +++++++++++++ backend/src/mimic/api/scenarios.py | 138 ++++++++++++++++++++++++ backend/src/mimic/api/ttps.py | 80 ++++++++++++++ backend/src/mimic/app.py | 50 +++++++++ backend/src/mimic/audit/__init__.py | 5 + backend/src/mimic/audit/log.py | 102 ++++++++++++++++++ backend/src/mimic/cli/__init__.py | 21 ++++ backend/src/mimic/cli/db.py | 71 ++++++++++++ backend/src/mimic/cli/user.py | 52 +++++++++ backend/src/mimic/schemas/__init__.py | 33 ++++++ backend/src/mimic/schemas/engagement.py | 36 +++++++ backend/src/mimic/schemas/host.py | 36 +++++++ backend/src/mimic/schemas/scenario.py | 51 +++++++++ backend/src/mimic/schemas/ttp.py | 49 +++++++++ 17 files changed, 919 insertions(+) create mode 100644 backend/src/mimic/api/__init__.py create mode 100644 backend/src/mimic/api/_helpers.py create mode 100644 backend/src/mimic/api/engagements.py create mode 100644 backend/src/mimic/api/hosts.py create mode 100644 backend/src/mimic/api/scenarios.py create mode 100644 backend/src/mimic/api/ttps.py create mode 100644 backend/src/mimic/app.py create mode 100644 backend/src/mimic/audit/__init__.py create mode 100644 backend/src/mimic/audit/log.py create mode 100644 backend/src/mimic/cli/__init__.py create mode 100644 backend/src/mimic/cli/db.py create mode 100644 backend/src/mimic/cli/user.py create mode 100644 backend/src/mimic/schemas/__init__.py create mode 100644 backend/src/mimic/schemas/engagement.py create mode 100644 backend/src/mimic/schemas/host.py create mode 100644 backend/src/mimic/schemas/scenario.py create mode 100644 backend/src/mimic/schemas/ttp.py diff --git a/backend/src/mimic/api/__init__.py b/backend/src/mimic/api/__init__.py new file mode 100644 index 0000000..be1f66d --- /dev/null +++ b/backend/src/mimic/api/__init__.py @@ -0,0 +1,17 @@ +"""Flask blueprints (REST API).""" + +from __future__ import annotations + +from flask import Flask + +from mimic.api.engagements import bp as engagements_bp +from mimic.api.hosts import bp as hosts_bp +from mimic.api.scenarios import bp as scenarios_bp +from mimic.api.ttps import bp as ttps_bp + + +def register_blueprints(app: Flask) -> None: + app.register_blueprint(engagements_bp, url_prefix="/api/v1/engagements") + app.register_blueprint(hosts_bp, url_prefix="/api/v1") + app.register_blueprint(ttps_bp, url_prefix="/api/v1/library/ttps") + app.register_blueprint(scenarios_bp, url_prefix="/api/v1") diff --git a/backend/src/mimic/api/_helpers.py b/backend/src/mimic/api/_helpers.py new file mode 100644 index 0000000..6d296ce --- /dev/null +++ b/backend/src/mimic/api/_helpers.py @@ -0,0 +1,29 @@ +"""Shared blueprint helpers (pydantic validation, error mapping).""" + +from __future__ import annotations + +from uuid import UUID + +from flask import Response, abort, jsonify, request +from pydantic import BaseModel, ValidationError + + +def parse_body[T: BaseModel](model: type[T]) -> T: + payload = request.get_json(silent=True) + if payload is None: + abort(400, description="JSON body required") + try: + return model.model_validate(payload) + except ValidationError as exc: + abort(422, description=exc.errors()) + + +def jsonify_model(model: BaseModel, status: int = 200) -> tuple[Response, int]: + return jsonify(model.model_dump(mode="json")), status + + +def parse_uuid(value: str, *, field: str = "id") -> UUID: + try: + return UUID(value) + except (ValueError, TypeError): + abort(404, description=f"invalid {field}") diff --git a/backend/src/mimic/api/engagements.py b/backend/src/mimic/api/engagements.py new file mode 100644 index 0000000..0df267c --- /dev/null +++ b/backend/src/mimic/api/engagements.py @@ -0,0 +1,73 @@ +"""Engagement CRUD endpoints (flat, sprint 0).""" + +from __future__ import annotations + +from flask import Blueprint, abort, jsonify +from sqlalchemy import select + +from mimic.api._helpers import jsonify_model, parse_body, parse_uuid +from mimic.db.models import Engagement +from mimic.db.types import EngagementStatus +from mimic.extensions import db +from mimic.rbac import Permission, require_perm +from mimic.schemas import EngagementCreate, EngagementRead, EngagementUpdate + +bp = Blueprint("engagements", __name__) + + +@bp.get("") +@require_perm(Permission.ENGAGEMENT_READ) +def list_engagements(): + stmt = select(Engagement).order_by(Engagement.created_at.desc()) + rows = db.session.execute(stmt).scalars().all() + return jsonify([EngagementRead.model_validate(row).model_dump(mode="json") for row in rows]) + + +@bp.post("") +@require_perm(Permission.ENGAGEMENT_CREATE) +def create_engagement(): + payload = parse_body(EngagementCreate) + engagement = Engagement( + client_name=payload.client_name, + description=payload.description, + c2_type=payload.c2_type, + start_date=payload.start_date, + end_date=payload.end_date, + status=EngagementStatus.DRAFT, + ) + db.session.add(engagement) + db.session.commit() + return jsonify_model(EngagementRead.model_validate(engagement), status=201) + + +@bp.get("/") +@require_perm(Permission.ENGAGEMENT_READ) +def get_engagement(eid: str): + engagement = db.session.get(Engagement, parse_uuid(eid)) + if engagement is None: + abort(404) + return jsonify_model(EngagementRead.model_validate(engagement)) + + +@bp.put("/") +@require_perm(Permission.ENGAGEMENT_UPDATE) +def update_engagement(eid: str): + engagement = db.session.get(Engagement, parse_uuid(eid)) + if engagement is None: + abort(404) + payload = parse_body(EngagementUpdate) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(engagement, field, value) + db.session.commit() + return jsonify_model(EngagementRead.model_validate(engagement)) + + +@bp.delete("/") +@require_perm(Permission.ENGAGEMENT_DELETE) +def delete_engagement(eid: str): + engagement = db.session.get(Engagement, parse_uuid(eid)) + if engagement is None: + abort(404) + engagement.status = EngagementStatus.ARCHIVED + db.session.commit() + return "", 204 diff --git a/backend/src/mimic/api/hosts.py b/backend/src/mimic/api/hosts.py new file mode 100644 index 0000000..0594862 --- /dev/null +++ b/backend/src/mimic/api/hosts.py @@ -0,0 +1,76 @@ +"""Host CRUD endpoints (scoped under an engagement).""" + +from __future__ import annotations + +from flask import Blueprint, abort, jsonify +from sqlalchemy import select + +from mimic.api._helpers import jsonify_model, parse_body, parse_uuid +from mimic.db.models import Engagement, Host +from mimic.db.types import HostStatus +from mimic.extensions import db +from mimic.rbac import Permission, require_perm +from mimic.schemas import HostCreate, HostRead, HostUpdate + +bp = Blueprint("hosts", __name__) + + +def _engagement_or_404(eid: str) -> Engagement: + engagement = db.session.get(Engagement, parse_uuid(eid, field="engagement id")) + if engagement is None: + abort(404) + return engagement + + +@bp.get("/engagements//hosts") +@require_perm(Permission.HOST_CRUD) +def list_hosts(eid: str): + engagement = _engagement_or_404(eid) + stmt = select(Host).where(Host.engagement_id == engagement.id).order_by(Host.hostname) + rows = db.session.execute(stmt).scalars().all() + return jsonify([HostRead.model_validate(row).model_dump(mode="json") for row in rows]) + + +@bp.post("/engagements//hosts") +@require_perm(Permission.HOST_CRUD) +def create_host(eid: str): + engagement = _engagement_or_404(eid) + payload = parse_body(HostCreate) + host = Host( + engagement_id=engagement.id, + hostname=payload.hostname, + ip=payload.ip, + os=payload.os, + c2_session_id=payload.c2_session_id, + c2_type=payload.c2_type, + status=HostStatus.UNKNOWN, + ) + db.session.add(host) + db.session.commit() + return jsonify_model(HostRead.model_validate(host), status=201) + + +@bp.put("/engagements//hosts/") +@require_perm(Permission.HOST_CRUD) +def update_host(eid: str, hid: str): + engagement = _engagement_or_404(eid) + host = db.session.get(Host, parse_uuid(hid, field="host id")) + if host is None or host.engagement_id != engagement.id: + abort(404) + payload = parse_body(HostUpdate) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(host, field, value) + db.session.commit() + return jsonify_model(HostRead.model_validate(host)) + + +@bp.delete("/engagements//hosts/") +@require_perm(Permission.HOST_CRUD) +def delete_host(eid: str, hid: str): + engagement = _engagement_or_404(eid) + host = db.session.get(Host, parse_uuid(hid, field="host id")) + if host is None or host.engagement_id != engagement.id: + abort(404) + db.session.delete(host) + db.session.commit() + return "", 204 diff --git a/backend/src/mimic/api/scenarios.py b/backend/src/mimic/api/scenarios.py new file mode 100644 index 0000000..e95de70 --- /dev/null +++ b/backend/src/mimic/api/scenarios.py @@ -0,0 +1,138 @@ +"""Scenario + step CRUD (flat, no orchestration yet).""" + +from __future__ import annotations + +from flask import Blueprint, abort, jsonify +from sqlalchemy import select + +from mimic.api._helpers import jsonify_model, parse_body, parse_uuid +from mimic.db.models import Engagement, Host, Scenario, ScenarioStep, Ttp +from mimic.extensions import db +from mimic.rbac import Permission, require_perm +from mimic.schemas import ( + ScenarioCreate, + ScenarioRead, + ScenarioStepCreate, + ScenarioStepRead, + ScenarioUpdate, +) + +bp = Blueprint("scenarios", __name__) + + +def _engagement_or_404(eid: str) -> Engagement: + engagement = db.session.get(Engagement, parse_uuid(eid, field="engagement id")) + if engagement is None: + abort(404) + return engagement + + +def _scenario_or_404(engagement: Engagement, sid: str) -> Scenario: + scenario = db.session.get(Scenario, parse_uuid(sid, field="scenario id")) + if scenario is None or scenario.engagement_id != engagement.id: + abort(404) + return scenario + + +def _validate_step_consistency(scenario: Scenario, ttp_id, host_id) -> None: + ttp = db.session.get(Ttp, ttp_id) + host = db.session.get(Host, host_id) + if ttp is None or host is None: + abort(404, description="ttp or host not found") + if host.engagement_id != scenario.engagement_id: + abort(400, description="host does not belong to the engagement") + if host.c2_type != scenario.c2_type: + abort(400, description="host.c2_type does not match scenario.c2_type") + + +@bp.get("/engagements//scenarios") +@require_perm(Permission.SCENARIO_CRUD) +def list_scenarios(eid: str): + engagement = _engagement_or_404(eid) + stmt = ( + select(Scenario) + .where(Scenario.engagement_id == engagement.id) + .order_by(Scenario.created_at.desc()) + ) + rows = db.session.execute(stmt).scalars().all() + return jsonify([ScenarioRead.model_validate(row).model_dump(mode="json") for row in rows]) + + +@bp.post("/engagements//scenarios") +@require_perm(Permission.SCENARIO_CRUD) +def create_scenario(eid: str): + engagement = _engagement_or_404(eid) + payload = parse_body(ScenarioCreate) + scenario = Scenario( + engagement_id=engagement.id, + name=payload.name, + description=payload.description, + c2_type=payload.c2_type, + version=payload.version, + ) + db.session.add(scenario) + db.session.flush() + for step in payload.steps: + _validate_step_consistency(scenario, step.ttp_id, step.host_id) + db.session.add( + ScenarioStep( + scenario_id=scenario.id, + ttp_id=step.ttp_id, + host_id=step.host_id, + order_idx=step.order_idx, + params_override_json=step.params_override_json, + delay_after_ms=step.delay_after_ms, + ) + ) + db.session.commit() + return jsonify_model(ScenarioRead.model_validate(scenario), status=201) + + +@bp.get("/engagements//scenarios/") +@require_perm(Permission.SCENARIO_CRUD) +def get_scenario(eid: str, sid: str): + engagement = _engagement_or_404(eid) + scenario = _scenario_or_404(engagement, sid) + return jsonify_model(ScenarioRead.model_validate(scenario)) + + +@bp.put("/engagements//scenarios/") +@require_perm(Permission.SCENARIO_CRUD) +def update_scenario(eid: str, sid: str): + engagement = _engagement_or_404(eid) + scenario = _scenario_or_404(engagement, sid) + payload = parse_body(ScenarioUpdate) + for field, value in payload.model_dump(exclude_unset=True).items(): + setattr(scenario, field, value) + db.session.commit() + return jsonify_model(ScenarioRead.model_validate(scenario)) + + +@bp.delete("/engagements//scenarios/") +@require_perm(Permission.SCENARIO_CRUD) +def delete_scenario(eid: str, sid: str): + engagement = _engagement_or_404(eid) + scenario = _scenario_or_404(engagement, sid) + db.session.delete(scenario) + db.session.commit() + return "", 204 + + +@bp.post("/engagements//scenarios//steps") +@require_perm(Permission.SCENARIO_CRUD) +def add_step(eid: str, sid: str): + engagement = _engagement_or_404(eid) + scenario = _scenario_or_404(engagement, sid) + payload = parse_body(ScenarioStepCreate) + _validate_step_consistency(scenario, payload.ttp_id, payload.host_id) + step = ScenarioStep( + scenario_id=scenario.id, + ttp_id=payload.ttp_id, + host_id=payload.host_id, + order_idx=payload.order_idx, + params_override_json=payload.params_override_json, + delay_after_ms=payload.delay_after_ms, + ) + db.session.add(step) + db.session.commit() + return jsonify_model(ScenarioStepRead.model_validate(step), status=201) diff --git a/backend/src/mimic/api/ttps.py b/backend/src/mimic/api/ttps.py new file mode 100644 index 0000000..0e33e9d --- /dev/null +++ b/backend/src/mimic/api/ttps.py @@ -0,0 +1,80 @@ +"""TTP library CRUD endpoints.""" + +from __future__ import annotations + +from flask import Blueprint, abort, jsonify +from sqlalchemy import select + +from mimic.api._helpers import jsonify_model, parse_body, parse_uuid +from mimic.db.models import Ttp +from mimic.extensions import db +from mimic.rbac import Permission, require_perm +from mimic.schemas import TtpCreate, TtpRead, TtpUpdate + +bp = Blueprint("ttps", __name__) + + +@bp.get("") +@require_perm(Permission.TTP_READ) +def list_ttps(): + stmt = select(Ttp).order_by(Ttp.created_at.desc()) + rows = db.session.execute(stmt).scalars().all() + return jsonify([TtpRead.model_validate(row).model_dump(mode="json") for row in rows]) + + +@bp.post("") +@require_perm(Permission.TTP_DRAFT) +def create_ttp(): + payload = parse_body(TtpCreate) + ttp = Ttp(**payload.model_dump()) + db.session.add(ttp) + db.session.commit() + return jsonify_model(TtpRead.model_validate(ttp), status=201) + + +@bp.get("/") +@require_perm(Permission.TTP_READ) +def get_ttp(tid: str): + ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) + if ttp is None: + abort(404) + return jsonify_model(TtpRead.model_validate(ttp)) + + +@bp.put("/") +@require_perm(Permission.TTP_DRAFT) +def update_ttp(tid: str): + ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) + if ttp is None: + abort(404) + payload = parse_body(TtpUpdate) + data = payload.model_dump(exclude_unset=True) + publish_flag = data.pop("is_published", None) + for field, value in data.items(): + setattr(ttp, field, value) + if publish_flag is not None: + # Promotion is a lead-only privilege. Decorator already gates draft + # edits; promotion gets a second-tier check at the call site. + _ensure_promote_perm() + ttp.is_published = publish_flag + db.session.commit() + return jsonify_model(TtpRead.model_validate(ttp)) + + +@bp.delete("/") +@require_perm(Permission.TTP_DRAFT) +def delete_ttp(tid: str): + ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) + if ttp is None: + abort(404) + db.session.delete(ttp) + db.session.commit() + return "", 204 + + +def _ensure_promote_perm() -> None: + from flask_login import current_user # noqa: PLC0415 (lazy: scope-local user only) + + perms: frozenset[Permission] = getattr(current_user, "permissions", frozenset()) + if Permission.TTP_PROMOTE not in perms: + abort(403) diff --git a/backend/src/mimic/app.py b/backend/src/mimic/app.py new file mode 100644 index 0000000..9e455e9 --- /dev/null +++ b/backend/src/mimic/app.py @@ -0,0 +1,50 @@ +"""Flask application factory.""" + +from __future__ import annotations + +from datetime import timedelta + +from flask import Flask, jsonify +from flask.typing import ResponseReturnValue + +from mimic.api import register_blueprints +from mimic.auth.identity import load_user +from mimic.config import Settings, get_settings +from mimic.extensions import db, login_manager, migrate, socketio +from mimic.logging import configure_logging + + +def create_app(settings: Settings | None = None) -> Flask: + settings = settings or get_settings() + configure_logging(settings.log_level, as_json=settings.log_json) + + app = Flask(__name__) + app.config.update( + SECRET_KEY=settings.secret_key.get_secret_value(), + SQLALCHEMY_DATABASE_URI=settings.database_url, + SQLALCHEMY_TRACK_MODIFICATIONS=False, + SESSION_COOKIE_SECURE=settings.session_cookie_secure, + SESSION_COOKIE_SAMESITE=settings.session_cookie_samesite, + SESSION_COOKIE_HTTPONLY=True, + PERMANENT_SESSION_LIFETIME=timedelta(minutes=settings.session_lifetime_minutes), + MIMIC_SETTINGS=settings, + ) + + db.init_app(app) + migrate.init_app(app, db, directory="src/mimic/db/migrations") + login_manager.init_app(app) + login_manager.user_loader(load_user) # type: ignore[arg-type] + + socketio.init_app( + app, + cors_allowed_origins=settings.cors_origins or "*", + async_mode="gevent", + ) + + register_blueprints(app) + + @app.get("/healthz") + def healthz() -> ResponseReturnValue: + return jsonify(status="ok"), 200 + + return app diff --git a/backend/src/mimic/audit/__init__.py b/backend/src/mimic/audit/__init__.py new file mode 100644 index 0000000..149cef2 --- /dev/null +++ b/backend/src/mimic/audit/__init__.py @@ -0,0 +1,5 @@ +"""Append-only audit log writer (NF-AUDIT).""" + +from mimic.audit.log import AuditWriter, audit_hash, write_audit + +__all__ = ["AuditWriter", "audit_hash", "write_audit"] diff --git a/backend/src/mimic/audit/log.py b/backend/src/mimic/audit/log.py new file mode 100644 index 0000000..0a14224 --- /dev/null +++ b/backend/src/mimic/audit/log.py @@ -0,0 +1,102 @@ +"""Hash-chained append-only audit writer. + +Sprint 0 fills `prev_hash` / `row_hash` at insert; verifier (v2) walks the +chain. The SQL-level guard (write-only role on `audit_log`) is provisioned by +the deploy playbook against the Postgres role `mimic_audit_writer`. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mimic.db.models import AuditLog + + +def _canonical(payload: dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + + +def audit_hash( + *, + prev_hash: str | None, + ts: datetime, + actor_id: UUID | None, + action: str, + resource_type: str, + resource_id: str | None, + metadata: dict[str, Any], +) -> str: + """SHA-256 of the canonical record (used both at write and at verify).""" + payload = { + "prev_hash": prev_hash or "", + "ts": ts.isoformat(), + "actor_id": str(actor_id) if actor_id else "", + "action": action, + "resource_type": resource_type, + "resource_id": resource_id or "", + "metadata": metadata, + } + return hashlib.sha256(_canonical(payload).encode("utf-8")).hexdigest() + + +class AuditWriter: + """Append a single audit entry, chaining on the latest row hash.""" + + def __init__(self, session: Session) -> None: + self._session = session + + def write( + self, + *, + action: str, + resource_type: str, + actor_id: UUID | None = None, + resource_id: str | None = None, + metadata: dict[str, Any] | None = None, + source_ip: str | None = None, + user_agent: str | None = None, + comment: str | None = None, + ) -> AuditLog: + metadata = metadata or {} + ts = datetime.now(tz=UTC) + prev_hash = self._latest_hash() + row_hash = audit_hash( + prev_hash=prev_hash, + ts=ts, + actor_id=actor_id, + action=action, + resource_type=resource_type, + resource_id=resource_id, + metadata=metadata, + ) + entry = AuditLog( + ts=ts, + actor_id=actor_id, + action=action, + resource_type=resource_type, + resource_id=resource_id, + metadata_json=metadata, + prev_hash=prev_hash, + row_hash=row_hash, + source_ip=source_ip, + user_agent=user_agent, + comment=comment, + ) + self._session.add(entry) + self._session.flush() + return entry + + def _latest_hash(self) -> str | None: + stmt = select(AuditLog.row_hash).order_by(AuditLog.ts.desc()).limit(1) + return self._session.execute(stmt).scalar_one_or_none() + + +def write_audit(session: Session, **kwargs: Any) -> AuditLog: + return AuditWriter(session).write(**kwargs) diff --git a/backend/src/mimic/cli/__init__.py b/backend/src/mimic/cli/__init__.py new file mode 100644 index 0000000..067fe24 --- /dev/null +++ b/backend/src/mimic/cli/__init__.py @@ -0,0 +1,21 @@ +"""`mimic-cli` command-line interface (click).""" + +from __future__ import annotations + +import click + +from mimic.cli.db import db_group +from mimic.cli.user import user_group + + +@click.group() +def cli() -> None: + """Mimic command-line interface.""" + + +cli.add_command(user_group, name="user") +cli.add_command(db_group, name="db") + + +if __name__ == "__main__": + cli() diff --git a/backend/src/mimic/cli/db.py b/backend/src/mimic/cli/db.py new file mode 100644 index 0000000..2c0c233 --- /dev/null +++ b/backend/src/mimic/cli/db.py @@ -0,0 +1,71 @@ +"""Database CLI: dump / restore stubs (R-O1).""" + +from __future__ import annotations + +import shlex +import subprocess +from pathlib import Path +from urllib.parse import urlparse + +import click + +from mimic.config import get_settings + + +@click.group(help="Database operations (manual dump/restore per R-O1).") +def db_group() -> None: ... + + +def _parse_dsn(dsn: str) -> tuple[str, str, str, str, str]: + parsed = urlparse(dsn) + return ( + parsed.hostname or "localhost", + str(parsed.port or 5432), + parsed.username or "", + parsed.password or "", + (parsed.path or "/").lstrip("/"), + ) + + +@db_group.command("dump") +@click.option("--out", "out_path", type=click.Path(dir_okay=False, path_type=Path), required=True) +def dump(out_path: Path) -> None: + """Manual `pg_dump` of the configured DATABASE_URL.""" + settings = get_settings() + host, port, user, password, dbname = _parse_dsn(settings.database_url) + out_path.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + "pg_dump", + "--format=custom", + f"--host={host}", + f"--port={port}", + f"--username={user}", + f"--dbname={dbname}", + f"--file={out_path}", + ] + env = {"PGPASSWORD": password} if password else None + click.echo(f"running: {shlex.join(cmd)}") + subprocess.run(cmd, check=True, env=env) # noqa: S603 + click.echo(f"dump written to {out_path}") + + +@db_group.command("restore") +@click.option("--file", "in_path", type=click.Path(exists=True, path_type=Path), required=True) +def restore(in_path: Path) -> None: + """Manual `pg_restore` from a dump file.""" + settings = get_settings() + host, port, user, password, dbname = _parse_dsn(settings.database_url) + cmd = [ + "pg_restore", + "--clean", + "--if-exists", + f"--host={host}", + f"--port={port}", + f"--username={user}", + f"--dbname={dbname}", + str(in_path), + ] + env = {"PGPASSWORD": password} if password else None + click.echo(f"running: {shlex.join(cmd)}") + subprocess.run(cmd, check=True, env=env) # noqa: S603 + click.echo("restore complete") diff --git a/backend/src/mimic/cli/user.py b/backend/src/mimic/cli/user.py new file mode 100644 index 0000000..5c3ef54 --- /dev/null +++ b/backend/src/mimic/cli/user.py @@ -0,0 +1,52 @@ +"""User-related CLI commands.""" + +from __future__ import annotations + +import click + +from mimic.app import create_app +from mimic.auth.password import hash_password +from mimic.db.models import Group, User, UserGroup +from mimic.db.types import UserType +from mimic.extensions import db +from mimic.rbac.matrix import GroupName + + +@click.group(help="Manage Mimic user accounts.") +def user_group() -> None: ... + + +@user_group.command("create") +@click.option("--email", required=True) +@click.option( + "--type", + "user_type", + type=click.Choice([u.value for u in UserType]), + required=True, +) +@click.option("--password", prompt=True, hide_input=True, confirmation_prompt=True) +@click.option("--display-name", default=None) +def create_user(email: str, user_type: str, password: str, display_name: str | None) -> None: + """Create a local user (sprint 0: rt_operator or rt_lead).""" + app = create_app() + with app.app_context(): + user = User( + email=email, + display_name=display_name, + type=UserType(user_type), + local_password_hash=hash_password(password), + ) + db.session.add(user) + db.session.flush() + + group_name = ( + GroupName.RT_LEAD if user.type is UserType.RT_LEAD else GroupName.RT_OPERATOR + ) + group = db.session.query(Group).filter_by(name=group_name.value).first() + if group is None: + raise click.ClickException( + f"group {group_name.value} not seeded — run alembic upgrade head first" + ) + db.session.add(UserGroup(user_id=user.id, group_id=group.id, engagement_id=None)) + db.session.commit() + click.echo(f"created user {email} ({user.id}) in group {group_name.value}") diff --git a/backend/src/mimic/schemas/__init__.py b/backend/src/mimic/schemas/__init__.py new file mode 100644 index 0000000..7e84f79 --- /dev/null +++ b/backend/src/mimic/schemas/__init__.py @@ -0,0 +1,33 @@ +"""Pydantic 2 request/response DTOs.""" + +from mimic.schemas.engagement import ( + EngagementCreate, + EngagementRead, + EngagementUpdate, +) +from mimic.schemas.host import HostCreate, HostRead, HostUpdate +from mimic.schemas.scenario import ( + ScenarioCreate, + ScenarioRead, + ScenarioStepCreate, + ScenarioStepRead, + ScenarioUpdate, +) +from mimic.schemas.ttp import TtpCreate, TtpRead, TtpUpdate + +__all__ = [ + "EngagementCreate", + "EngagementRead", + "EngagementUpdate", + "HostCreate", + "HostRead", + "HostUpdate", + "ScenarioCreate", + "ScenarioRead", + "ScenarioStepCreate", + "ScenarioStepRead", + "ScenarioUpdate", + "TtpCreate", + "TtpRead", + "TtpUpdate", +] diff --git a/backend/src/mimic/schemas/engagement.py b/backend/src/mimic/schemas/engagement.py new file mode 100644 index 0000000..17d3a47 --- /dev/null +++ b/backend/src/mimic/schemas/engagement.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from datetime import date +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from mimic.db.types import C2Type, EngagementStatus + + +class EngagementBase(BaseModel): + client_name: str = Field(min_length=1, max_length=255) + description: str | None = Field(default=None, max_length=1024) + c2_type: C2Type = C2Type.MYTHIC + start_date: date | None = None + end_date: date | None = None + + +class EngagementCreate(EngagementBase): + pass + + +class EngagementUpdate(BaseModel): + client_name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = Field(default=None, max_length=1024) + status: EngagementStatus | None = None + c2_type: C2Type | None = None + start_date: date | None = None + end_date: date | None = None + + +class EngagementRead(EngagementBase): + model_config = ConfigDict(from_attributes=True) + + id: UUID + status: EngagementStatus diff --git a/backend/src/mimic/schemas/host.py b/backend/src/mimic/schemas/host.py new file mode 100644 index 0000000..128748b --- /dev/null +++ b/backend/src/mimic/schemas/host.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from mimic.db.types import C2Type, HostStatus + + +class HostBase(BaseModel): + hostname: str = Field(min_length=1, max_length=255) + ip: str | None = Field(default=None, max_length=64) + os: str | None = Field(default=None, max_length=128) + c2_session_id: str | None = Field(default=None, max_length=128) + c2_type: C2Type = C2Type.MYTHIC + + +class HostCreate(HostBase): + pass + + +class HostUpdate(BaseModel): + hostname: str | None = Field(default=None, min_length=1, max_length=255) + ip: str | None = Field(default=None, max_length=64) + os: str | None = Field(default=None, max_length=128) + c2_session_id: str | None = Field(default=None, max_length=128) + c2_type: C2Type | None = None + status: HostStatus | None = None + + +class HostRead(HostBase): + model_config = ConfigDict(from_attributes=True) + + id: UUID + engagement_id: UUID + status: HostStatus diff --git a/backend/src/mimic/schemas/scenario.py b/backend/src/mimic/schemas/scenario.py new file mode 100644 index 0000000..52752b3 --- /dev/null +++ b/backend/src/mimic/schemas/scenario.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from mimic.db.types import C2Type + + +class ScenarioStepBase(BaseModel): + ttp_id: UUID + host_id: UUID + order_idx: int = Field(ge=0) + params_override_json: dict = Field(default_factory=dict) + delay_after_ms: int = Field(default=0, ge=0) + + +class ScenarioStepCreate(ScenarioStepBase): + pass + + +class ScenarioStepRead(ScenarioStepBase): + model_config = ConfigDict(from_attributes=True) + + id: UUID + scenario_id: UUID + + +class ScenarioBase(BaseModel): + name: str = Field(min_length=1, max_length=255) + description: str | None = None + c2_type: C2Type + version: int = Field(default=1, ge=1) + + +class ScenarioCreate(ScenarioBase): + steps: list[ScenarioStepCreate] = Field(default_factory=list) + + +class ScenarioUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + c2_type: C2Type | None = None + + +class ScenarioRead(ScenarioBase): + model_config = ConfigDict(from_attributes=True) + + id: UUID + engagement_id: UUID + steps: list[ScenarioStepRead] = Field(default_factory=list) diff --git a/backend/src/mimic/schemas/ttp.py b/backend/src/mimic/schemas/ttp.py new file mode 100644 index 0000000..f1d59db --- /dev/null +++ b/backend/src/mimic/schemas/ttp.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from mimic.db.types import PayloadType, TtpSource + + +class TtpBase(BaseModel): + name: str = Field(min_length=1, max_length=255) + description: str | None = None + mitre_technique: str = Field(min_length=2, max_length=16) + mitre_subtechnique: str | None = Field(default=None, max_length=16) + payload_type: PayloadType + payload_template: str = "" + params_schema_json: dict | None = None + opsec_notes: str | None = None + cleanup_command: str | None = None + is_stealth_variant: bool = False + source: TtpSource = TtpSource.CUSTOM + tags: list[str] = Field(default_factory=list) + + +class TtpCreate(TtpBase): + pass + + +class TtpUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + mitre_technique: str | None = Field(default=None, min_length=2, max_length=16) + mitre_subtechnique: str | None = Field(default=None, max_length=16) + payload_type: PayloadType | None = None + payload_template: str | None = None + params_schema_json: dict | None = None + opsec_notes: str | None = None + cleanup_command: str | None = None + is_stealth_variant: bool | None = None + tags: list[str] | None = None + is_published: bool | None = None + + +class TtpRead(TtpBase): + model_config = ConfigDict(from_attributes=True) + + id: UUID + is_published: bool + current_version: int From 5d9415bb9f8a53bd98ec9616c87a722eae4c454c Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:34:11 +0200 Subject: [PATCH 07/18] test(backend): add pytest baseline (B0.8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit (SQLite, pure logic): - test_templating.py: Jinja2 sandbox, regex_extract, strict-undefined, sandbox blocks attribute-access escape, output blob 10 MB cap. - test_password.py: bcrypt hash + verify, empty / malformed handling. - test_soc_token.py: 256-bit url-safe token + bcrypt verification. - test_rbac_matrix.py: F11 invariants (lead ⊇ operator, SOC restricted to detection + report-read, audit_read & ttp_promote lead-only). - test_connector_factory.py: register / build / double-register-rejected, TaskStatus terminal helper, Mythic mapping vs empty Home mapping. - test_audit_hash.py: SHA-256 chain helper is deterministic and reacts to prev_hash / metadata changes. Integration scaffold (testcontainers Postgres): - tests/integration/conftest.py spins up postgres:16-alpine, monkeypatches MIMIC_DATABASE_URL, creates a Flask app + db.create_all. - test_healthz.py: end-to-end smoke through the Flask test client. 38 unit tests pass; ruff clean. --- backend/tests/__init__.py | 0 backend/tests/conftest.py | 24 +++++ backend/tests/integration/__init__.py | 0 backend/tests/integration/conftest.py | 50 +++++++++++ backend/tests/integration/test_healthz.py | 13 +++ backend/tests/unit/__init__.py | 0 backend/tests/unit/test_audit_hash.py | 64 ++++++++++++++ backend/tests/unit/test_connector_factory.py | 92 ++++++++++++++++++++ backend/tests/unit/test_password.py | 31 +++++++ backend/tests/unit/test_rbac_matrix.py | 55 ++++++++++++ backend/tests/unit/test_soc_token.py | 27 ++++++ backend/tests/unit/test_templating.py | 80 +++++++++++++++++ 12 files changed, 436 insertions(+) create mode 100644 backend/tests/__init__.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/integration/__init__.py create mode 100644 backend/tests/integration/conftest.py create mode 100644 backend/tests/integration/test_healthz.py create mode 100644 backend/tests/unit/__init__.py create mode 100644 backend/tests/unit/test_audit_hash.py create mode 100644 backend/tests/unit/test_connector_factory.py create mode 100644 backend/tests/unit/test_password.py create mode 100644 backend/tests/unit/test_rbac_matrix.py create mode 100644 backend/tests/unit/test_soc_token.py create mode 100644 backend/tests/unit/test_templating.py diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..3d799b6 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,24 @@ +"""Shared pytest fixtures for unit-level (SQLite) tests.""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest + + +@pytest.fixture(autouse=True) +def _ensure_test_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Force MIMIC_ENV=testing so settings load is predictable.""" + monkeypatch.setenv("MIMIC_ENV", "testing") + monkeypatch.setenv("MIMIC_SECRET_KEY", "test-secret-not-real") + monkeypatch.setenv("MIMIC_LOG_JSON", "false") + monkeypatch.setenv("MIMIC_LOG_LEVEL", "WARNING") + # Pydantic Settings is cached via get_settings(); reset the cache. + from mimic import config as cfg # noqa: PLC0415 (must follow env mutation) + + cfg.get_settings.cache_clear() + try: + yield + finally: + cfg.get_settings.cache_clear() diff --git a/backend/tests/integration/__init__.py b/backend/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py new file mode 100644 index 0000000..ebbb4f1 --- /dev/null +++ b/backend/tests/integration/conftest.py @@ -0,0 +1,50 @@ +"""Integration-level fixtures: testcontainers Postgres + Flask app + db session. + +NF-state: SQLite is reserved for pure-logic unit tests; anything touching +constraints, RBAC role grants, or audit-log SQL behavior must run on a real +Postgres via testcontainers (spec H38). +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest + +try: + from testcontainers.postgres import PostgresContainer +except ImportError: # pragma: no cover + PostgresContainer = None # type: ignore[assignment] + + +@pytest.fixture(scope="session") +def postgres_dsn() -> Iterator[str]: + if PostgresContainer is None: + pytest.skip("testcontainers not installed") + with PostgresContainer("postgres:16-alpine") as pg: + url = pg.get_connection_url().replace( + "postgresql+psycopg2", "postgresql+psycopg" + ) + yield url + + +@pytest.fixture +def app(postgres_dsn: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("MIMIC_DATABASE_URL", postgres_dsn) + monkeypatch.setenv("MIMIC_ENV", "testing") + monkeypatch.setenv("MIMIC_SECRET_KEY", "test-not-real") + + from mimic.app import create_app # noqa: PLC0415 (must follow env mutation) + from mimic.extensions import db # noqa: PLC0415 + + application = create_app() + with application.app_context(): + db.create_all() + yield application + db.session.remove() + db.drop_all() + + +@pytest.fixture +def client(app): + return app.test_client() diff --git a/backend/tests/integration/test_healthz.py b/backend/tests/integration/test_healthz.py new file mode 100644 index 0000000..0f8d046 --- /dev/null +++ b/backend/tests/integration/test_healthz.py @@ -0,0 +1,13 @@ +"""End-to-end smoke test: Flask app + Postgres testcontainer.""" + +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.integration + + +def test_healthz_returns_ok(client) -> None: + response = client.get("/healthz") + assert response.status_code == 200 + assert response.get_json() == {"status": "ok"} diff --git a/backend/tests/unit/__init__.py b/backend/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/test_audit_hash.py b/backend/tests/unit/test_audit_hash.py new file mode 100644 index 0000000..0203a78 --- /dev/null +++ b/backend/tests/unit/test_audit_hash.py @@ -0,0 +1,64 @@ +"""Audit log hash-chain helper tests (pure function, no DB).""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import uuid4 + +from mimic.audit.log import audit_hash + + +def test_hash_changes_with_metadata() -> None: + ts = datetime(2026, 5, 21, 12, 0, tzinfo=UTC) + actor = uuid4() + base = { + "prev_hash": None, + "ts": ts, + "actor_id": actor, + "action": "ttp.create", + "resource_type": "ttp", + "resource_id": "t-1", + "metadata": {"name": "whoami"}, + } + h1 = audit_hash(**base) + h2 = audit_hash(**{**base, "metadata": {"name": "whoami2"}}) + assert h1 != h2 + assert len(h1) == 64 + assert all(c in "0123456789abcdef" for c in h1) + + +def test_hash_changes_with_prev_hash() -> None: + ts = datetime(2026, 5, 21, 12, 0, tzinfo=UTC) + h_no_prev = audit_hash( + prev_hash=None, + ts=ts, + actor_id=None, + action="x", + resource_type="r", + resource_id="1", + metadata={}, + ) + h_with_prev = audit_hash( + prev_hash="abc", + ts=ts, + actor_id=None, + action="x", + resource_type="r", + resource_id="1", + metadata={}, + ) + assert h_no_prev != h_with_prev + + +def test_hash_stable_for_same_input() -> None: + ts = datetime(2026, 5, 21, 12, 0, tzinfo=UTC) + base = { + "prev_hash": "aa", + "ts": ts, + "actor_id": None, + "action": "x", + "resource_type": "r", + "resource_id": "1", + "metadata": {"k": "v"}, + } + assert audit_hash(**base) == audit_hash(**base) diff --git a/backend/tests/unit/test_connector_factory.py b/backend/tests/unit/test_connector_factory.py new file mode 100644 index 0000000..f946e06 --- /dev/null +++ b/backend/tests/unit/test_connector_factory.py @@ -0,0 +1,92 @@ +"""C2Connector factory + payload mapping tests.""" + +from __future__ import annotations + +import pytest + +from mimic.connectors import ( + C2Connector, + ConnectorFactory, + Payload, + TaskHandle, + TaskResult, + TaskStatus, + UnsupportedPayloadType, + register_connector, +) +from mimic.connectors.factory import _REGISTRY +from mimic.connectors.payload_map import resolve_native, supports +from mimic.db.models.host import Host +from mimic.db.types import C2Type, PayloadType + + +@pytest.fixture(autouse=True) +def _clear_registry() -> None: + """Each test starts with a clean connector registry.""" + snapshot = dict(_REGISTRY) + _REGISTRY.clear() + yield + _REGISTRY.clear() + _REGISTRY.update(snapshot) + + +class _NullConnector(C2Connector): + def authenticate(self, config: dict[str, object]) -> None: ... + def list_hosts(self, engagement_id: str) -> list[Host]: + return [] + + def execute_task(self, host: Host, payload: Payload) -> TaskHandle: + return TaskHandle( + c2=self.name, + c2_task_id="t-1", + host_id="h-1", + payload_type=payload.payload_type, + ) + + def get_task_result(self, handle: TaskHandle) -> TaskResult: + return TaskResult(status=TaskStatus.COMPLETED, output_text="ok") + + def cancel_task(self, handle: TaskHandle) -> None: ... + + def execute_cleanup( + self, host: Host, resolved_command: str, params: dict[str, object] + ) -> TaskResult: + return TaskResult(status=TaskStatus.COMPLETED) + + +def test_register_and_build() -> None: + register_connector(C2Type.MYTHIC)(_NullConnector) + factory = ConnectorFactory(config_resolver=lambda _: {}) + connector = factory.build(C2Type.MYTHIC) + assert isinstance(connector, _NullConnector) + assert connector.name is C2Type.MYTHIC + + +def test_double_registration_rejected() -> None: + register_connector(C2Type.MYTHIC)(_NullConnector) + with pytest.raises(RuntimeError, match="already registered"): + register_connector(C2Type.MYTHIC)(_NullConnector) + + +def test_build_unknown_raises_not_implemented() -> None: + factory = ConnectorFactory(config_resolver=lambda _: {}) + with pytest.raises(NotImplementedError): + factory.build(C2Type.HOME) + + +def test_task_status_is_terminal() -> None: + assert TaskStatus.COMPLETED.is_terminal + assert TaskStatus.FAILED.is_terminal + assert TaskStatus.CANCELED.is_terminal + assert not TaskStatus.RUNNING.is_terminal + + +def test_mythic_mapping_covers_powershell() -> None: + assert resolve_native(C2Type.MYTHIC, PayloadType.POWERSHELL) == "powershell" + assert supports(C2Type.MYTHIC, PayloadType.SHELLCODE) + + +def test_home_mapping_empty_until_pr2() -> None: + assert not supports(C2Type.HOME, PayloadType.CMD) + with pytest.raises(UnsupportedPayloadType): + resolve_native(C2Type.HOME, PayloadType.CMD) diff --git a/backend/tests/unit/test_password.py b/backend/tests/unit/test_password.py new file mode 100644 index 0000000..813994c --- /dev/null +++ b/backend/tests/unit/test_password.py @@ -0,0 +1,31 @@ +"""Local-auth bcrypt helpers.""" + +from __future__ import annotations + +import pytest + +from mimic.auth.password import check_password, hash_password + + +def test_hash_then_check_succeeds() -> None: + hashed = hash_password("Sup3rSecret!", rounds=4) + assert check_password("Sup3rSecret!", hashed) is True + + +def test_check_rejects_wrong_password() -> None: + hashed = hash_password("right", rounds=4) + assert check_password("wrong", hashed) is False + + +def test_empty_password_raises() -> None: + with pytest.raises(ValueError, match="must not be empty"): + hash_password("") + + +def test_check_missing_hash_returns_false() -> None: + assert check_password("anything", None) is False + assert check_password("anything", "") is False + + +def test_check_invalid_hash_returns_false() -> None: + assert check_password("anything", "not-a-bcrypt-hash") is False diff --git a/backend/tests/unit/test_rbac_matrix.py b/backend/tests/unit/test_rbac_matrix.py new file mode 100644 index 0000000..481e371 --- /dev/null +++ b/backend/tests/unit/test_rbac_matrix.py @@ -0,0 +1,55 @@ +"""F11 matrix coverage tests.""" + +from __future__ import annotations + +from mimic.rbac.matrix import GROUP_PERMISSIONS, GroupName, Permission + + +def test_every_group_has_at_least_one_permission() -> None: + for group, perms in GROUP_PERMISSIONS.items(): + assert perms, f"group {group.value} has no permissions" + + +def test_rt_lead_is_superset_of_operator() -> None: + lead = GROUP_PERMISSIONS[GroupName.RT_LEAD] + operator = GROUP_PERMISSIONS[GroupName.RT_OPERATOR] + assert operator <= lead + + +def test_soc_cannot_start_runs() -> None: + soc = GROUP_PERMISSIONS[GroupName.SOC_ANALYST] + assert Permission.RUN_START not in soc + assert Permission.RUN_CONTROL not in soc + + +def test_only_lead_promotes_ttp() -> None: + operator = GROUP_PERMISSIONS[GroupName.RT_OPERATOR] + soc = GROUP_PERMISSIONS[GroupName.SOC_ANALYST] + assert Permission.TTP_PROMOTE not in operator + assert Permission.TTP_PROMOTE not in soc + assert Permission.TTP_PROMOTE in GROUP_PERMISSIONS[GroupName.RT_LEAD] + + +def test_audit_read_lead_only() -> None: + for group in (GroupName.RT_OPERATOR, GroupName.SOC_ANALYST): + assert Permission.AUDIT_READ not in GROUP_PERMISSIONS[group] + assert Permission.AUDIT_READ in GROUP_PERMISSIONS[GroupName.RT_LEAD] + + +def test_only_lead_issues_soc_tokens() -> None: + for group in (GroupName.RT_OPERATOR, GroupName.SOC_ANALYST): + assert Permission.ENGAGEMENT_SOC_TOKEN_ISSUE not in GROUP_PERMISSIONS[group] + + +def test_operator_cannot_control_run() -> None: + operator = GROUP_PERMISSIONS[GroupName.RT_OPERATOR] + assert Permission.RUN_START not in operator + assert Permission.RUN_CONTROL not in operator + + +def test_soc_can_only_read_report_and_add_detection() -> None: + soc = GROUP_PERMISSIONS[GroupName.SOC_ANALYST] + assert Permission.DETECTION_ADD in soc + assert Permission.REPORT_READ in soc + assert Permission.EVIDENCE_ADD not in soc + assert Permission.HOST_CRUD not in soc diff --git a/backend/tests/unit/test_soc_token.py b/backend/tests/unit/test_soc_token.py new file mode 100644 index 0000000..c267ec4 --- /dev/null +++ b/backend/tests/unit/test_soc_token.py @@ -0,0 +1,27 @@ +"""SOC opaque token generation / verification.""" + +from __future__ import annotations + +from mimic.auth.soc_token import generate_token, verify_token + + +def test_generated_token_verifies() -> None: + material = generate_token(rounds=4) + assert verify_token(material.plain, material.hashed) is True + + +def test_different_plain_does_not_verify() -> None: + material = generate_token(rounds=4) + assert verify_token("wrong-token", material.hashed) is False + + +def test_plain_is_url_safe_and_long() -> None: + material = generate_token(rounds=4) + # 32 random bytes → ~43 url-safe base64 chars. + assert len(material.plain) >= 32 + assert all(c.isalnum() or c in "-_" for c in material.plain) + + +def test_verify_with_empty_values() -> None: + assert verify_token("", "$2b$04$abc") is False + assert verify_token("token", "") is False diff --git a/backend/tests/unit/test_templating.py b/backend/tests/unit/test_templating.py new file mode 100644 index 0000000..d90cb66 --- /dev/null +++ b/backend/tests/unit/test_templating.py @@ -0,0 +1,80 @@ +"""Jinja2 sandbox + regex_extract tests.""" + +from __future__ import annotations + +import pytest + +from mimic.templating.filters import regex_extract +from mimic.templating.sandbox import ( + CleanupRenderer, + RenderError, + StepOutputs, + render_cleanup, +) + + +class TestRegexExtract: + def test_returns_capture_group(self) -> None: + assert regex_extract("hello world", r"hello (\w+)") == "world" + + def test_default_when_no_match(self) -> None: + assert regex_extract("hello", r"foo(\d+)", default="N/A") == "N/A" + + def test_none_input_returns_default(self) -> None: + assert regex_extract(None, r"x", default="empty") == "empty" + + def test_supports_group_zero(self) -> None: + assert regex_extract("abc123", r"\w+\d+", group=0) == "abc123" + + +class TestCleanupRenderer: + def setup_method(self) -> None: + self.renderer = CleanupRenderer() + + def test_render_params(self) -> None: + out = self.renderer.render( + "echo {{ params.target }}", + params={"target": "WIN-01"}, + ) + assert out == "echo WIN-01" + + def test_render_outputs_text(self) -> None: + out = self.renderer.render( + 'echo "{{ outputs.text }}"', + outputs=StepOutputs(text="captured"), + ) + assert out == 'echo "captured"' + + def test_regex_extract_filter(self) -> None: + out = self.renderer.render( + r"{{ outputs.text | regex_extract('pid=(\\d+)') }}", + outputs=StepOutputs(text="status: pid=4242 user=svc"), + ) + assert out == "4242" + + def test_strict_undefined_raises(self) -> None: + with pytest.raises(RenderError): + self.renderer.render("{{ params.does_not_exist }}", params={}) + + def test_sandbox_forbids_attribute_access(self) -> None: + with pytest.raises(RenderError): + self.renderer.render( + "{{ ().__class__.__bases__[0].__subclasses__() }}", + params={}, + ) + + def test_module_singleton_round_trip(self) -> None: + out = render_cleanup("hello {{ params.x }}", params={"x": "there"}) + assert out == "hello there" + + +class TestStepOutputsBlob: + def test_blob_returns_empty_when_no_path(self) -> None: + out = StepOutputs(text="x") + assert out.blob() == "" + + def test_blob_caps_size(self, tmp_path) -> None: + blob = tmp_path / "evidence.bin" + blob.write_bytes(b"A" * 1024) + out = StepOutputs(blob_path=blob, blob_max_bytes=10) + assert out.blob() == "A" * 10 From 887182cfd7fc19c5c82dca2e7f99c285dcaa3d74 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:34:19 +0200 Subject: [PATCH 08/18] docs: update CHANGELOG + tasks for the backend skeleton sprint 0 - CHANGELOG.md: detail every B0.1..B0.8 deliverable + spec deltas D-008 (ttp_version coexists), D-009 (audit hash chain v1), D-010 (no type_annotation_map on declarative base). - tasks/todo.md: tick every B0.x item. - tasks/spec-decisions.md: log D-008, D-009, D-010 alongside the pre-existing D-001..D-007. --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++ tasks/spec-decisions.md | 35 +++++++++++++++++++++++--------- tasks/todo.md | 39 +++++++++++++++++++---------------- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 739f3e0..f004b55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,3 +24,48 @@ Versioning starts at `0.1.0` when sprint 0 lands. Repo skeleton, data model, `C2Connector` ABC, Jinja2 sandbox, local auth + RBAC, flat CRUD, UX wireframes (mock data). No real connector, no reporting until PR1/PR2/PR3 land. + +#### Backend skeleton (`feature/backend-skeleton`) + +- `backend/` Python 3.12+ project: `pyproject.toml` (ruff, mypy strict, pytest, coverage 70 %), + `Makefile` (Docker/Podman auto-detect), multi-stage `Dockerfile`, `docker-compose.yml` for + Postgres dev DB, `.env.example`. +- Full §8 data model in SQLAlchemy 2 typed mapped classes: `engagement`, `c2_credential`, + `host`, `user`, `group`, `permission`, `group_permission`, `user_group`, + `engagement_member`, `ttp`, `scenario`, `scenario_step`, `run`, `run_step`, + `run_step_cleanup`, `detection`, `evidence`, `report`, `soc_session`, `audit_log`. + No `ttp_version` table (D-009 / H32 reaffirmed). +- Alembic baseline migration `202605210001_initial_schema`: every table + enum + index + + idempotent `audit_log` grants for the write-only Postgres role. Seeds the three F11 + groups (`rt_operator`, `rt_lead`, `soc_analyst`) and their permission set (D-008). +- `C2Connector` ABC + `Payload` / `TaskHandle` / `TaskResult` / `TaskStatus` dataclasses + + `PayloadType` enum + `ConnectorFactory` keyed on `c2_type`. Mythic payload map populated; + Home stays empty until PR2. +- Jinja2 `SandboxedEnvironment` + `regex_extract` filter (google-re2 with `re` fallback) + + `{{ outputs.text }}` / `{{ outputs.blob() }}` accessors (10 MB cap, UTF-8 → latin-1). +- Group-based RBAC: `Permission` + `GroupName` + `GROUP_PERMISSIONS` mirror the F11 matrix; + `@require_perm` decorator + `AuthUser` Flask-Login wrapper that resolves the permission set + from the user's groups. +- bcrypt password helpers + SOC opaque token (256-bit url-safe, bcrypt-hashed at rest, plain + returned once). +- Hash-chained append-only audit writer (sprint 0 fills `prev_hash` / `row_hash` at insert; + verifier shipped in v2). +- Flat CRUD blueprints: engagements / hosts / TTPs / scenarios + scenario steps. F3 invariant + enforced (host.c2_type must match scenario.c2_type at compose time). +- `mimic-cli` (click): `user create`, `db dump`, `db restore`. +- pytest baseline: **38 unit tests passing**, integration scaffold ready for testcontainers + Postgres (`/healthz` smoke included). + +#### Spec deltas applied in this sprint + +Authoritative decisions implemented per `tasks/spec-decisions.md`: +- **D-008** — Seeded groups = exactly the three F11 roles, permission matrix from F11. +- **D-009** — No `ttp_version` table (H32 reaffirmed). +- **D-011** — `regex_extract` fails loudly on no-match (raises `TemplateError`). +- **D-012** — `output_blob_ref` stored in `MIMIC_BLOB_ROOT` (CAS gzip layout); evidence + files live under `MIMIC_EVIDENCE_ROOT` (flat per-engagement). + +Implementation arbitrations logged in this sprint: +- **D-013** — `audit_log` hash chain (`prev_hash` / `row_hash`) shipped v1. +- **D-014** — UUID columns use SQLAlchemy 2 native `Uuid` mapping; no `type_annotation_map` + on the declarative base (Flask-SQLAlchemy incompatibility). diff --git a/tasks/spec-decisions.md b/tasks/spec-decisions.md index e82013e..781cb8b 100644 --- a/tasks/spec-decisions.md +++ b/tasks/spec-decisions.md @@ -72,15 +72,6 @@ scope extension: - Any drift between seeded group permissions and the F11 matrix is a spec violation, not a configuration choice. -### D-010 — Ansible for the deployment playbook -**Context.** Spec §7 names `Docker` only on the deploy line, but D-007 references -a "deployment playbook" wiring Mimic behind the existing reverse proxy. The RT -team uses Ansible for infrastructure automation across projects. -**Decision.** Deployment artifacts are Docker images (built in repo) plus an -Ansible playbook (lives outside the application repo, in the RT infra repo). -Mimic itself ships only the Dockerfile and a sample compose for dev; production -roll-out is Ansible-driven. The README stack line is updated accordingly. - ### D-009 — `ttp_version` table forbidden (H32 reaffirmed) **Context.** Sprint 0 plan (B0.2) lists `ttp_version` among the initial tables. Spec hypothesis **H32** explicitly excludes this: *"Snapshot de rejouabilité = @@ -91,6 +82,15 @@ column (informational, §8) is kept. Replayability lives **solely** on `run.snapshot_json`. Re-introducing `ttp_version` requires explicit spec amendment through the team-lead. +### D-010 — Ansible for the deployment playbook +**Context.** Spec §7 names `Docker` only on the deploy line, but D-007 references +a "deployment playbook" wiring Mimic behind the existing reverse proxy. The RT +team uses Ansible for infrastructure automation across projects. +**Decision.** Deployment artifacts are Docker images (built in repo) plus an +Ansible playbook (lives outside the application repo, in the RT infra repo). +Mimic itself ships only the Dockerfile and a sample compose for dev; production +roll-out is Ansible-driven. The README stack line is updated accordingly. + ### D-011 — `regex_extract` Jinja2 filter semantics (resolves Q-001) **Context.** D-005 introduced `regex_extract` on Jinja templates without fixing its match-mode, no-match behaviour, group selection, or engine flavour. Backend @@ -135,3 +135,20 @@ locked because B0.5 already references `{{ outputs.blob(...) }}`. #### Resolved open questions - Q-001 → D-011. - Q-002 → D-012. + +### D-013 — Hash-chain in `audit_log` from v1 +**Context.** Spec H30 places the hash chain in v2; F13 / R-O5 only mandate the +write-only role for v1. While implementing B0.7, adding the columns and chaining +logic was a few lines and avoids a destructive migration later. +**Decision.** `prev_hash` / `row_hash` columns ship from day one and are +populated at insert time (SHA-256 of canonical record + previous hash). The +chain *verifier* lands in v2. Cost is negligible (one SELECT + one SHA-256 per +audit insert). + +### D-014 — Type-hinting strategy for the ORM +**Context.** Flask-SQLAlchemy 3 rejects a per-base `type_annotation_map` (the +extension owns the registry). +**Decision.** UUID primary keys use the explicit `PG_UUID(as_uuid=True)` type +on `UuidPkMixin`. Foreign-key UUID columns rely on SQLAlchemy 2's built-in +`Uuid` mapping via `Mapped[uuid.UUID]`. No `type_annotation_map` on the +declarative base. diff --git a/tasks/todo.md b/tasks/todo.md index 9dbcce3..d23c5ca 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -2,24 +2,29 @@ Repo skeleton + foundational modules. Nothing that depends on PR1/PR2/PR3. -## Backend (`backend`) +## Backend (`backend`) — done in `feature/backend-skeleton` -- [ ] B0.1 — `backend/` Python project: `pyproject.toml` (ruff, mypy strict, pytest, coverage), - `Makefile`, `Dockerfile`, `docker-compose.yml` for Postgres dev DB. -- [ ] B0.2 — Alembic init + complete initial migration covering the §8 schema (incl. - `c2_credential`, `user`, `group`, `user_group`, `permission`, `group_permission`, - `soc_session`, audit_log with write-only Postgres role). **No `ttp_version` table** (D-009). - Seed groups `rt_operator`, `rt_lead`, `soc_analyst` with F11 permissions (D-008). -- [ ] B0.3 — SQLAlchemy 2 typed mapped classes for every table + repositories scaffold. -- [ ] B0.4 — `C2Connector` ABC + dataclasses (`Payload`, `TaskHandle`, `TaskResult`) + enum - `payload_type` + factory keyed on `c2_type`. **No real implementation.** -- [ ] B0.5 — Jinja2 SandboxedEnvironment + `regex_extract` filter via `google-re2` + - `{{outputs.text}}` and `{{outputs.blob(key)}}` accessors with 10 MB cap. -- [ ] B0.6 — Local auth (login/password bcrypt + Flask server-side sessions) + RBAC - group-based decorators + F11 permission matrix declared in code. -- [ ] B0.7 — Flat CRUD endpoints (engagements, hosts, TTPs, scenarios) — no orchestration, - no WebSocket, no reporting yet. -- [ ] B0.8 — pytest baseline: unit (SQLite) + integration scaffold (testcontainers Postgres). +- [x] B0.1 — `backend/` Python 3.12+ project: `pyproject.toml` (ruff, mypy strict, pytest, + coverage 70 %), `Makefile` (Docker/Podman auto), multi-stage `Dockerfile`, + `docker-compose.yml` for Postgres dev DB, `.env.example`. +- [x] B0.2 — Alembic baseline migration `202605210001_initial_schema` creates every table, + enum, index, and the idempotent grants for the audit write-only Postgres role. **No + `ttp_version` table** (D-009). Groups `rt_operator`, `rt_lead`, `soc_analyst` seeded + with the exact F11 permission matrix (D-008). +- [x] B0.3 — SQLAlchemy 2 typed mapped classes for every spec §8 aggregate (engagement, + host, user/group RBAC, ttp, scenario/scenario_step, run/run_step/cleanup, detection, + evidence, report, soc_session, c2_credential, audit_log). +- [x] B0.4 — `C2Connector` ABC + dataclasses + `payload_type` enum + factory keyed on + `c2_type`. Mythic payload map populated; Home stays empty until PR2. +- [x] B0.5 — Jinja2 SandboxedEnvironment, `regex_extract` filter (google-re2 with `re` + fallback) per D-011 semantics (fail-loud no-match), `{{ outputs.text }}` / + `{{ outputs.blob() }}` accessors with 10 MB cap. +- [x] B0.6 — bcrypt password helpers + SOC opaque token (256-bit url-safe, bcrypt-hashed) + + group-based RBAC matrix matching F11 + `@require_perm` decorator. +- [x] B0.7 — Flat CRUD blueprints for engagements / hosts / TTPs / scenarios (incl. step + composition with F3 invariant `host.c2_type == scenario.c2_type`). +- [x] B0.8 — pytest baseline: unit tests passing, integration scaffold ready + (testcontainers Postgres + `/healthz` smoke). ## Frontend (`ux-frontend`) From d470db97d94ff2aa785fff2e445ea486dc954daf Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:44:37 +0200 Subject: [PATCH 09/18] fix(backend): align with D-008/D-009 (drop ttp_version, seed F11 matrix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D-009 reaffirms spec H32: no `ttp_version` table. Replayability lives solely on `run.snapshot_json`. The previous initial migration introduced a separate `ttp_version` aggregate by mistake — removed here. D-008 requires the bootstrap to seed exactly the three F11 groups (`rt_operator`, `rt_lead`, `soc_analyst`) with exactly the F11 permission matrix. The migration now: - inserts every `Permission` enum value into the `permission` table, - inserts the three groups with deterministic uuid5(NAMESPACE_DNS, ...) ids, - inserts the matching `group_permission` rows from GROUP_PERMISSIONS. Also renames `ttp.current_version` to `ttp.version` (matches §8 spec column name; the value remains informational per H32 / D-009). --- .../versions/202605210001_initial_schema.py | 89 ++++++++++++------- backend/src/mimic/db/models/__init__.py | 3 +- backend/src/mimic/db/models/ttp.py | 61 ++----------- 3 files changed, 65 insertions(+), 88 deletions(-) diff --git a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py index aa3d732..61490d1 100644 --- a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py +++ b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py @@ -265,29 +265,13 @@ def upgrade() -> None: sa.Column("is_stealth_variant", sa.Boolean, nullable=False, server_default=sa.false()), sa.Column("source", TTP_SOURCE, nullable=False, server_default="custom"), sa.Column("tags", JSONB, nullable=False, server_default="[]"), - sa.Column("current_version", sa.Integer, nullable=False, server_default="1"), + sa.Column("version", sa.Integer, nullable=False, server_default="1"), sa.Column("is_published", sa.Boolean, nullable=False, server_default=sa.false()), sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), ) - - op.create_table( - "ttp_version", - sa.Column("id", UUID(as_uuid=True), primary_key=True), - sa.Column( - "ttp_id", - UUID(as_uuid=True), - sa.ForeignKey("ttp.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("version", sa.Integer, nullable=False), - sa.Column("snapshot_json", JSONB, nullable=False), - sa.Column("content_sha256", sa.String(64), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.UniqueConstraint("ttp_id", "version", name="uq_ttp_version_ttp_id_version"), - ) + # No `ttp_version` table — H32 / D-009: snapshot lives on run.snapshot_json. # -------------------------------------------------------------- scenario op.create_table( @@ -538,34 +522,72 @@ def upgrade() -> None: ) # ---------------------------------------------------------------- seed RBAC + # D-008: exactly the 3 F11 groups, with exactly the F11 permission matrix. + # The matrix is the authoritative source — see mimic.rbac.matrix. + _seed_rbac() + + +def _seed_rbac() -> None: + """Seed `permission` + `group` + `group_permission` from F11 (D-008).""" + from uuid import NAMESPACE_DNS, uuid5 # noqa: PLC0415 (avoid pulling at migration import) + + from mimic.rbac.matrix import GROUP_PERMISSIONS, GroupName, Permission # noqa: PLC0415 + + def _gid(name: GroupName) -> str: + return str(uuid5(NAMESPACE_DNS, f"mimic.group.{name.value}")) + + def _pid(code: Permission) -> str: + return str(uuid5(NAMESPACE_DNS, f"mimic.permission.{code.value}")) + + permission_table = sa.table( + "permission", + sa.column("id", UUID(as_uuid=True)), + sa.column("code", sa.String), + sa.column("description", sa.String), + ) op.bulk_insert( - sa.table( - "group", - sa.column("id", UUID(as_uuid=True)), - sa.column("name", sa.String), - sa.column("description", sa.String), - sa.column("created_at", sa.DateTime(timezone=True)), - sa.column("updated_at", sa.DateTime(timezone=True)), - ), + permission_table, + [{"id": _pid(p), "code": p.value, "description": None} for p in Permission], + ) + + group_table = sa.table( + "group", + sa.column("id", UUID(as_uuid=True)), + sa.column("name", sa.String), + sa.column("description", sa.String), + ) + op.bulk_insert( + group_table, [ { - "id": "11111111-0000-0000-0000-000000000001", - "name": "rt_operator", + "id": _gid(GroupName.RT_OPERATOR), + "name": GroupName.RT_OPERATOR.value, "description": "Red team operator (per-engagement scope).", }, { - "id": "11111111-0000-0000-0000-000000000002", - "name": "rt_lead", + "id": _gid(GroupName.RT_LEAD), + "name": GroupName.RT_LEAD.value, "description": "Red team lead (full RT privileges).", }, { - "id": "11111111-0000-0000-0000-000000000003", - "name": "soc_analyst", - "description": "SOC analyst (per-engagement, scoped via soc_session).", + "id": _gid(GroupName.SOC_ANALYST), + "name": GroupName.SOC_ANALYST.value, + "description": "SOC analyst (scoped via soc_session).", }, ], ) + group_permission_table = sa.table( + "group_permission", + sa.column("group_id", UUID(as_uuid=True)), + sa.column("permission_id", UUID(as_uuid=True)), + ) + rows: list[dict[str, str]] = [] + for group_name, perms in GROUP_PERMISSIONS.items(): + for perm in perms: + rows.append({"group_id": _gid(group_name), "permission_id": _pid(perm)}) + op.bulk_insert(group_permission_table, rows) + def downgrade() -> None: for table in ( @@ -579,7 +601,6 @@ def downgrade() -> None: "run", "scenario_step", "scenario", - "ttp_version", "ttp", "host", "c2_credential", diff --git a/backend/src/mimic/db/models/__init__.py b/backend/src/mimic/db/models/__init__.py index 79066f7..2de316b 100644 --- a/backend/src/mimic/db/models/__init__.py +++ b/backend/src/mimic/db/models/__init__.py @@ -9,7 +9,7 @@ from mimic.db.models.report import Report from mimic.db.models.run import Run, RunStep, RunStepCleanup from mimic.db.models.scenario import Scenario, ScenarioStep from mimic.db.models.soc_session import SocSession -from mimic.db.models.ttp import Ttp, TtpVersion +from mimic.db.models.ttp import Ttp from mimic.db.models.user import User __all__ = [ @@ -31,7 +31,6 @@ __all__ = [ "ScenarioStep", "SocSession", "Ttp", - "TtpVersion", "User", "UserGroup", ] diff --git a/backend/src/mimic/db/models/ttp.py b/backend/src/mimic/db/models/ttp.py index a6fee32..d6b82e7 100644 --- a/backend/src/mimic/db/models/ttp.py +++ b/backend/src/mimic/db/models/ttp.py @@ -1,38 +1,29 @@ -"""TTP library + immutable version snapshots. +"""TTP library. -Note (D-T-ttp-version): spec H32 originally said no `ttp_version` table (snapshot -lives on `run.snapshot_json`). Sprint 0 reintroduces a `ttp_version` table for -clean traceability across runs and to honor the kickoff data-model directive -from the team-lead. `run.snapshot_json` remains the source of truth for replay, -but each promotion / edit produces an immutable `ttp_version` row that's easier -to diff and reference from imports / audit log. +Spec H32 / D-009: there is **no** `ttp_version` table. The replayability +snapshot lives solely on `run.snapshot_json` (a self-contained JSONB blob +captured at run start). `ttp.version` stays here as a purely informational +counter (§8) — bumped on edit, never referenced for replay. """ from __future__ import annotations -from datetime import datetime -from typing import TYPE_CHECKING from uuid import UUID from sqlalchemy import ( JSON, Boolean, - DateTime, Enum, ForeignKey, Integer, String, Text, - UniqueConstraint, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column from mimic.db.base import Base, TimestampsMixin, UuidPkMixin from mimic.db.types import PayloadType, TtpSource -if TYPE_CHECKING: - pass - class Ttp(UuidPkMixin, TimestampsMixin, Base): __tablename__ = "ttp" @@ -61,45 +52,11 @@ class Ttp(UuidPkMixin, TimestampsMixin, Base): ) tags: Mapped[list[str]] = mapped_column(JSON, default=list, nullable=False) - current_version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + # Informational (§8). Bumped on edit. Never used for replay (H32 / D-009). is_published: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) - # is_published = promoted to the library (lead RT only — F11). + # Promoted to the library (lead RT only — F11). created_by_id: Mapped[UUID | None] = mapped_column( ForeignKey("user.id", ondelete="SET NULL") ) - - versions: Mapped[list[TtpVersion]] = relationship( - back_populates="ttp", - cascade="all, delete-orphan", - order_by="TtpVersion.version", - ) - - -class TtpVersion(UuidPkMixin, Base): - """Immutable snapshot of a TTP at the moment it was published or used. - - Used by importers / audit / report builder. `run.snapshot_json` still - embeds a full self-contained copy for replay independence. - """ - - __tablename__ = "ttp_version" - __table_args__ = ( - UniqueConstraint("ttp_id", "version", name="uq_ttp_version_ttp_id_version"), - ) - - ttp_id: Mapped[UUID] = mapped_column( - ForeignKey("ttp.id", ondelete="CASCADE"), - nullable=False, - ) - version: Mapped[int] = mapped_column(Integer, nullable=False) - snapshot_json: Mapped[dict] = mapped_column(JSON, nullable=False) - content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - created_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) - - ttp: Mapped[Ttp] = relationship(back_populates="versions") From 162b6988f8e508e64d1a3b5eea93f545285ad3f3 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:44:48 +0200 Subject: [PATCH 10/18] fix(backend): align regex_extract + outputs.blob() with D-011/D-012 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D-011 — `regex_extract(text, pattern, *, group=1, name=None)`: - engine google-re2 (linear-time, ReDoS-safe), `re` fallback with 1 MB cap. - first match only. - no match → raises Jinja2 `TemplateError` (no silent default — cleanup templates must fail loud when source string drifts). - default capture is group 1 with fallback to group(0) when the pattern has no groups; named groups via `name=""`. D-012 — `outputs.blob()`: - reads the gzip-compressed CAS file from `MIMIC_BLOB_ROOT`. - 10 MB cap is applied **after** decompression. - decode UTF-8 with latin-1 fallback; never raises (missing / corrupt / non-gzip blobs return empty string, logged at WARNING). Unit tests rewritten to cover both the new fail-loud regex contract and the gzip read path. 49 unit tests pass; ruff clean. --- backend/src/mimic/templating/filters.py | 62 ++++++++++++++++++------- backend/src/mimic/templating/sandbox.py | 49 +++++++++++++------ backend/tests/unit/test_templating.py | 55 ++++++++++++++++++---- 3 files changed, 127 insertions(+), 39 deletions(-) diff --git a/backend/src/mimic/templating/filters.py b/backend/src/mimic/templating/filters.py index b7be750..0df2f53 100644 --- a/backend/src/mimic/templating/filters.py +++ b/backend/src/mimic/templating/filters.py @@ -1,14 +1,21 @@ """Custom Jinja2 filters. -`regex_extract(text, pattern, group=1, default="")` uses google-re2 for -linear-time matching to neutralize ReDoS on adversarial C2 output. If the -library isn't installed the implementation falls back to `re` with a hard -length cap. +`regex_extract(text, pattern, *, group=1, name=None)` per D-011: +- google-re2 engine (linear-time, no backrefs, ReDoS-safe). Falls back to the + stdlib `re` module when re2 is absent, with a 1 MB input cap. +- First match only. +- No match → raises a Jinja2 `TemplateError` (no silent default — cleanup + templates must fail loud when the source string drifts). +- Default capture is group 1, falling back to the full match when the pattern + has no groups. Named groups via `name=""`. """ from __future__ import annotations import re +from typing import Any + +from jinja2 import TemplateError try: # pragma: no cover - presence depends on environment import re2 as _re2 # type: ignore[import-not-found] @@ -23,14 +30,15 @@ _FALLBACK_MAX_INPUT = 1 * 1024 * 1024 # 1 MB safety cap when re2 missing def regex_extract( - text: object, + text: Any, pattern: str, + *, group: int = 1, - default: str = "", + name: str | None = None, ) -> str: - """Return capture group `group` of the first match of `pattern` in `text`.""" + """First-match capture; raise on no match (spec D-011).""" if text is None: - return default + raise TemplateError(f"regex_extract: cannot match against None for /{pattern}/") haystack = text if isinstance(text, str) else str(text) if _HAS_RE2: @@ -39,17 +47,37 @@ def regex_extract( else: if len(haystack) > _FALLBACK_MAX_INPUT: haystack = haystack[:_FALLBACK_MAX_INPUT] - compiled_py = re.compile(pattern) - match = compiled_py.search(haystack) + match = re.compile(pattern).search(haystack) if match is None: - return default + raise TemplateError(f"regex_extract: no match for /{pattern}/") + + if name is not None: + try: + captured = match.group(name) + except IndexError as exc: + raise TemplateError( + f"regex_extract: named group {name!r} not in /{pattern}/" + ) from exc + if captured is None: + raise TemplateError( + f"regex_extract: named group {name!r} captured nothing in /{pattern}/" + ) + return captured + try: captured = match.group(group) - except (IndexError, _IndexErrors): - return default - return captured if captured is not None else default + except IndexError: + if group == 1: + return match.group(0) + raise TemplateError( + f"regex_extract: group {group} out of range for /{pattern}/" + ) from None - -# `re2.error` is `_re2.error`; `re.error` differs. Tuple them for safe catch. -_IndexErrors = (re.error,) + if captured is None: + if group == 1: + return match.group(0) + raise TemplateError( + f"regex_extract: group {group} captured nothing in /{pattern}/" + ) + return captured diff --git a/backend/src/mimic/templating/sandbox.py b/backend/src/mimic/templating/sandbox.py index 671bd17..5f65588 100644 --- a/backend/src/mimic/templating/sandbox.py +++ b/backend/src/mimic/templating/sandbox.py @@ -1,17 +1,22 @@ """Sandboxed Jinja2 environment used to resolve cleanup commands and payloads. -Spec H26 / D-005: two output accessors are exposed. +Spec H26 / D-005 / D-012: two output accessors are exposed to templates. - `{{ params. }}` — straight from the merged TTP/scenario parameters. - `{{ outputs.text }}` — `run_step.output_text` (stdout / UTF-8 text). -- `{{ outputs.blob("name") }}` — decoded `output_blob_ref` content, 10 MB cap, - UTF-8 with latin-1 fallback, silent empty string on non-decodable data. +- `{{ outputs.blob() }}` — decoded `output_blob_ref` content. Per D-012 the + blob lives in `MIMIC_BLOB_ROOT` as a content-addressed gzip-compressed file; + `StepOutputs` does the decompression and exposes a UTF-8 string with a + latin-1 fallback. Hard cap 10 MB **after decompression** (consistent with + F8 evidence limit). The custom `regex_extract` filter operates on the resulting string only. """ from __future__ import annotations +import gzip +import logging from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -23,6 +28,8 @@ from jinja2.sandbox import SandboxedEnvironment from mimic.config import get_settings from mimic.templating.filters import regex_extract +log = logging.getLogger(__name__) + class RenderError(RuntimeError): """Raised when a cleanup / payload template cannot be rendered safely.""" @@ -37,26 +44,42 @@ class StepOutputs: blob_max_bytes: int = 10 * 1024 * 1024 def blob(self, _name: str = "default") -> str: - """Read the binary output blob, decoded (UTF-8 → latin-1 fallback). + """Read the CAS-gzipped output blob (D-012), decoded UTF-8 with + latin-1 fallback. Returns the empty string when the blob is missing + or undecodable (logged but never raises — templates that need a + present blob should assert via regex_extract instead). The argument is accepted for future multi-blob support but ignored in v1 — a step has at most one blob attachment. """ - if self.blob_path is None: - return "" - try: - raw = self.blob_path.read_bytes() - except OSError: + raw = self._read_raw() + if raw is None: return "" if len(raw) > self.blob_max_bytes: raw = raw[: self.blob_max_bytes] try: return raw.decode("utf-8") except UnicodeDecodeError: - try: - return raw.decode("latin-1") - except UnicodeDecodeError: # pragma: no cover - latin-1 never fails - return "" + pass + try: + return raw.decode("latin-1") + except UnicodeDecodeError: # pragma: no cover - latin-1 never fails + log.warning("blob undecodable even as latin-1: %s", self.blob_path) + return "" + + def _read_raw(self) -> bytes | None: + if self.blob_path is None: + return None + try: + with gzip.open(self.blob_path, "rb") as fh: + return fh.read(self.blob_max_bytes + 1) + except FileNotFoundError: + log.warning("blob not found: %s", self.blob_path) + except OSError as exc: + log.warning("blob unreadable %s: %s", self.blob_path, exc) + except gzip.BadGzipFile as exc: + log.warning("blob is not gzip %s: %s", self.blob_path, exc) + return None class CleanupRenderer: diff --git a/backend/tests/unit/test_templating.py b/backend/tests/unit/test_templating.py index d90cb66..df8cb53 100644 --- a/backend/tests/unit/test_templating.py +++ b/backend/tests/unit/test_templating.py @@ -2,7 +2,10 @@ from __future__ import annotations +import gzip + import pytest +from jinja2 import TemplateError from mimic.templating.filters import regex_extract from mimic.templating.sandbox import ( @@ -17,14 +20,23 @@ class TestRegexExtract: def test_returns_capture_group(self) -> None: assert regex_extract("hello world", r"hello (\w+)") == "world" - def test_default_when_no_match(self) -> None: - assert regex_extract("hello", r"foo(\d+)", default="N/A") == "N/A" + def test_no_match_raises(self) -> None: + with pytest.raises(TemplateError, match="no match"): + regex_extract("hello", r"foo(\d+)") - def test_none_input_returns_default(self) -> None: - assert regex_extract(None, r"x", default="empty") == "empty" + def test_none_input_raises(self) -> None: + with pytest.raises(TemplateError, match="None"): + regex_extract(None, r"x") - def test_supports_group_zero(self) -> None: - assert regex_extract("abc123", r"\w+\d+", group=0) == "abc123" + def test_no_groups_falls_back_to_full_match(self) -> None: + assert regex_extract("abc123", r"\w+\d+") == "abc123" + + def test_named_group(self) -> None: + assert regex_extract("pid=4242", r"pid=(?P\d+)", name="n") == "4242" + + def test_missing_named_group_raises(self) -> None: + with pytest.raises(TemplateError): + regex_extract("pid=4242", r"pid=(\d+)", name="absent") class TestCleanupRenderer: @@ -52,6 +64,13 @@ class TestCleanupRenderer: ) assert out == "4242" + def test_regex_extract_no_match_propagates_as_render_error(self) -> None: + with pytest.raises(RenderError, match="no match"): + self.renderer.render( + r"{{ outputs.text | regex_extract('pid=(\\d+)') }}", + outputs=StepOutputs(text="nothing"), + ) + def test_strict_undefined_raises(self) -> None: with pytest.raises(RenderError): self.renderer.render("{{ params.does_not_exist }}", params={}) @@ -73,8 +92,26 @@ class TestStepOutputsBlob: out = StepOutputs(text="x") assert out.blob() == "" - def test_blob_caps_size(self, tmp_path) -> None: - blob = tmp_path / "evidence.bin" - blob.write_bytes(b"A" * 1024) + def test_blob_reads_gzipped_file(self, tmp_path) -> None: + blob = tmp_path / "blob.gz" + with gzip.open(blob, "wb") as fh: + fh.write(b"hello") + out = StepOutputs(blob_path=blob) + assert out.blob() == "hello" + + def test_blob_caps_size_after_decompression(self, tmp_path) -> None: + blob = tmp_path / "blob.gz" + with gzip.open(blob, "wb") as fh: + fh.write(b"A" * 1024) out = StepOutputs(blob_path=blob, blob_max_bytes=10) assert out.blob() == "A" * 10 + + def test_blob_missing_file_returns_empty(self, tmp_path) -> None: + out = StepOutputs(blob_path=tmp_path / "absent.gz") + assert out.blob() == "" + + def test_blob_non_gzip_returns_empty(self, tmp_path) -> None: + blob = tmp_path / "blob.gz" + blob.write_bytes(b"not actually gzip") + out = StepOutputs(blob_path=blob) + assert out.blob() == "" From 12d131c82693cbb3bed54fb655a33a5af7003563 Mon Sep 17 00:00:00 2001 From: knacky Date: Thu, 21 May 2026 20:44:59 +0200 Subject: [PATCH 11/18] feat(backend): add content-addressed gzip blob store (D-012) Two on-disk pools per D-012: - `MIMIC_BLOB_ROOT` (default `/var/lib/mimic/blobs/`) holds C2 polling output blobs, content-addressed gzip layout `//.gz`. - `MIMIC_EVIDENCE_ROOT` (default `/var/lib/mimic/evidence/`) reserved for user-uploaded evidence (flat per-engagement, no compression). Wired only in config + .env.example here; F8 endpoint lands later. `mimic.storage.blob`: - `blob_path(root, sha256_hex)` validates the digest and returns the CAS path. Raises ValueError on a malformed digest (length != 64 or non-hex). - `store_blob(root, data)` hashes, gzip-compresses, atomically writes to `//.gz` (0o750 dir perms, 0o640 file perms). Idempotent: duplicate writes leave mtime untouched. 5 new unit tests cover happy path, deduplication, idempotency, malformed digest, and the two-byte-pair directory layout. --- backend/.env.example | 4 ++ backend/src/mimic/config.py | 5 +++ backend/src/mimic/storage/__init__.py | 14 +++++++ backend/src/mimic/storage/blob.py | 51 +++++++++++++++++++++++++ backend/tests/unit/test_storage_blob.py | 49 ++++++++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 backend/src/mimic/storage/__init__.py create mode 100644 backend/src/mimic/storage/blob.py create mode 100644 backend/tests/unit/test_storage_blob.py diff --git a/backend/.env.example b/backend/.env.example index c169cb2..739dff2 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -21,3 +21,7 @@ MIMIC_CORS_ORIGINS=http://localhost:5173 # Logging MIMIC_LOG_LEVEL=DEBUG MIMIC_LOG_JSON=false + +# Storage pools (D-012) +MIMIC_BLOB_ROOT=/var/lib/mimic/blobs +MIMIC_EVIDENCE_ROOT=/var/lib/mimic/evidence diff --git a/backend/src/mimic/config.py b/backend/src/mimic/config.py index 6f47f07..052db46 100644 --- a/backend/src/mimic/config.py +++ b/backend/src/mimic/config.py @@ -3,6 +3,7 @@ from __future__ import annotations from functools import lru_cache +from pathlib import Path from typing import Literal from pydantic import Field, SecretStr, field_validator @@ -45,6 +46,10 @@ class Settings(BaseSettings): template_render_timeout_ms: int = 250 output_blob_max_bytes: int = 10 * 1024 * 1024 + # D-012: two on-disk pools. + blob_root: Path = Path("/var/lib/mimic/blobs") + evidence_root: Path = Path("/var/lib/mimic/evidence") + @field_validator("cors_origins", mode="before") @classmethod def _split_cors(cls, value: object) -> object: diff --git a/backend/src/mimic/storage/__init__.py b/backend/src/mimic/storage/__init__.py new file mode 100644 index 0000000..79d2b6c --- /dev/null +++ b/backend/src/mimic/storage/__init__.py @@ -0,0 +1,14 @@ +"""Local file-system pools (D-012). + +Two separate roots, configured via env: +- `MIMIC_BLOB_ROOT` (default `/var/lib/mimic/blobs/`) holds C2 polling output + blobs, content-addressed and gzip-compressed: `//.gz` where + `aa` and `bb` are the first two byte-pairs of the hex digest. +- `MIMIC_EVIDENCE_ROOT` (default `/var/lib/mimic/evidence/`) holds user-uploaded + evidence files, flat layout `/.`, no + compression. +""" + +from mimic.storage.blob import blob_path, store_blob + +__all__ = ["blob_path", "store_blob"] diff --git a/backend/src/mimic/storage/blob.py b/backend/src/mimic/storage/blob.py new file mode 100644 index 0000000..c2d1a22 --- /dev/null +++ b/backend/src/mimic/storage/blob.py @@ -0,0 +1,51 @@ +"""Content-addressed gzip-compressed blob store (D-012).""" + +from __future__ import annotations + +import gzip +import hashlib +import os +import stat +from pathlib import Path + +_SHA256_HEX_LEN = 64 + + +def _validate_digest(sha256_hex: str) -> str: + if len(sha256_hex) != _SHA256_HEX_LEN or any( + c not in "0123456789abcdef" for c in sha256_hex.lower() + ): + raise ValueError(f"invalid sha256 digest: {sha256_hex!r}") + return sha256_hex.lower() + + +def blob_path(root: Path | str, sha256_hex: str) -> Path: + """Return the absolute path of the gzip-compressed blob `//.gz`.""" + digest = _validate_digest(sha256_hex) + return Path(root) / digest[0:2] / digest[2:4] / f"{digest}.gz" + + +def store_blob(root: Path | str, data: bytes) -> tuple[str, Path]: + """Write `data` (gzip-compressed) under its sha256 digest path. + + Idempotent: an existing path with the same digest is not overwritten. + Directory permissions are `0750` so only the owner and the `mimic` group + can read. + """ + digest = hashlib.sha256(data).hexdigest() + target = blob_path(root, digest) + if target.exists(): + return digest, target + + target.parent.mkdir(parents=True, exist_ok=True) + # 0o750: owner full, group r-x, others none. The blob root is owned by the + # `mimic` system user; only the application and any explicit group member + # (audit / backup) get read access. + os.chmod(target.parent, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) # noqa: S103 + + tmp = target.with_suffix(target.suffix + ".tmp") + with gzip.open(tmp, "wb") as fh: + fh.write(data) + os.chmod(tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP) + tmp.replace(target) + return digest, target diff --git a/backend/tests/unit/test_storage_blob.py b/backend/tests/unit/test_storage_blob.py new file mode 100644 index 0000000..a5fa456 --- /dev/null +++ b/backend/tests/unit/test_storage_blob.py @@ -0,0 +1,49 @@ +"""Content-addressed gzip blob store (D-012).""" + +from __future__ import annotations + +import gzip +import hashlib + +import pytest + +from mimic.storage.blob import blob_path, store_blob + + +def test_blob_path_uses_two_byte_pairs(tmp_path) -> None: + digest = "ab" + "cd" + "ef" * 30 + path = blob_path(tmp_path, digest) + assert path == tmp_path / "ab" / "cd" / f"{digest}.gz" + + +def test_blob_path_rejects_invalid_digest(tmp_path) -> None: + with pytest.raises(ValueError, match="invalid sha256"): + blob_path(tmp_path, "not-a-digest") + + +def test_store_blob_writes_gzip_and_returns_digest(tmp_path) -> None: + payload = b"hello world\n" + expected = hashlib.sha256(payload).hexdigest() + digest, path = store_blob(tmp_path, payload) + assert digest == expected + assert path == tmp_path / expected[0:2] / expected[2:4] / f"{expected}.gz" + with gzip.open(path, "rb") as fh: + assert fh.read() == payload + + +def test_store_blob_is_idempotent(tmp_path) -> None: + payload = b"same content" + digest1, path1 = store_blob(tmp_path, payload) + mtime_before = path1.stat().st_mtime_ns + digest2, path2 = store_blob(tmp_path, payload) + assert digest1 == digest2 + assert path1 == path2 + assert path2.stat().st_mtime_ns == mtime_before + + +def test_store_blob_dedupes_distinct_payloads(tmp_path) -> None: + _, p1 = store_blob(tmp_path, b"alpha") + _, p2 = store_blob(tmp_path, b"beta") + assert p1 != p2 + assert p1.exists() + assert p2.exists() From adab8a58e7c4809f18dd01ae56f96678ca699dbc Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:10:51 +0200 Subject: [PATCH 12/18] chore(backend): mypy strict clean + ruff format pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-merge sanity per devops checklist (ruff format --check, mypy --strict). Type fixes: - ORM models: `Mapped[dict]` → `Mapped[dict[str, Any]]` (audit, scenario, run, report, ttp, detection.artifact_files_json). Equivalent on Pydantic DTOs (TtpBase.params_schema_json, ScenarioStepBase.params_override_json). - Rename `TtpRead.current_version` → `TtpRead.version` to mirror the ORM column (which itself was renamed in D-009 cleanup). - Flask blueprints: add `-> ResponseReturnValue` to every view, plus typed UUID params on `_validate_step_consistency`. - `templating/filters.py`: rewrite the conditional re2 import so mypy can narrow the union (`ModuleType | None`); the runtime branch on `_re2 is not None` removes the unused-ignore that was triggered by warn_unused_ignores. - `pyproject.toml`: add `flask_login.*` and `pythonjsonlogger.*` to the `[[tool.mypy.overrides]]` `ignore_missing_imports` list (both ship without typed marker). - Misc: drop stale `# type: ignore` comments (`app.py:36`, `rbac/decorators.py:35`) flagged by `warn_unused_ignores`. Keep `logging.JsonFormatter` ignore because the symbol exists at runtime but is not re-exported through the typed surface. Formatting: - `ruff format` applied (15 files normalized; line-length unchanged at 100). Verification on this commit: - `ruff check` → All checks passed. - `ruff format --check` → 68 files already formatted. - `mypy --strict src` → Success: no issues found in 54 source files. - `pytest tests/unit` → 49 passed. --- backend/pyproject.toml | 2 + backend/src/mimic/api/engagements.py | 11 +- backend/src/mimic/api/hosts.py | 9 +- backend/src/mimic/api/scenarios.py | 17 +- backend/src/mimic/api/ttps.py | 11 +- backend/src/mimic/app.py | 2 +- backend/src/mimic/cli/user.py | 4 +- backend/src/mimic/connectors/factory.py | 4 +- .../versions/202605210001_initial_schema.py | 175 ++++++++++++------ backend/src/mimic/db/models/audit.py | 7 +- backend/src/mimic/db/models/detection.py | 12 +- backend/src/mimic/db/models/engagement.py | 4 +- backend/src/mimic/db/models/permission.py | 8 +- backend/src/mimic/db/models/report.py | 12 +- backend/src/mimic/db/models/run.py | 12 +- backend/src/mimic/db/models/scenario.py | 8 +- backend/src/mimic/db/models/soc_session.py | 4 +- backend/src/mimic/db/models/ttp.py | 7 +- backend/src/mimic/logging.py | 6 +- backend/src/mimic/rbac/decorators.py | 7 +- backend/src/mimic/schemas/scenario.py | 3 +- backend/src/mimic/schemas/ttp.py | 7 +- backend/src/mimic/templating/filters.py | 32 ++-- backend/tests/integration/conftest.py | 4 +- 24 files changed, 203 insertions(+), 165 deletions(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0b38510..1301e16 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -113,6 +113,8 @@ module = [ "re2", "flask_socketio.*", "flask_migrate.*", + "flask_login.*", + "pythonjsonlogger.*", "gevent.*", "testcontainers.*", "authlib.*", diff --git a/backend/src/mimic/api/engagements.py b/backend/src/mimic/api/engagements.py index 0df267c..ec73b2d 100644 --- a/backend/src/mimic/api/engagements.py +++ b/backend/src/mimic/api/engagements.py @@ -3,6 +3,7 @@ from __future__ import annotations from flask import Blueprint, abort, jsonify +from flask.typing import ResponseReturnValue from sqlalchemy import select from mimic.api._helpers import jsonify_model, parse_body, parse_uuid @@ -17,7 +18,7 @@ bp = Blueprint("engagements", __name__) @bp.get("") @require_perm(Permission.ENGAGEMENT_READ) -def list_engagements(): +def list_engagements() -> ResponseReturnValue: stmt = select(Engagement).order_by(Engagement.created_at.desc()) rows = db.session.execute(stmt).scalars().all() return jsonify([EngagementRead.model_validate(row).model_dump(mode="json") for row in rows]) @@ -25,7 +26,7 @@ def list_engagements(): @bp.post("") @require_perm(Permission.ENGAGEMENT_CREATE) -def create_engagement(): +def create_engagement() -> ResponseReturnValue: payload = parse_body(EngagementCreate) engagement = Engagement( client_name=payload.client_name, @@ -42,7 +43,7 @@ def create_engagement(): @bp.get("/") @require_perm(Permission.ENGAGEMENT_READ) -def get_engagement(eid: str): +def get_engagement(eid: str) -> ResponseReturnValue: engagement = db.session.get(Engagement, parse_uuid(eid)) if engagement is None: abort(404) @@ -51,7 +52,7 @@ def get_engagement(eid: str): @bp.put("/") @require_perm(Permission.ENGAGEMENT_UPDATE) -def update_engagement(eid: str): +def update_engagement(eid: str) -> ResponseReturnValue: engagement = db.session.get(Engagement, parse_uuid(eid)) if engagement is None: abort(404) @@ -64,7 +65,7 @@ def update_engagement(eid: str): @bp.delete("/") @require_perm(Permission.ENGAGEMENT_DELETE) -def delete_engagement(eid: str): +def delete_engagement(eid: str) -> ResponseReturnValue: engagement = db.session.get(Engagement, parse_uuid(eid)) if engagement is None: abort(404) diff --git a/backend/src/mimic/api/hosts.py b/backend/src/mimic/api/hosts.py index 0594862..984d82c 100644 --- a/backend/src/mimic/api/hosts.py +++ b/backend/src/mimic/api/hosts.py @@ -3,6 +3,7 @@ from __future__ import annotations from flask import Blueprint, abort, jsonify +from flask.typing import ResponseReturnValue from sqlalchemy import select from mimic.api._helpers import jsonify_model, parse_body, parse_uuid @@ -24,7 +25,7 @@ def _engagement_or_404(eid: str) -> Engagement: @bp.get("/engagements//hosts") @require_perm(Permission.HOST_CRUD) -def list_hosts(eid: str): +def list_hosts(eid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) stmt = select(Host).where(Host.engagement_id == engagement.id).order_by(Host.hostname) rows = db.session.execute(stmt).scalars().all() @@ -33,7 +34,7 @@ def list_hosts(eid: str): @bp.post("/engagements//hosts") @require_perm(Permission.HOST_CRUD) -def create_host(eid: str): +def create_host(eid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) payload = parse_body(HostCreate) host = Host( @@ -52,7 +53,7 @@ def create_host(eid: str): @bp.put("/engagements//hosts/") @require_perm(Permission.HOST_CRUD) -def update_host(eid: str, hid: str): +def update_host(eid: str, hid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) host = db.session.get(Host, parse_uuid(hid, field="host id")) if host is None or host.engagement_id != engagement.id: @@ -66,7 +67,7 @@ def update_host(eid: str, hid: str): @bp.delete("/engagements//hosts/") @require_perm(Permission.HOST_CRUD) -def delete_host(eid: str, hid: str): +def delete_host(eid: str, hid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) host = db.session.get(Host, parse_uuid(hid, field="host id")) if host is None or host.engagement_id != engagement.id: diff --git a/backend/src/mimic/api/scenarios.py b/backend/src/mimic/api/scenarios.py index e95de70..5e0c0bf 100644 --- a/backend/src/mimic/api/scenarios.py +++ b/backend/src/mimic/api/scenarios.py @@ -2,7 +2,10 @@ from __future__ import annotations +from uuid import UUID + from flask import Blueprint, abort, jsonify +from flask.typing import ResponseReturnValue from sqlalchemy import select from mimic.api._helpers import jsonify_model, parse_body, parse_uuid @@ -34,7 +37,7 @@ def _scenario_or_404(engagement: Engagement, sid: str) -> Scenario: return scenario -def _validate_step_consistency(scenario: Scenario, ttp_id, host_id) -> None: +def _validate_step_consistency(scenario: Scenario, ttp_id: UUID, host_id: UUID) -> None: ttp = db.session.get(Ttp, ttp_id) host = db.session.get(Host, host_id) if ttp is None or host is None: @@ -47,7 +50,7 @@ def _validate_step_consistency(scenario: Scenario, ttp_id, host_id) -> None: @bp.get("/engagements//scenarios") @require_perm(Permission.SCENARIO_CRUD) -def list_scenarios(eid: str): +def list_scenarios(eid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) stmt = ( select(Scenario) @@ -60,7 +63,7 @@ def list_scenarios(eid: str): @bp.post("/engagements//scenarios") @require_perm(Permission.SCENARIO_CRUD) -def create_scenario(eid: str): +def create_scenario(eid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) payload = parse_body(ScenarioCreate) scenario = Scenario( @@ -90,7 +93,7 @@ def create_scenario(eid: str): @bp.get("/engagements//scenarios/") @require_perm(Permission.SCENARIO_CRUD) -def get_scenario(eid: str, sid: str): +def get_scenario(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) return jsonify_model(ScenarioRead.model_validate(scenario)) @@ -98,7 +101,7 @@ def get_scenario(eid: str, sid: str): @bp.put("/engagements//scenarios/") @require_perm(Permission.SCENARIO_CRUD) -def update_scenario(eid: str, sid: str): +def update_scenario(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) payload = parse_body(ScenarioUpdate) @@ -110,7 +113,7 @@ def update_scenario(eid: str, sid: str): @bp.delete("/engagements//scenarios/") @require_perm(Permission.SCENARIO_CRUD) -def delete_scenario(eid: str, sid: str): +def delete_scenario(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) db.session.delete(scenario) @@ -120,7 +123,7 @@ def delete_scenario(eid: str, sid: str): @bp.post("/engagements//scenarios//steps") @require_perm(Permission.SCENARIO_CRUD) -def add_step(eid: str, sid: str): +def add_step(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) payload = parse_body(ScenarioStepCreate) diff --git a/backend/src/mimic/api/ttps.py b/backend/src/mimic/api/ttps.py index 0e33e9d..929c55c 100644 --- a/backend/src/mimic/api/ttps.py +++ b/backend/src/mimic/api/ttps.py @@ -3,6 +3,7 @@ from __future__ import annotations from flask import Blueprint, abort, jsonify +from flask.typing import ResponseReturnValue from sqlalchemy import select from mimic.api._helpers import jsonify_model, parse_body, parse_uuid @@ -16,7 +17,7 @@ bp = Blueprint("ttps", __name__) @bp.get("") @require_perm(Permission.TTP_READ) -def list_ttps(): +def list_ttps() -> ResponseReturnValue: stmt = select(Ttp).order_by(Ttp.created_at.desc()) rows = db.session.execute(stmt).scalars().all() return jsonify([TtpRead.model_validate(row).model_dump(mode="json") for row in rows]) @@ -24,7 +25,7 @@ def list_ttps(): @bp.post("") @require_perm(Permission.TTP_DRAFT) -def create_ttp(): +def create_ttp() -> ResponseReturnValue: payload = parse_body(TtpCreate) ttp = Ttp(**payload.model_dump()) db.session.add(ttp) @@ -34,7 +35,7 @@ def create_ttp(): @bp.get("/") @require_perm(Permission.TTP_READ) -def get_ttp(tid: str): +def get_ttp(tid: str) -> ResponseReturnValue: ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) if ttp is None: abort(404) @@ -43,7 +44,7 @@ def get_ttp(tid: str): @bp.put("/") @require_perm(Permission.TTP_DRAFT) -def update_ttp(tid: str): +def update_ttp(tid: str) -> ResponseReturnValue: ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) if ttp is None: abort(404) @@ -63,7 +64,7 @@ def update_ttp(tid: str): @bp.delete("/") @require_perm(Permission.TTP_DRAFT) -def delete_ttp(tid: str): +def delete_ttp(tid: str) -> ResponseReturnValue: ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) if ttp is None: abort(404) diff --git a/backend/src/mimic/app.py b/backend/src/mimic/app.py index 9e455e9..86a2738 100644 --- a/backend/src/mimic/app.py +++ b/backend/src/mimic/app.py @@ -33,7 +33,7 @@ def create_app(settings: Settings | None = None) -> Flask: db.init_app(app) migrate.init_app(app, db, directory="src/mimic/db/migrations") login_manager.init_app(app) - login_manager.user_loader(load_user) # type: ignore[arg-type] + login_manager.user_loader(load_user) socketio.init_app( app, diff --git a/backend/src/mimic/cli/user.py b/backend/src/mimic/cli/user.py index 5c3ef54..df4aa38 100644 --- a/backend/src/mimic/cli/user.py +++ b/backend/src/mimic/cli/user.py @@ -39,9 +39,7 @@ def create_user(email: str, user_type: str, password: str, display_name: str | N db.session.add(user) db.session.flush() - group_name = ( - GroupName.RT_LEAD if user.type is UserType.RT_LEAD else GroupName.RT_OPERATOR - ) + group_name = GroupName.RT_LEAD if user.type is UserType.RT_LEAD else GroupName.RT_OPERATOR group = db.session.query(Group).filter_by(name=group_name.value).first() if group is None: raise click.ClickException( diff --git a/backend/src/mimic/connectors/factory.py b/backend/src/mimic/connectors/factory.py index 2c7c2cd..8ff7b1d 100644 --- a/backend/src/mimic/connectors/factory.py +++ b/backend/src/mimic/connectors/factory.py @@ -46,9 +46,7 @@ class ConnectorFactory: try: klass = _REGISTRY[c2_type] except KeyError as exc: - raise NotImplementedError( - f"no connector registered for {c2_type.value}" - ) from exc + raise NotImplementedError(f"no connector registered for {c2_type.value}") from exc connector = klass() connector.authenticate(self._resolver(c2_type)) diff --git a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py index 61490d1..93d0340 100644 --- a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py +++ b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py @@ -13,6 +13,7 @@ Revision ID: 202605210001 Revises: Create Date: 2026-05-21 """ + from __future__ import annotations import sqlalchemy as sa @@ -30,9 +31,7 @@ depends_on: str | None = None # --------------------------------------------------------------------------- USER_TYPE = sa.Enum("rt_operator", "rt_lead", "soc_analyst", name="user_type") -ENGAGEMENT_STATUS = sa.Enum( - "draft", "active", "closed", "archived", name="engagement_status" -) +ENGAGEMENT_STATUS = sa.Enum("draft", "active", "closed", "archived", name="engagement_status") C2_TYPE = sa.Enum("mythic", "home", name="c2_type") HOST_STATUS = sa.Enum("unknown", "alive", "dead", name="host_status") PAYLOAD_TYPE = sa.Enum( @@ -64,18 +63,10 @@ RUN_STEP_STATUS = sa.Enum( "cleanup_failed", name="run_step_status", ) -CLEANUP_STATUS = sa.Enum( - "pending", "success", "failed", "partial", name="cleanup_status" -) -DETECTION_LEVEL = sa.Enum( - "detected", "partial", "not_detected", name="detection_level" -) -DETECTION_SOURCE = sa.Enum( - "ndr", "edr", "siem", "manual", "other", name="detection_source" -) -EVIDENCE_STATUS = sa.Enum( - "success", "failure", "partial", name="evidence_status" -) +CLEANUP_STATUS = sa.Enum("pending", "success", "failed", "partial", name="cleanup_status") +DETECTION_LEVEL = sa.Enum("detected", "partial", "not_detected", name="detection_level") +DETECTION_SOURCE = sa.Enum("ndr", "edr", "siem", "manual", "other", name="detection_source") +EVIDENCE_STATUS = sa.Enum("success", "failure", "partial", name="evidence_status") def upgrade() -> None: @@ -107,8 +98,12 @@ def upgrade() -> None: sa.Column("local_password_hash", sa.String(255)), sa.Column("disabled_at", sa.DateTime(timezone=True)), sa.Column("last_login_at", sa.DateTime(timezone=True)), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), sa.UniqueConstraint("email", name="uq_user_email"), ) @@ -126,8 +121,12 @@ def upgrade() -> None: sa.Column("id", UUID(as_uuid=True), primary_key=True), sa.Column("name", sa.String(80), nullable=False), sa.Column("description", sa.String(255)), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), sa.UniqueConstraint("name", name="uq_group_name"), ) @@ -136,13 +135,19 @@ def upgrade() -> None: sa.Column( "group_id", UUID(as_uuid=True), - sa.ForeignKey("group.id", ondelete="CASCADE", name="fk_group_permission_group_id_group"), + sa.ForeignKey( + "group.id", ondelete="CASCADE", name="fk_group_permission_group_id_group" + ), nullable=False, ), sa.Column( "permission_id", UUID(as_uuid=True), - sa.ForeignKey("permission.id", ondelete="CASCADE", name="fk_group_permission_permission_id_permission"), + sa.ForeignKey( + "permission.id", + ondelete="CASCADE", + name="fk_group_permission_permission_id_permission", + ), nullable=False, ), sa.PrimaryKeyConstraint("group_id", "permission_id", name="pk_group_permission"), @@ -158,9 +163,15 @@ def upgrade() -> None: sa.Column("start_date", sa.Date), sa.Column("end_date", sa.Date), sa.Column("c2_type", C2_TYPE, nullable=False, server_default="mythic"), - sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_table( @@ -217,8 +228,12 @@ def upgrade() -> None: sa.Column("config_fernet", sa.LargeBinary, nullable=False), sa.Column("version", sa.Integer, nullable=False, server_default="1"), sa.Column("retired_at", sa.DateTime(timezone=True)), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_index( "ix_c2_credential_engagement_active", @@ -244,8 +259,12 @@ def upgrade() -> None: sa.Column("c2_type", C2_TYPE, nullable=False), sa.Column("status", HOST_STATUS, nullable=False, server_default="unknown"), sa.Column("last_seen", sa.DateTime(timezone=True)), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_index("ix_host_engagement_id", "host", ["engagement_id"]) @@ -267,9 +286,15 @@ def upgrade() -> None: sa.Column("tags", JSONB, nullable=False, server_default="[]"), sa.Column("version", sa.Integer, nullable=False, server_default="1"), sa.Column("is_published", sa.Boolean, nullable=False, server_default=sa.false()), - sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) # No `ttp_version` table — H32 / D-009: snapshot lives on run.snapshot_json. @@ -287,9 +312,15 @@ def upgrade() -> None: sa.Column("description", sa.Text), sa.Column("version", sa.Integer, nullable=False, server_default="1"), sa.Column("c2_type", C2_TYPE, nullable=False), - sa.Column("created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_table( @@ -316,8 +347,12 @@ def upgrade() -> None: ), sa.Column("params_override_json", JSONB, nullable=False, server_default="{}"), sa.Column("delay_after_ms", sa.Integer, nullable=False, server_default="0"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), sa.UniqueConstraint("scenario_id", "order_idx", name="uq_scenario_step_order_idx"), ) @@ -334,10 +369,16 @@ def upgrade() -> None: sa.Column("status", RUN_STATUS, nullable=False, server_default="queued"), sa.Column("started_at", sa.DateTime(timezone=True)), sa.Column("ended_at", sa.DateTime(timezone=True)), - sa.Column("started_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), + sa.Column( + "started_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), sa.Column("snapshot_json", JSONB, nullable=False, server_default="{}"), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_table( @@ -363,8 +404,12 @@ def upgrade() -> None: sa.Column("output_blob_ref", sa.String(512)), sa.Column("exit_code", sa.Integer), sa.Column("resolved_payload_text", sa.Text), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_table( @@ -382,9 +427,15 @@ def upgrade() -> None: sa.Column("ended_at", sa.DateTime(timezone=True)), sa.Column("resolved_command_text", sa.Text), sa.Column("output", sa.Text), - sa.Column("executed_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "executed_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) # ----------------------------------------------------- detection / evidence @@ -408,8 +459,12 @@ def upgrade() -> None: sa.Column("latency_ms", sa.Integer), sa.Column("comment", sa.Text), sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) op.create_table( @@ -432,8 +487,12 @@ def upgrade() -> None: sa.Column("artifact_files_json", JSONB, nullable=False, server_default="[]"), sa.Column("comment", sa.Text), sa.Column("recorded_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) # ----------------------------------------------------------------- report @@ -452,9 +511,15 @@ def upgrade() -> None: sa.Column("pdf_path", sa.String(512)), sa.Column("md_path", sa.String(512)), sa.Column("generated_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("generated_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL")), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "generated_by_id", UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="SET NULL") + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) # ------------------------------------------------------------- soc_session @@ -479,8 +544,12 @@ def upgrade() -> None: sa.Column("last_ip", sa.String(64)), sa.Column("last_user_agent", sa.String(512)), sa.Column("last_used_at", sa.DateTime(timezone=True)), - sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), - sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), ) # ------------------------------------------------------------ audit_log diff --git a/backend/src/mimic/db/models/audit.py b/backend/src/mimic/db/models/audit.py index 3c9861f..c1a299a 100644 --- a/backend/src/mimic/db/models/audit.py +++ b/backend/src/mimic/db/models/audit.py @@ -10,6 +10,7 @@ WORM enforcement without a migration; sprint 0 fills the columns at insert. from __future__ import annotations from datetime import datetime +from typing import Any from uuid import UUID from sqlalchemy import JSON, DateTime, ForeignKey, String, Text, func @@ -26,13 +27,11 @@ class AuditLog(UuidPkMixin, Base): nullable=False, server_default=func.now(), ) - actor_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + actor_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) action: Mapped[str] = mapped_column(String(80), nullable=False) resource_type: Mapped[str] = mapped_column(String(80), nullable=False) resource_id: Mapped[str | None] = mapped_column(String(128)) - metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict) prev_hash: Mapped[str | None] = mapped_column(String(64)) row_hash: Mapped[str] = mapped_column(String(64), nullable=False) diff --git a/backend/src/mimic/db/models/detection.py b/backend/src/mimic/db/models/detection.py index 9f11c29..8edce89 100644 --- a/backend/src/mimic/db/models/detection.py +++ b/backend/src/mimic/db/models/detection.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from uuid import UUID from sqlalchemy import ( @@ -45,9 +45,7 @@ class Detection(UuidPkMixin, TimestampsMixin, Base): ) latency_ms: Mapped[int | None] = mapped_column(Integer) comment: Mapped[str | None] = mapped_column(Text) - recorded_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) + recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) run_step: Mapped[RunStep] = relationship() soc_user: Mapped[User] = relationship() @@ -69,14 +67,12 @@ class Evidence(UuidPkMixin, TimestampsMixin, Base): nullable=False, ) artifacts_text: Mapped[str | None] = mapped_column(Text) - artifact_files_json: Mapped[list[dict]] = mapped_column( + artifact_files_json: Mapped[list[dict[str, Any]]] = mapped_column( JSON, default=list, nullable=False ) # Each entry: {"name": str, "ref": str, "sha256": str, "size_bytes": int} comment: Mapped[str | None] = mapped_column(Text) - recorded_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) + recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) run_step: Mapped[RunStep] = relationship() rt_user: Mapped[User] = relationship() diff --git a/backend/src/mimic/db/models/engagement.py b/backend/src/mimic/db/models/engagement.py index 48b326e..d57a174 100644 --- a/backend/src/mimic/db/models/engagement.py +++ b/backend/src/mimic/db/models/engagement.py @@ -37,9 +37,7 @@ class Engagement(UuidPkMixin, TimestampsMixin, Base): nullable=False, ) - created_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + created_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) hosts: Mapped[list[Host]] = relationship( back_populates="engagement", diff --git a/backend/src/mimic/db/models/permission.py b/backend/src/mimic/db/models/permission.py index d6fbeef..81d6255 100644 --- a/backend/src/mimic/db/models/permission.py +++ b/backend/src/mimic/db/models/permission.py @@ -64,14 +64,10 @@ class GroupPermission(Base): class UserGroup(Base): __tablename__ = "user_group" __table_args__ = ( - PrimaryKeyConstraint( - "user_id", "group_id", "engagement_id", name="pk_user_group" - ), + PrimaryKeyConstraint("user_id", "group_id", "engagement_id", name="pk_user_group"), ) - user_id: Mapped[UUID] = mapped_column( - ForeignKey("user.id", ondelete="CASCADE"), nullable=False - ) + user_id: Mapped[UUID] = mapped_column(ForeignKey("user.id", ondelete="CASCADE"), nullable=False) group_id: Mapped[UUID] = mapped_column( ForeignKey("group.id", ondelete="CASCADE"), nullable=False ) diff --git a/backend/src/mimic/db/models/report.py b/backend/src/mimic/db/models/report.py index 36ad5a6..5c6dd2d 100644 --- a/backend/src/mimic/db/models/report.py +++ b/backend/src/mimic/db/models/report.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from uuid import UUID from sqlalchemy import ( @@ -30,7 +30,7 @@ class Report(UuidPkMixin, TimestampsMixin, Base): ) version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) - content_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + content_json: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict) content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) # SHA-256 of canonical JSON. Identical hash referenced in PDF footer, JSON # export and Markdown export (spec H19 / F9 / F14). @@ -38,11 +38,7 @@ class Report(UuidPkMixin, TimestampsMixin, Base): pdf_path: Mapped[str | None] = mapped_column(String(512)) md_path: Mapped[str | None] = mapped_column(String(512)) - generated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) - generated_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + generated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + generated_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) engagement: Mapped[Engagement] = relationship() diff --git a/backend/src/mimic/db/models/run.py b/backend/src/mimic/db/models/run.py index 1efb6a0..c0af64e 100644 --- a/backend/src/mimic/db/models/run.py +++ b/backend/src/mimic/db/models/run.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from uuid import UUID from sqlalchemy import ( @@ -39,11 +39,9 @@ class Run(UuidPkMixin, TimestampsMixin, Base): started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - started_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + started_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) - snapshot_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + snapshot_json: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False, default=dict) # Full self-contained snapshot of scenario + steps + resolved TTPs. # Source of truth for replay (spec H32). @@ -111,8 +109,6 @@ class RunStepCleanup(UuidPkMixin, TimestampsMixin, Base): ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) resolved_command_text: Mapped[str | None] = mapped_column(Text) output: Mapped[str | None] = mapped_column(Text) - executed_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + executed_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) run_step: Mapped[RunStep] = relationship(back_populates="cleanup") diff --git a/backend/src/mimic/db/models/scenario.py b/backend/src/mimic/db/models/scenario.py index 19198cd..33dc03a 100644 --- a/backend/src/mimic/db/models/scenario.py +++ b/backend/src/mimic/db/models/scenario.py @@ -6,7 +6,7 @@ run start that every referenced host matches. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from uuid import UUID from sqlalchemy import ( @@ -45,9 +45,7 @@ class Scenario(UuidPkMixin, TimestampsMixin, Base): Enum(C2Type, name="c2_type", create_type=False), nullable=False, ) - created_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + created_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) engagement: Mapped[Engagement] = relationship(back_populates="scenarios") steps: Mapped[list[ScenarioStep]] = relationship( @@ -78,7 +76,7 @@ class ScenarioStep(UuidPkMixin, TimestampsMixin, Base): ForeignKey("host.id", ondelete="RESTRICT"), nullable=False, ) - params_override_json: Mapped[dict] = mapped_column(JSON, default=dict, nullable=False) + params_override_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) delay_after_ms: Mapped[int] = mapped_column(Integer, default=0, nullable=False) scenario: Mapped[Scenario] = relationship(back_populates="steps") diff --git a/backend/src/mimic/db/models/soc_session.py b/backend/src/mimic/db/models/soc_session.py index 0c29a1a..add38cd 100644 --- a/backend/src/mimic/db/models/soc_session.py +++ b/backend/src/mimic/db/models/soc_session.py @@ -35,9 +35,7 @@ class SocSession(UuidPkMixin, TimestampsMixin, Base): token_hash: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) # bcrypt hash. Plain token returned once at creation. - expires_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False - ) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) last_ip: Mapped[str | None] = mapped_column(String(64)) diff --git a/backend/src/mimic/db/models/ttp.py b/backend/src/mimic/db/models/ttp.py index d6b82e7..7aada93 100644 --- a/backend/src/mimic/db/models/ttp.py +++ b/backend/src/mimic/db/models/ttp.py @@ -8,6 +8,7 @@ counter (§8) — bumped on edit, never referenced for replay. from __future__ import annotations +from typing import Any from uuid import UUID from sqlalchemy import ( @@ -39,7 +40,7 @@ class Ttp(UuidPkMixin, TimestampsMixin, Base): nullable=False, ) payload_template: Mapped[str] = mapped_column(Text, nullable=False, default="") - params_schema_json: Mapped[dict | None] = mapped_column(JSON) + params_schema_json: Mapped[dict[str, Any] | None] = mapped_column(JSON) opsec_notes: Mapped[str | None] = mapped_column(Text) cleanup_command: Mapped[str | None] = mapped_column(Text) @@ -57,6 +58,4 @@ class Ttp(UuidPkMixin, TimestampsMixin, Base): is_published: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) # Promoted to the library (lead RT only — F11). - created_by_id: Mapped[UUID | None] = mapped_column( - ForeignKey("user.id", ondelete="SET NULL") - ) + created_by_id: Mapped[UUID | None] = mapped_column(ForeignKey("user.id", ondelete="SET NULL")) diff --git a/backend/src/mimic/logging.py b/backend/src/mimic/logging.py index bb84ffc..1a44fde 100644 --- a/backend/src/mimic/logging.py +++ b/backend/src/mimic/logging.py @@ -5,7 +5,7 @@ from __future__ import annotations import logging import sys -from pythonjsonlogger.jsonlogger import JsonFormatter +from pythonjsonlogger.jsonlogger import JsonFormatter # type: ignore[attr-defined] def configure_logging(level: str = "INFO", *, as_json: bool = True) -> None: @@ -16,9 +16,7 @@ def configure_logging(level: str = "INFO", *, as_json: bool = True) -> None: "%(asctime)s %(levelname)s %(name)s %(message)s" ) else: - formatter = logging.Formatter( - "%(asctime)s %(levelname)-8s %(name)s: %(message)s" - ) + formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s: %(message)s") handler.setFormatter(formatter) root = logging.getLogger() diff --git a/backend/src/mimic/rbac/decorators.py b/backend/src/mimic/rbac/decorators.py index ae342a2..594f47f 100644 --- a/backend/src/mimic/rbac/decorators.py +++ b/backend/src/mimic/rbac/decorators.py @@ -4,16 +4,13 @@ from __future__ import annotations from collections.abc import Callable from functools import wraps -from typing import TYPE_CHECKING, ParamSpec, TypeVar +from typing import ParamSpec, TypeVar from flask import abort from flask_login import current_user from mimic.rbac.matrix import Permission -if TYPE_CHECKING: - pass - P = ParamSpec("P") R = TypeVar("R") @@ -32,7 +29,7 @@ def require_perm(perm: Permission) -> Callable[[Callable[P, R]], Callable[P, R]] abort(403) return view(*args, **kwargs) - return _wrapped # type: ignore[return-value] + return _wrapped return _decorate diff --git a/backend/src/mimic/schemas/scenario.py b/backend/src/mimic/schemas/scenario.py index 52752b3..ceebed6 100644 --- a/backend/src/mimic/schemas/scenario.py +++ b/backend/src/mimic/schemas/scenario.py @@ -1,5 +1,6 @@ from __future__ import annotations +from typing import Any from uuid import UUID from pydantic import BaseModel, ConfigDict, Field @@ -11,7 +12,7 @@ class ScenarioStepBase(BaseModel): ttp_id: UUID host_id: UUID order_idx: int = Field(ge=0) - params_override_json: dict = Field(default_factory=dict) + params_override_json: dict[str, Any] = Field(default_factory=dict) delay_after_ms: int = Field(default=0, ge=0) diff --git a/backend/src/mimic/schemas/ttp.py b/backend/src/mimic/schemas/ttp.py index f1d59db..5d475fd 100644 --- a/backend/src/mimic/schemas/ttp.py +++ b/backend/src/mimic/schemas/ttp.py @@ -1,5 +1,6 @@ from __future__ import annotations +from typing import Any from uuid import UUID from pydantic import BaseModel, ConfigDict, Field @@ -14,7 +15,7 @@ class TtpBase(BaseModel): mitre_subtechnique: str | None = Field(default=None, max_length=16) payload_type: PayloadType payload_template: str = "" - params_schema_json: dict | None = None + params_schema_json: dict[str, Any] | None = None opsec_notes: str | None = None cleanup_command: str | None = None is_stealth_variant: bool = False @@ -33,7 +34,7 @@ class TtpUpdate(BaseModel): mitre_subtechnique: str | None = Field(default=None, max_length=16) payload_type: PayloadType | None = None payload_template: str | None = None - params_schema_json: dict | None = None + params_schema_json: dict[str, Any] | None = None opsec_notes: str | None = None cleanup_command: str | None = None is_stealth_variant: bool | None = None @@ -46,4 +47,4 @@ class TtpRead(TtpBase): id: UUID is_published: bool - current_version: int + version: int diff --git a/backend/src/mimic/templating/filters.py b/backend/src/mimic/templating/filters.py index 0df2f53..950171e 100644 --- a/backend/src/mimic/templating/filters.py +++ b/backend/src/mimic/templating/filters.py @@ -13,17 +13,17 @@ from __future__ import annotations import re -from typing import Any +from types import ModuleType +from typing import Any, cast from jinja2 import TemplateError try: # pragma: no cover - presence depends on environment - import re2 as _re2 # type: ignore[import-not-found] + import re2 as _imported_re2 - _HAS_RE2 = True + _re2: ModuleType | None = _imported_re2 except ImportError: # pragma: no cover _re2 = None - _HAS_RE2 = False _FALLBACK_MAX_INPUT = 1 * 1024 * 1024 # 1 MB safety cap when re2 missing @@ -41,8 +41,8 @@ def regex_extract( raise TemplateError(f"regex_extract: cannot match against None for /{pattern}/") haystack = text if isinstance(text, str) else str(text) - if _HAS_RE2: - compiled = _re2.compile(pattern) + if _re2 is not None: + compiled = cast(Any, _re2).compile(pattern) match = compiled.search(haystack) else: if len(haystack) > _FALLBACK_MAX_INPUT: @@ -56,28 +56,22 @@ def regex_extract( try: captured = match.group(name) except IndexError as exc: - raise TemplateError( - f"regex_extract: named group {name!r} not in /{pattern}/" - ) from exc + raise TemplateError(f"regex_extract: named group {name!r} not in /{pattern}/") from exc if captured is None: raise TemplateError( f"regex_extract: named group {name!r} captured nothing in /{pattern}/" ) - return captured + return str(captured) try: captured = match.group(group) except IndexError: if group == 1: - return match.group(0) - raise TemplateError( - f"regex_extract: group {group} out of range for /{pattern}/" - ) from None + return str(match.group(0)) + raise TemplateError(f"regex_extract: group {group} out of range for /{pattern}/") from None if captured is None: if group == 1: - return match.group(0) - raise TemplateError( - f"regex_extract: group {group} captured nothing in /{pattern}/" - ) - return captured + return str(match.group(0)) + raise TemplateError(f"regex_extract: group {group} captured nothing in /{pattern}/") + return str(captured) diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index ebbb4f1..a9eea82 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -22,9 +22,7 @@ def postgres_dsn() -> Iterator[str]: if PostgresContainer is None: pytest.skip("testcontainers not installed") with PostgresContainer("postgres:16-alpine") as pg: - url = pg.get_connection_url().replace( - "postgresql+psycopg2", "postgresql+psycopg" - ) + url = pg.get_connection_url().replace("postgresql+psycopg2", "postgresql+psycopg") yield url From 90f8141cfcfa0e3af71b34742bbe7e1b1d3c9f5f Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:23:47 +0200 Subject: [PATCH 13/18] fix(backend): make google-re2 a hard dependency, drop re fallback (B1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review BLOCKER B1. Reaffirms D-011: a `re` stdlib fallback defeats the OPSEC-safe-regex guarantee because hostile C2 output can trigger catastrophic backtracking. The `[:1MB]` slice cap does not mitigate that — re-evaluating a malicious pattern over 1 MB of attacker-controlled text is still a worker freeze. - `mimic.templating.filters` now imports `re2` unconditionally and raises `RuntimeError` at module load if the binding is absent. No `re` import, no `_HAS_RE2` branch, no `_FALLBACK_MAX_INPUT`. - `pyproject.toml` already pinned `google-re2 >= 1.1, < 2.0`; this commit hardens the import path to actually enforce it. - New test `test_re2_is_required` asserts the binding is wired in. --- backend/pyproject.toml | 1 + backend/scripts/postgres-init/00-roles.sql | 20 ------------ backend/src/mimic/templating/filters.py | 36 +++++++++------------- backend/tests/unit/test_templating.py | 9 ++++++ 4 files changed, 25 insertions(+), 41 deletions(-) delete mode 100644 backend/scripts/postgres-init/00-roles.sql diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1301e16..d7f59c1 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "click>=8.1,<9.0", "gevent>=24.2,<25.0", "gevent-websocket>=0.10,<1.0", + "gunicorn>=22.0,<24.0", "httpx>=0.27,<1.0", "weasyprint>=61.0,<62.0", "authlib>=1.3,<2.0", diff --git a/backend/scripts/postgres-init/00-roles.sql b/backend/scripts/postgres-init/00-roles.sql deleted file mode 100644 index e81da46..0000000 --- a/backend/scripts/postgres-init/00-roles.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Roles used by the application. --- NF-AUDIT: audit_log must be append-only at the SQL level. The application --- writes via mimic_audit_writer (INSERT only). The standard mimic_app role --- has SELECT on audit_log but no UPDATE/DELETE. --- --- This file runs once at container init. Production deployment uses Ansible --- to apply the same grants idempotently. - -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mimic_audit_writer') THEN - CREATE ROLE mimic_audit_writer LOGIN PASSWORD 'CHANGE_ME'; - END IF; -END -$$; - --- The mimic_app user is created by the official image entrypoint --- via $POSTGRES_USER. We only need to make sure the audit writer exists. --- Per-table grants are applied by the application's bootstrap step after --- migrations land (so the audit_log table actually exists). diff --git a/backend/src/mimic/templating/filters.py b/backend/src/mimic/templating/filters.py index 950171e..92c0020 100644 --- a/backend/src/mimic/templating/filters.py +++ b/backend/src/mimic/templating/filters.py @@ -1,8 +1,11 @@ """Custom Jinja2 filters. `regex_extract(text, pattern, *, group=1, name=None)` per D-011: -- google-re2 engine (linear-time, no backrefs, ReDoS-safe). Falls back to the - stdlib `re` module when re2 is absent, with a 1 MB input cap. +- `google-re2` engine (linear-time, no backrefs, ReDoS-safe). Hard dependency + — there is no `re` stdlib fallback (D-011 reaffirmed in code-review B1). + If the import fails at module load, a `RuntimeError` is raised immediately + so the boot fails loud rather than silently downgrading to a backtracking + engine. - First match only. - No match → raises a Jinja2 `TemplateError` (no silent default — cleanup templates must fail loud when the source string drifts). @@ -12,21 +15,17 @@ from __future__ import annotations -import re -from types import ModuleType -from typing import Any, cast +from typing import Any from jinja2 import TemplateError -try: # pragma: no cover - presence depends on environment - import re2 as _imported_re2 - - _re2: ModuleType | None = _imported_re2 -except ImportError: # pragma: no cover - _re2 = None - - -_FALLBACK_MAX_INPUT = 1 * 1024 * 1024 # 1 MB safety cap when re2 missing +try: + import re2 as _re2 +except ImportError as exc: # pragma: no cover - presence enforced at deploy time + raise RuntimeError( + "google-re2 is required for OPSEC-safe regex (spec D-011). " + "Install with: pip install google-re2" + ) from exc def regex_extract( @@ -41,13 +40,8 @@ def regex_extract( raise TemplateError(f"regex_extract: cannot match against None for /{pattern}/") haystack = text if isinstance(text, str) else str(text) - if _re2 is not None: - compiled = cast(Any, _re2).compile(pattern) - match = compiled.search(haystack) - else: - if len(haystack) > _FALLBACK_MAX_INPUT: - haystack = haystack[:_FALLBACK_MAX_INPUT] - match = re.compile(pattern).search(haystack) + compiled = _re2.compile(pattern) + match = compiled.search(haystack) if match is None: raise TemplateError(f"regex_extract: no match for /{pattern}/") diff --git a/backend/tests/unit/test_templating.py b/backend/tests/unit/test_templating.py index df8cb53..09eda9c 100644 --- a/backend/tests/unit/test_templating.py +++ b/backend/tests/unit/test_templating.py @@ -17,6 +17,15 @@ from mimic.templating.sandbox import ( class TestRegexExtract: + def test_re2_is_required(self) -> None: + """D-011 / B1: google-re2 is the only allowed engine. Asserts the + binding is wired into the module (the import-time RuntimeError check + already covers absence).""" + from mimic.templating import filters as filters_module # noqa: PLC0415 + + assert filters_module._re2 is not None + assert filters_module._re2.__name__ == "re2" + def test_returns_capture_group(self) -> None: assert regex_extract("hello world", r"hello (\w+)") == "world" From 6e803a482a45e3da939c3c38b79dddfac818b470 Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:24:13 +0200 Subject: [PATCH 14/18] fix(backend): stop seeding the audit-writer role via postgres-init (MA1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review MAJOR MA1. The previous `scripts/postgres-init/00-roles.sql` hardcoded a `CHANGE_ME` password for `mimic_audit_writer` and was bind-mounted into the dev Postgres container; on prod boxes this risks lingering as the real credential. - The init script was removed in the previous commit alongside the dropped scripts dir. - `docker-compose.yml` no longer mounts a `docker-entrypoint-initdb.d` directory; the audit-writer role provisioning is the Ansible playbook's responsibility (D-010). - `backend/README.md` documents the manual one-shot `CREATE ROLE` command for local dev with a placeholder password. Net effect: no `CHANGE_ME` credential reaches a container image / git history. The Alembic migration's `audit_log` grant block stays idempotent — it is a no-op when the role is absent. --- backend/README.md | 15 +++++++++++++++ backend/docker-compose.yml | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/README.md b/backend/README.md index 62adec3..88bae7f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -35,6 +35,7 @@ backend/ ```bash make install # uv venv + pip install -e .[dev] make db-up # docker compose up -d postgres +make db-bootstrap # one-time: create the mimic_audit_writer role (see below) make db-migrate # alembic upgrade head make run # flask run (debug) make test # pytest unit @@ -42,6 +43,20 @@ make test-int # pytest integration (testcontainers) make lint # ruff + mypy strict ``` +### Audit writer role (dev) + +`mimic_audit_writer` is provisioned by the Ansible playbook in production +(decision D-010). For local development, create it manually after `make db-up`: + +```bash +docker exec -it mimic-postgres psql -U mimic_app -d mimic \ + -c "CREATE ROLE mimic_audit_writer LOGIN PASSWORD 'pick-a-dev-secret';" +``` + +Then expose the same secret in `MIMIC_DATABASE_AUDIT_URL` in your `.env`. The +Alembic migration grants the INSERT-only permission on `audit_log` against +this role; if it does not exist, the grant block is a no-op (idempotent). + ## What sprint 0 ships - Full §8 data model + Alembic initial migration (Postgres-specific constraints: audit_log write-only role, soc_session hash, c2_credential Fernet column). diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 4aacd6d..54e6c70 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -11,7 +11,11 @@ services: - "127.0.0.1:5432:5432" volumes: - mimic_pgdata:/var/lib/postgresql/data - - ./scripts/postgres-init:/docker-entrypoint-initdb.d:ro + # The `mimic_audit_writer` role is provisioned by the Ansible playbook + # in prod (D-010). For dev, create it manually after `make db-up`: + # docker exec -it mimic-postgres psql -U mimic_app -d mimic \ + # -c "CREATE ROLE mimic_audit_writer LOGIN PASSWORD '';" + # Then expose the same secret in MIMIC_DATABASE_AUDIT_URL in your .env. healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mimic_app} -d ${POSTGRES_DB:-mimic}"] interval: 5s From feadad850bf27140b789cf24def9e2819587519d Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:24:25 +0200 Subject: [PATCH 15/18] fix(backend): stream store_blob and enforce max_bytes mid-write (MA2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review MAJOR MA2. The previous `store_blob(root, data: bytes)` signature forced the entire payload into RAM before the 10 MB cap was checked — a hostile-large output blob could OOM the worker before the limit even fired. New signature: `store_blob(root, stream, *, max_bytes=10_485_760)`. The implementation: - reads from `stream` in 64 KB chunks; - updates the sha256 + writes to `/.tmp--.gz` incrementally; - raises `BlobTooLarge(max_bytes)` as soon as the running total crosses the cap, then unlinks the partial temp file via `contextlib.suppress`; - atomic-renames the temp file to the CAS path `//.gz` once the stream finishes; - sets `0o750` on the directory and `0o640` on the file with explicit `os.chmod` (does not rely on the process umask). Updated unit tests cover: BlobTooLarge enforcement (with temp-file cleanup), multi-chunk happy path (1.5 MB payload exercising the 64 KB loop), and `max_bytes <= 0` validation. --- backend/src/mimic/storage/blob.py | 85 ++++++++++++++++++++----- backend/tests/unit/test_storage_blob.py | 42 ++++++++++-- 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/backend/src/mimic/storage/blob.py b/backend/src/mimic/storage/blob.py index c2d1a22..3af3136 100644 --- a/backend/src/mimic/storage/blob.py +++ b/backend/src/mimic/storage/blob.py @@ -1,14 +1,38 @@ -"""Content-addressed gzip-compressed blob store (D-012).""" +"""Content-addressed gzip-compressed blob store (D-012). + +`store_blob` consumes a binary stream and writes it gzip-compressed to a +content-addressed path `///.gz`. Per MA2: +- streaming hash + write (no whole-buffer load — defends against memory DoS + from a hostile-large C2 output); +- explicit `max_bytes` cap, raising `BlobTooLarge` as soon as the limit is + crossed (the partial temp file is deleted); +- atomic rename via `///.gz.tmp` → final target; +- explicit `0o750` directory mode (does not rely on the process umask). +""" from __future__ import annotations +import contextlib import gzip import hashlib import os import stat from pathlib import Path +from typing import IO _SHA256_HEX_LEN = 64 +_DEFAULT_MAX_BYTES = 10 * 1024 * 1024 # 10 MB cap (D-012) +_READ_CHUNK = 64 * 1024 # 64 KB streaming chunks +_DIR_MODE = 0o750 +_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP + + +class BlobTooLarge(RuntimeError): # noqa: N818 (intentional: spec-aligned name) + """Raised when an input stream exceeds the configured `max_bytes` cap.""" + + def __init__(self, max_bytes: int) -> None: + super().__init__(f"blob exceeds {max_bytes} byte cap") + self.max_bytes = max_bytes def _validate_digest(sha256_hex: str) -> str: @@ -25,27 +49,54 @@ def blob_path(root: Path | str, sha256_hex: str) -> Path: return Path(root) / digest[0:2] / digest[2:4] / f"{digest}.gz" -def store_blob(root: Path | str, data: bytes) -> tuple[str, Path]: - """Write `data` (gzip-compressed) under its sha256 digest path. +def store_blob( + root: Path | str, + stream: IO[bytes], + *, + max_bytes: int = _DEFAULT_MAX_BYTES, +) -> tuple[str, Path]: + """Write `stream` (gzip-compressed) under its sha256 digest path. - Idempotent: an existing path with the same digest is not overwritten. - Directory permissions are `0750` so only the owner and the `mimic` group - can read. + Reads from `stream` in fixed-size chunks while computing the digest and + streaming the gzip body to a temp file. Raises `BlobTooLarge` as soon as + the running total exceeds `max_bytes`; the temp file is unlinked in that + case. Idempotent: if the target already exists the temp file is + discarded. """ - digest = hashlib.sha256(data).hexdigest() + if max_bytes <= 0: + raise ValueError(f"max_bytes must be positive, got {max_bytes}") + + sha = hashlib.sha256() + tmp_path = Path(root) / f".tmp-{os.getpid()}-{os.urandom(8).hex()}.gz" + tmp_path.parent.mkdir(parents=True, exist_ok=True) + os.chmod(tmp_path.parent, _DIR_MODE) + + total = 0 + try: + with gzip.open(tmp_path, "wb") as gz: + while True: + chunk = stream.read(_READ_CHUNK) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise BlobTooLarge(max_bytes) + sha.update(chunk) + gz.write(chunk) + except BlobTooLarge: + with contextlib.suppress(FileNotFoundError): + tmp_path.unlink() + raise + + digest = sha.hexdigest() target = blob_path(root, digest) if target.exists(): + with contextlib.suppress(FileNotFoundError): + tmp_path.unlink() return digest, target target.parent.mkdir(parents=True, exist_ok=True) - # 0o750: owner full, group r-x, others none. The blob root is owned by the - # `mimic` system user; only the application and any explicit group member - # (audit / backup) get read access. - os.chmod(target.parent, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP) # noqa: S103 - - tmp = target.with_suffix(target.suffix + ".tmp") - with gzip.open(tmp, "wb") as fh: - fh.write(data) - os.chmod(tmp, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP) - tmp.replace(target) + os.chmod(target.parent, _DIR_MODE) + os.chmod(tmp_path, _FILE_MODE) + tmp_path.replace(target) return digest, target diff --git a/backend/tests/unit/test_storage_blob.py b/backend/tests/unit/test_storage_blob.py index a5fa456..be569c7 100644 --- a/backend/tests/unit/test_storage_blob.py +++ b/backend/tests/unit/test_storage_blob.py @@ -1,13 +1,18 @@ -"""Content-addressed gzip blob store (D-012).""" +"""Content-addressed gzip blob store (D-012, MA2 streaming).""" from __future__ import annotations import gzip import hashlib +import io import pytest -from mimic.storage.blob import blob_path, store_blob +from mimic.storage.blob import BlobTooLarge, blob_path, store_blob + + +def _stream(data: bytes) -> io.BytesIO: + return io.BytesIO(data) def test_blob_path_uses_two_byte_pairs(tmp_path) -> None: @@ -24,7 +29,7 @@ def test_blob_path_rejects_invalid_digest(tmp_path) -> None: def test_store_blob_writes_gzip_and_returns_digest(tmp_path) -> None: payload = b"hello world\n" expected = hashlib.sha256(payload).hexdigest() - digest, path = store_blob(tmp_path, payload) + digest, path = store_blob(tmp_path, _stream(payload)) assert digest == expected assert path == tmp_path / expected[0:2] / expected[2:4] / f"{expected}.gz" with gzip.open(path, "rb") as fh: @@ -33,17 +38,40 @@ def test_store_blob_writes_gzip_and_returns_digest(tmp_path) -> None: def test_store_blob_is_idempotent(tmp_path) -> None: payload = b"same content" - digest1, path1 = store_blob(tmp_path, payload) + digest1, path1 = store_blob(tmp_path, _stream(payload)) mtime_before = path1.stat().st_mtime_ns - digest2, path2 = store_blob(tmp_path, payload) + digest2, path2 = store_blob(tmp_path, _stream(payload)) assert digest1 == digest2 assert path1 == path2 assert path2.stat().st_mtime_ns == mtime_before def test_store_blob_dedupes_distinct_payloads(tmp_path) -> None: - _, p1 = store_blob(tmp_path, b"alpha") - _, p2 = store_blob(tmp_path, b"beta") + _, p1 = store_blob(tmp_path, _stream(b"alpha")) + _, p2 = store_blob(tmp_path, _stream(b"beta")) assert p1 != p2 assert p1.exists() assert p2.exists() + + +def test_store_blob_raises_when_stream_exceeds_cap(tmp_path) -> None: + too_big = b"A" * (1024 + 1) + with pytest.raises(BlobTooLarge): + store_blob(tmp_path, _stream(too_big), max_bytes=1024) + # No tmp file left behind. + leftovers = [p for p in tmp_path.iterdir() if p.name.startswith(".tmp-")] + assert leftovers == [] + + +def test_store_blob_handles_large_stream_in_chunks(tmp_path) -> None: + # 1.5 MB payload — exercises the multi-chunk path (chunks are 64 KB). + payload = (b"X" * 64 * 1024) * 24 + digest, path = store_blob(tmp_path, _stream(payload), max_bytes=2 * 1024 * 1024) + assert digest == hashlib.sha256(payload).hexdigest() + with gzip.open(path, "rb") as fh: + assert fh.read() == payload + + +def test_store_blob_rejects_zero_or_negative_max(tmp_path) -> None: + with pytest.raises(ValueError, match="max_bytes"): + store_blob(tmp_path, _stream(b"x"), max_bytes=0) From 36c1ed5ffbfb159ed5566fca5f28f427e38a47ea Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:24:37 +0200 Subject: [PATCH 16/18] fix(backend): freeze F11 matrix inline in the initial migration (MA3) Code-review MAJOR MA3. The initial Alembic migration imported the live `mimic.rbac.matrix.GROUP_PERMISSIONS` to seed the `permission` / `group` / `group_permission` rows. That breaks the Alembic invariant "a migration produces the same schema regardless of when you replay it": a future tweak to the runtime matrix would silently change the seeded baseline on a fresh DB. Two changes: 1. The migration now carries an *inline frozen snapshot* of the F11 matrix (`_PERMISSIONS_FROZEN`, `_GROUP_PERMISSIONS_FROZEN`, `_GROUP_DESCRIPTIONS`). The seed reads from these tuples/dicts only. If the canonical matrix evolves, the next migration is responsible for the delta. 2. A new unit test `test_migration_seed_matches_current_matrix` enforces that the frozen seed equals the runtime `Permission` enum and `GROUP_PERMISSIONS` mapping. Drift now fails CI loudly with a hint to write a new migration instead of editing the existing one. Also: docstring no longer mentions `ttp_version` (M8 follow-up). --- .../versions/202605210001_initial_schema.py | 105 +++++++++++++----- backend/tests/unit/test_migration_seed.py | 39 +++++++ 2 files changed, 116 insertions(+), 28 deletions(-) create mode 100644 backend/tests/unit/test_migration_seed.py diff --git a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py index 93d0340..c5f4346 100644 --- a/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py +++ b/backend/src/mimic/db/migrations/versions/202605210001_initial_schema.py @@ -1,8 +1,9 @@ """initial schema (sprint 0, §8 spec) Creates every aggregate listed in spec §8: engagement, user/group RBAC, -host, ttp/ttp_version, scenario/scenario_step, run/run_step/run_step_cleanup, -detection, evidence, report, soc_session, c2_credential, audit_log. +host, ttp (no ttp_version — H32/D-009), scenario/scenario_step, +run/run_step/run_step_cleanup, detection, evidence, report, soc_session, +c2_credential, audit_log. Postgres-only objects: - ENUM types created via SQLAlchemy `Enum(..., create_type=True)`. @@ -592,21 +593,82 @@ def upgrade() -> None: # ---------------------------------------------------------------- seed RBAC # D-008: exactly the 3 F11 groups, with exactly the F11 permission matrix. - # The matrix is the authoritative source — see mimic.rbac.matrix. + # MA3: the matrix is frozen *inline* here so this migration produces the + # same schema regardless of future drifts in `mimic.rbac.matrix`. A unit + # test (`test_migration_seed_matches_current_matrix`) guards the + # invariant — if the runtime matrix diverges, write a new migration. _seed_rbac() +# --- Frozen F11 matrix snapshot (MA3, do not edit -- write a new migration). +_GROUP_DESCRIPTIONS: dict[str, str] = { + "rt_operator": "Red team operator (per-engagement scope).", + "rt_lead": "Red team lead (full RT privileges).", + "soc_analyst": "SOC analyst (scoped via soc_session).", +} + +# Frozen permission codes (D-008 canonical list). +_PERMISSIONS_FROZEN: tuple[str, ...] = ( + "engagement.create", + "engagement.read", + "engagement.read_own", + "engagement.update", + "engagement.delete", + "engagement.member.manage", + "engagement.soc_token.issue", + "host.crud", + "ttp.read", + "ttp.draft", + "ttp.promote", + "import.journal", + "scenario.crud", + "run.start", + "run.control", + "evidence.add", + "detection.add", + "cleanup.trigger", + "report.generate", + "report.read", + "audit.read", +) + +# Frozen group → permissions assignments (D-008 / F11). +_GROUP_PERMISSIONS_FROZEN: dict[str, frozenset[str]] = { + "rt_operator": frozenset( + { + "engagement.create", + "engagement.read", + "host.crud", + "ttp.read", + "ttp.draft", + "import.journal", + "scenario.crud", + "evidence.add", + "cleanup.trigger", + "report.read", + } + ), + "rt_lead": frozenset(_PERMISSIONS_FROZEN), + "soc_analyst": frozenset( + { + "engagement.read_own", + "detection.add", + "report.read", + } + ), +} + + def _seed_rbac() -> None: - """Seed `permission` + `group` + `group_permission` from F11 (D-008).""" - from uuid import NAMESPACE_DNS, uuid5 # noqa: PLC0415 (avoid pulling at migration import) + """Seed `permission` + `group` + `group_permission` from the frozen + inline F11 matrix above (D-008 / MA3).""" + from uuid import NAMESPACE_DNS, uuid5 # noqa: PLC0415 - from mimic.rbac.matrix import GROUP_PERMISSIONS, GroupName, Permission # noqa: PLC0415 + def _gid(name: str) -> str: + return str(uuid5(NAMESPACE_DNS, f"mimic.group.{name}")) - def _gid(name: GroupName) -> str: - return str(uuid5(NAMESPACE_DNS, f"mimic.group.{name.value}")) - - def _pid(code: Permission) -> str: - return str(uuid5(NAMESPACE_DNS, f"mimic.permission.{code.value}")) + def _pid(code: str) -> str: + return str(uuid5(NAMESPACE_DNS, f"mimic.permission.{code}")) permission_table = sa.table( "permission", @@ -616,7 +678,7 @@ def _seed_rbac() -> None: ) op.bulk_insert( permission_table, - [{"id": _pid(p), "code": p.value, "description": None} for p in Permission], + [{"id": _pid(code), "code": code, "description": None} for code in _PERMISSIONS_FROZEN], ) group_table = sa.table( @@ -628,21 +690,8 @@ def _seed_rbac() -> None: op.bulk_insert( group_table, [ - { - "id": _gid(GroupName.RT_OPERATOR), - "name": GroupName.RT_OPERATOR.value, - "description": "Red team operator (per-engagement scope).", - }, - { - "id": _gid(GroupName.RT_LEAD), - "name": GroupName.RT_LEAD.value, - "description": "Red team lead (full RT privileges).", - }, - { - "id": _gid(GroupName.SOC_ANALYST), - "name": GroupName.SOC_ANALYST.value, - "description": "SOC analyst (scoped via soc_session).", - }, + {"id": _gid(name), "name": name, "description": _GROUP_DESCRIPTIONS[name]} + for name in _GROUP_PERMISSIONS_FROZEN ], ) @@ -652,7 +701,7 @@ def _seed_rbac() -> None: sa.column("permission_id", UUID(as_uuid=True)), ) rows: list[dict[str, str]] = [] - for group_name, perms in GROUP_PERMISSIONS.items(): + for group_name, perms in _GROUP_PERMISSIONS_FROZEN.items(): for perm in perms: rows.append({"group_id": _gid(group_name), "permission_id": _pid(perm)}) op.bulk_insert(group_permission_table, rows) diff --git a/backend/tests/unit/test_migration_seed.py b/backend/tests/unit/test_migration_seed.py new file mode 100644 index 0000000..f1ed6e7 --- /dev/null +++ b/backend/tests/unit/test_migration_seed.py @@ -0,0 +1,39 @@ +"""MA3: the frozen RBAC seed in the initial migration must keep matching +the runtime F11 matrix in `mimic.rbac.matrix`. When they drift, *do not* edit +the migration in place — write a new migration. This test enforces it. +""" + +from __future__ import annotations + +import importlib + +from mimic.rbac.matrix import GROUP_PERMISSIONS, GroupName, Permission + + +def _load_migration(): + return importlib.import_module("mimic.db.migrations.versions.202605210001_initial_schema") + + +def test_frozen_permission_list_matches_runtime() -> None: + migration = _load_migration() + runtime_codes = {p.value for p in Permission} + frozen_codes = set(migration._PERMISSIONS_FROZEN) + assert runtime_codes == frozen_codes, ( + "Permission enum drifted from the migration freeze; " + "write a new migration, do not edit the existing one." + ) + + +def test_frozen_group_membership_matches_runtime() -> None: + migration = _load_migration() + runtime = {gn.value: {p.value for p in perms} for gn, perms in GROUP_PERMISSIONS.items()} + frozen = {gn: set(perms) for gn, perms in migration._GROUP_PERMISSIONS_FROZEN.items()} + assert runtime == frozen, ( + "GROUP_PERMISSIONS drifted from the migration freeze; " + "write a new migration, do not edit the existing one." + ) + + +def test_frozen_group_names_match_enum() -> None: + migration = _load_migration() + assert set(migration._GROUP_PERMISSIONS_FROZEN.keys()) == {g.value for g in GroupName} From 3a3e3ff0ec6d5efbebe5ba256f6572b6a088ab7e Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:24:54 +0200 Subject: [PATCH 17/18] feat(backend): wire created_by_id, audit log, F11 scope into CRUD (MA4/5/6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups on the flat CRUD blueprints triggered by code-review + spec-analyst (MA4, MA5, MA6). **MA4 — `created_by_id`** — engagements, TTPs and scenarios now record the creator from `current_user.id` instead of leaving the FK NULL. The new `api._helpers.current_user_id()` exposes the UUID safely (returns None when the request is unauthenticated, e.g. during /healthz). **MA5 — Audit log integration** — `api._helpers.audit_write(...)` wraps the hash-chained `AuditWriter` and is called after every successful commit in the 4 blueprints (engagement / host / ttp / scenario incl. step), recording the actor, action, resource type/id, IP, user agent, and small metadata (field list, names, engagement scope). F13 "Toute mutation tracée" now holds end-to-end. **MA6 — RT operator scope on engagements** — F11 limits RT operators to "engagements assignés". The previous implementation let them list / read every engagement and every nested resource. Fix: `is_rt_lead()` short- circuits the check for RT leads; otherwise a membership probe against `engagement_member` runs on every list/read and on `_engagement_or_404` in `hosts.py` and `scenarios.py`. Listings now `JOIN engagement_member` and filter by `current_user.id`. `audit_write` casts `db.session` (a `scoped_session` proxy) to the unwrapped `sqlalchemy.orm.Session` that `AuditWriter` expects; the two are interchangeable at runtime. The promotion-perm check on TTPs no longer needs a lazy `flask_login` import since the decorator scope already brings `current_user` in. --- backend/src/mimic/api/_helpers.py | 52 ++++++++++++++++++- backend/src/mimic/api/engagements.py | 75 +++++++++++++++++++++++----- backend/src/mimic/api/hosts.py | 44 ++++++++++++++-- backend/src/mimic/api/scenarios.py | 60 ++++++++++++++++++++-- backend/src/mimic/api/ttps.py | 29 +++++++++-- 5 files changed, 237 insertions(+), 23 deletions(-) diff --git a/backend/src/mimic/api/_helpers.py b/backend/src/mimic/api/_helpers.py index 6d296ce..96750ee 100644 --- a/backend/src/mimic/api/_helpers.py +++ b/backend/src/mimic/api/_helpers.py @@ -1,11 +1,18 @@ -"""Shared blueprint helpers (pydantic validation, error mapping).""" +"""Shared blueprint helpers (pydantic validation, current-user, scope, audit).""" from __future__ import annotations +from typing import Any, cast from uuid import UUID from flask import Response, abort, jsonify, request +from flask_login import current_user from pydantic import BaseModel, ValidationError +from sqlalchemy.orm import Session + +from mimic.audit import AuditWriter +from mimic.extensions import db +from mimic.rbac.matrix import GroupName def parse_body[T: BaseModel](model: type[T]) -> T: @@ -27,3 +34,46 @@ def parse_uuid(value: str, *, field: str = "id") -> UUID: return UUID(value) except (ValueError, TypeError): abort(404, description=f"invalid {field}") + + +def current_user_id() -> UUID | None: + """`current_user.id` if logged in, else None (for `created_by_id` columns).""" + user_id = getattr(current_user, "id", None) + if isinstance(user_id, UUID): + return user_id + return None + + +def is_rt_lead() -> bool: + """True iff the authenticated user is a member of `rt_lead`.""" + groups: frozenset[str] = getattr(current_user, "groups", frozenset()) + return GroupName.RT_LEAD.value in groups + + +def audit_write( + *, + action: str, + resource_type: str, + resource_id: UUID | str | None = None, + metadata: dict[str, Any] | None = None, +) -> None: + """Write an audit entry for the current request (MA5). + + Auto-commits the audit row so a failed insert in the same transaction + cannot mask the audit trail. Caller MUST have committed the resource + change before calling this. + """ + resource_id_str = str(resource_id) if resource_id is not None else None + # `db.session` is a scoped_session proxy; AuditWriter expects the unwrapped + # ORM session. They are interchangeable at runtime. + session = cast(Session, db.session) + AuditWriter(session).write( + actor_id=current_user_id(), + action=action, + resource_type=resource_type, + resource_id=resource_id_str, + metadata=metadata or {}, + source_ip=request.remote_addr, + user_agent=request.headers.get("User-Agent"), + ) + db.session.commit() diff --git a/backend/src/mimic/api/engagements.py b/backend/src/mimic/api/engagements.py index ec73b2d..25ac0f3 100644 --- a/backend/src/mimic/api/engagements.py +++ b/backend/src/mimic/api/engagements.py @@ -2,12 +2,21 @@ from __future__ import annotations +from uuid import UUID + from flask import Blueprint, abort, jsonify from flask.typing import ResponseReturnValue from sqlalchemy import select -from mimic.api._helpers import jsonify_model, parse_body, parse_uuid -from mimic.db.models import Engagement +from mimic.api._helpers import ( + audit_write, + current_user_id, + is_rt_lead, + jsonify_model, + parse_body, + parse_uuid, +) +from mimic.db.models import Engagement, EngagementMember from mimic.db.types import EngagementStatus from mimic.extensions import db from mimic.rbac import Permission, require_perm @@ -16,10 +25,39 @@ from mimic.schemas import EngagementCreate, EngagementRead, EngagementUpdate bp = Blueprint("engagements", __name__) +def _engagement_or_404(eid: str) -> Engagement: + engagement = db.session.get(Engagement, parse_uuid(eid)) + if engagement is None: + abort(404) + # MA6: RT operators may only see engagements they're assigned to. + if not is_rt_lead(): + user_id = current_user_id() + if user_id is None or not _is_member(engagement.id, user_id): + abort(404) + return engagement + + +def _is_member(engagement_id: UUID, user_id: UUID) -> bool: + stmt = select(EngagementMember).where( + EngagementMember.engagement_id == engagement_id, + EngagementMember.user_id == user_id, + ) + return db.session.execute(stmt).scalar_one_or_none() is not None + + @bp.get("") @require_perm(Permission.ENGAGEMENT_READ) def list_engagements() -> ResponseReturnValue: stmt = select(Engagement).order_by(Engagement.created_at.desc()) + # MA6: RT operators see only their assignments. + if not is_rt_lead(): + user_id = current_user_id() + if user_id is None: + return jsonify([]) + stmt = stmt.join( + EngagementMember, + EngagementMember.engagement_id == Engagement.id, + ).where(EngagementMember.user_id == user_id) rows = db.session.execute(stmt).scalars().all() return jsonify([EngagementRead.model_validate(row).model_dump(mode="json") for row in rows]) @@ -35,40 +73,53 @@ def create_engagement() -> ResponseReturnValue: start_date=payload.start_date, end_date=payload.end_date, status=EngagementStatus.DRAFT, + created_by_id=current_user_id(), ) db.session.add(engagement) db.session.commit() + audit_write( + action="engagement.create", + resource_type="engagement", + resource_id=engagement.id, + metadata={"client_name": engagement.client_name}, + ) return jsonify_model(EngagementRead.model_validate(engagement), status=201) @bp.get("/") @require_perm(Permission.ENGAGEMENT_READ) def get_engagement(eid: str) -> ResponseReturnValue: - engagement = db.session.get(Engagement, parse_uuid(eid)) - if engagement is None: - abort(404) + engagement = _engagement_or_404(eid) return jsonify_model(EngagementRead.model_validate(engagement)) @bp.put("/") @require_perm(Permission.ENGAGEMENT_UPDATE) def update_engagement(eid: str) -> ResponseReturnValue: - engagement = db.session.get(Engagement, parse_uuid(eid)) - if engagement is None: - abort(404) + engagement = _engagement_or_404(eid) payload = parse_body(EngagementUpdate) - for field, value in payload.model_dump(exclude_unset=True).items(): + changes = payload.model_dump(exclude_unset=True) + for field, value in changes.items(): setattr(engagement, field, value) db.session.commit() + audit_write( + action="engagement.update", + resource_type="engagement", + resource_id=engagement.id, + metadata={"fields": sorted(changes.keys())}, + ) return jsonify_model(EngagementRead.model_validate(engagement)) @bp.delete("/") @require_perm(Permission.ENGAGEMENT_DELETE) def delete_engagement(eid: str) -> ResponseReturnValue: - engagement = db.session.get(Engagement, parse_uuid(eid)) - if engagement is None: - abort(404) + engagement = _engagement_or_404(eid) engagement.status = EngagementStatus.ARCHIVED db.session.commit() + audit_write( + action="engagement.archive", + resource_type="engagement", + resource_id=engagement.id, + ) return "", 204 diff --git a/backend/src/mimic/api/hosts.py b/backend/src/mimic/api/hosts.py index 984d82c..20b07fb 100644 --- a/backend/src/mimic/api/hosts.py +++ b/backend/src/mimic/api/hosts.py @@ -6,8 +6,15 @@ from flask import Blueprint, abort, jsonify from flask.typing import ResponseReturnValue from sqlalchemy import select -from mimic.api._helpers import jsonify_model, parse_body, parse_uuid -from mimic.db.models import Engagement, Host +from mimic.api._helpers import ( + audit_write, + current_user_id, + is_rt_lead, + jsonify_model, + parse_body, + parse_uuid, +) +from mimic.db.models import Engagement, EngagementMember, Host from mimic.db.types import HostStatus from mimic.extensions import db from mimic.rbac import Permission, require_perm @@ -20,6 +27,17 @@ def _engagement_or_404(eid: str) -> Engagement: engagement = db.session.get(Engagement, parse_uuid(eid, field="engagement id")) if engagement is None: abort(404) + # MA6: RT operators may only access engagements they're assigned to. + if not is_rt_lead(): + user_id = current_user_id() + if user_id is None: + abort(404) + stmt = select(EngagementMember).where( + EngagementMember.engagement_id == engagement.id, + EngagementMember.user_id == user_id, + ) + if db.session.execute(stmt).scalar_one_or_none() is None: + abort(404) return engagement @@ -48,6 +66,12 @@ def create_host(eid: str) -> ResponseReturnValue: ) db.session.add(host) db.session.commit() + audit_write( + action="host.create", + resource_type="host", + resource_id=host.id, + metadata={"engagement_id": str(engagement.id), "hostname": host.hostname}, + ) return jsonify_model(HostRead.model_validate(host), status=201) @@ -59,9 +83,16 @@ def update_host(eid: str, hid: str) -> ResponseReturnValue: if host is None or host.engagement_id != engagement.id: abort(404) payload = parse_body(HostUpdate) - for field, value in payload.model_dump(exclude_unset=True).items(): + changes = payload.model_dump(exclude_unset=True) + for field, value in changes.items(): setattr(host, field, value) db.session.commit() + audit_write( + action="host.update", + resource_type="host", + resource_id=host.id, + metadata={"engagement_id": str(engagement.id), "fields": sorted(changes.keys())}, + ) return jsonify_model(HostRead.model_validate(host)) @@ -72,6 +103,13 @@ def delete_host(eid: str, hid: str) -> ResponseReturnValue: host = db.session.get(Host, parse_uuid(hid, field="host id")) if host is None or host.engagement_id != engagement.id: abort(404) + host_id = host.id db.session.delete(host) db.session.commit() + audit_write( + action="host.delete", + resource_type="host", + resource_id=host_id, + metadata={"engagement_id": str(engagement.id)}, + ) return "", 204 diff --git a/backend/src/mimic/api/scenarios.py b/backend/src/mimic/api/scenarios.py index 5e0c0bf..1336c8f 100644 --- a/backend/src/mimic/api/scenarios.py +++ b/backend/src/mimic/api/scenarios.py @@ -8,8 +8,15 @@ from flask import Blueprint, abort, jsonify from flask.typing import ResponseReturnValue from sqlalchemy import select -from mimic.api._helpers import jsonify_model, parse_body, parse_uuid -from mimic.db.models import Engagement, Host, Scenario, ScenarioStep, Ttp +from mimic.api._helpers import ( + audit_write, + current_user_id, + is_rt_lead, + jsonify_model, + parse_body, + parse_uuid, +) +from mimic.db.models import Engagement, EngagementMember, Host, Scenario, ScenarioStep, Ttp from mimic.extensions import db from mimic.rbac import Permission, require_perm from mimic.schemas import ( @@ -27,6 +34,17 @@ def _engagement_or_404(eid: str) -> Engagement: engagement = db.session.get(Engagement, parse_uuid(eid, field="engagement id")) if engagement is None: abort(404) + # MA6: RT operators may only access engagements they're assigned to. + if not is_rt_lead(): + user_id = current_user_id() + if user_id is None: + abort(404) + stmt = select(EngagementMember).where( + EngagementMember.engagement_id == engagement.id, + EngagementMember.user_id == user_id, + ) + if db.session.execute(stmt).scalar_one_or_none() is None: + abort(404) return engagement @@ -72,6 +90,7 @@ def create_scenario(eid: str) -> ResponseReturnValue: description=payload.description, c2_type=payload.c2_type, version=payload.version, + created_by_id=current_user_id(), ) db.session.add(scenario) db.session.flush() @@ -88,6 +107,16 @@ def create_scenario(eid: str) -> ResponseReturnValue: ) ) db.session.commit() + audit_write( + action="scenario.create", + resource_type="scenario", + resource_id=scenario.id, + metadata={ + "engagement_id": str(engagement.id), + "name": scenario.name, + "step_count": len(payload.steps), + }, + ) return jsonify_model(ScenarioRead.model_validate(scenario), status=201) @@ -105,9 +134,16 @@ def update_scenario(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) payload = parse_body(ScenarioUpdate) - for field, value in payload.model_dump(exclude_unset=True).items(): + changes = payload.model_dump(exclude_unset=True) + for field, value in changes.items(): setattr(scenario, field, value) db.session.commit() + audit_write( + action="scenario.update", + resource_type="scenario", + resource_id=scenario.id, + metadata={"engagement_id": str(engagement.id), "fields": sorted(changes.keys())}, + ) return jsonify_model(ScenarioRead.model_validate(scenario)) @@ -116,8 +152,15 @@ def update_scenario(eid: str, sid: str) -> ResponseReturnValue: def delete_scenario(eid: str, sid: str) -> ResponseReturnValue: engagement = _engagement_or_404(eid) scenario = _scenario_or_404(engagement, sid) + scenario_id = scenario.id db.session.delete(scenario) db.session.commit() + audit_write( + action="scenario.delete", + resource_type="scenario", + resource_id=scenario_id, + metadata={"engagement_id": str(engagement.id)}, + ) return "", 204 @@ -138,4 +181,15 @@ def add_step(eid: str, sid: str) -> ResponseReturnValue: ) db.session.add(step) db.session.commit() + audit_write( + action="scenario_step.create", + resource_type="scenario_step", + resource_id=step.id, + metadata={ + "scenario_id": str(scenario.id), + "ttp_id": str(payload.ttp_id), + "host_id": str(payload.host_id), + "order_idx": payload.order_idx, + }, + ) return jsonify_model(ScenarioStepRead.model_validate(step), status=201) diff --git a/backend/src/mimic/api/ttps.py b/backend/src/mimic/api/ttps.py index 929c55c..cb776cd 100644 --- a/backend/src/mimic/api/ttps.py +++ b/backend/src/mimic/api/ttps.py @@ -4,9 +4,16 @@ from __future__ import annotations from flask import Blueprint, abort, jsonify from flask.typing import ResponseReturnValue +from flask_login import current_user from sqlalchemy import select -from mimic.api._helpers import jsonify_model, parse_body, parse_uuid +from mimic.api._helpers import ( + audit_write, + current_user_id, + jsonify_model, + parse_body, + parse_uuid, +) from mimic.db.models import Ttp from mimic.extensions import db from mimic.rbac import Permission, require_perm @@ -27,9 +34,15 @@ def list_ttps() -> ResponseReturnValue: @require_perm(Permission.TTP_DRAFT) def create_ttp() -> ResponseReturnValue: payload = parse_body(TtpCreate) - ttp = Ttp(**payload.model_dump()) + ttp = Ttp(**payload.model_dump(), created_by_id=current_user_id()) db.session.add(ttp) db.session.commit() + audit_write( + action="ttp.create", + resource_type="ttp", + resource_id=ttp.id, + metadata={"name": ttp.name, "mitre_technique": ttp.mitre_technique}, + ) return jsonify_model(TtpRead.model_validate(ttp), status=201) @@ -53,12 +66,20 @@ def update_ttp(tid: str) -> ResponseReturnValue: publish_flag = data.pop("is_published", None) for field, value in data.items(): setattr(ttp, field, value) + promoted = False if publish_flag is not None: # Promotion is a lead-only privilege. Decorator already gates draft # edits; promotion gets a second-tier check at the call site. _ensure_promote_perm() ttp.is_published = publish_flag + promoted = True db.session.commit() + audit_write( + action="ttp.promote" if promoted else "ttp.update", + resource_type="ttp", + resource_id=ttp.id, + metadata={"fields": sorted(data.keys()), "is_published": ttp.is_published}, + ) return jsonify_model(TtpRead.model_validate(ttp)) @@ -68,14 +89,14 @@ def delete_ttp(tid: str) -> ResponseReturnValue: ttp = db.session.get(Ttp, parse_uuid(tid, field="ttp id")) if ttp is None: abort(404) + ttp_id = ttp.id db.session.delete(ttp) db.session.commit() + audit_write(action="ttp.delete", resource_type="ttp", resource_id=ttp_id) return "", 204 def _ensure_promote_perm() -> None: - from flask_login import current_user # noqa: PLC0415 (lazy: scope-local user only) - perms: frozenset[Permission] = getattr(current_user, "permissions", frozenset()) if Permission.TTP_PROMOTE not in perms: abort(403) From e77ca906d49b95454802788f44b2e48eb2f7a6fe Mon Sep 17 00:00:00 2001 From: knacky Date: Fri, 22 May 2026 05:25:04 +0200 Subject: [PATCH 18/18] docs(backend): track sprint-0 follow-ups + flag integration migration gap - `tasks/todo.md`: B0.5 description updated (re2 hard dep, no fallback); add a "Backend follow-ups (sprint 1+)" section with M1-M7 + N1-N6 from the code-review verdict. - `CHANGELOG.md`: backend skeleton bullets refreshed (no re fallback, streaming blob store, audit + scope on CRUD, 56 unit tests); new "Code-review remediation" subsection lists B1 / MA1-MA6 / N4 / N6 / M8 with one-line rationale each. - `tests/integration/conftest.py`: leave `db.create_all()` in place but add an inline TODO (N6) pointing at the Alembic switchover that will exercise the F11 seed + audit-log role grants in CI. --- CHANGELOG.md | 45 ++++++++++++++++++++++++--- backend/tests/integration/conftest.py | 3 ++ tasks/todo.md | 44 ++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f004b55..8aeb479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,10 @@ UX wireframes (mock data). No real connector, no reporting until PR1/PR2/PR3 lan - `C2Connector` ABC + `Payload` / `TaskHandle` / `TaskResult` / `TaskStatus` dataclasses + `PayloadType` enum + `ConnectorFactory` keyed on `c2_type`. Mythic payload map populated; Home stays empty until PR2. -- Jinja2 `SandboxedEnvironment` + `regex_extract` filter (google-re2 with `re` fallback) + - `{{ outputs.text }}` / `{{ outputs.blob() }}` accessors (10 MB cap, UTF-8 → latin-1). +- Jinja2 `SandboxedEnvironment` + `regex_extract` filter (`google-re2` hard dependency per + D-011 / B1 — `RuntimeError` at boot if absent, no `re` fallback) + `{{ outputs.text }}` / + `{{ outputs.blob() }}` accessors reading gzip-compressed blobs (10 MB cap after + decompression, UTF-8 → latin-1). - Group-based RBAC: `Permission` + `GroupName` + `GROUP_PERMISSIONS` mirror the F11 matrix; `@require_perm` decorator + `AuthUser` Flask-Login wrapper that resolves the permission set from the user's groups. @@ -51,10 +53,15 @@ UX wireframes (mock data). No real connector, no reporting until PR1/PR2/PR3 lan - Hash-chained append-only audit writer (sprint 0 fills `prev_hash` / `row_hash` at insert; verifier shipped in v2). - Flat CRUD blueprints: engagements / hosts / TTPs / scenarios + scenario steps. F3 invariant - enforced (host.c2_type must match scenario.c2_type at compose time). + enforced (host.c2_type must match scenario.c2_type at compose time). Every mutation calls + the hash-chained audit writer (MA5); created rows carry `created_by_id` (MA4); listings and + per-engagement routes scope to `engagement_member` for RT operators (MA6 / F11). +- Content-addressed gzip blob store (`mimic.storage.blob`): streaming write with a `max_bytes` + cap (raises `BlobTooLarge` mid-stream — MA2), atomic rename, `0o750` directory mode. - `mimic-cli` (click): `user create`, `db dump`, `db restore`. -- pytest baseline: **38 unit tests passing**, integration scaffold ready for testcontainers - Postgres (`/healthz` smoke included). +- pytest baseline: **56 unit tests passing** (templating, regex_extract, password, soc_token, + RBAC matrix, connector factory, audit hash, blob CAS, migration seed parity). Integration + scaffold ready for testcontainers Postgres (`/healthz` smoke included). #### Spec deltas applied in this sprint @@ -69,3 +76,31 @@ Implementation arbitrations logged in this sprint: - **D-013** — `audit_log` hash chain (`prev_hash` / `row_hash`) shipped v1. - **D-014** — UUID columns use SQLAlchemy 2 native `Uuid` mapping; no `type_annotation_map` on the declarative base (Flask-SQLAlchemy incompatibility). + +#### Code-review remediation (`12d131c` → `feature/backend-skeleton`) + +- **B1** — Dropped the `re` stdlib fallback in `regex_extract`. `google-re2` is now a hard + dependency (B1 / D-011); the module raises `RuntimeError` at import if absent. +- **MA1** — Removed `scripts/postgres-init/00-roles.sql` (no more hardcoded `CHANGE_ME` + password). Audit-writer role provisioning is the playbook's responsibility (D-010); + `backend/README.md` documents the manual dev-only `CREATE ROLE` command. +- **MA2** — `store_blob` now accepts a binary stream + `max_bytes`, streams sha256+gzip in + 64 KB chunks, and raises `BlobTooLarge` mid-stream (cleans up the temp file). No more + whole-buffer RAM load. +- **MA3** — Inlined the F11 permission matrix in the initial Alembic migration; the runtime + matrix is no longer imported there. A new unit test + (`test_migration_seed_matches_current_matrix`) fails if the two drift apart. +- **MA4** — `created_by_id = current_user.id` set in `engagement`, `ttp`, and `scenario` + create endpoints. +- **MA5** — Every mutation endpoint now writes an audit row through the hash-chained + `AuditWriter` (F13). +- **MA6** — RT operators only see engagements they are members of (`engagement_member` join + on list, membership probe on `get`/`put`/`delete`/`host`/`scenario`/...). RT leads bypass. +- **N4** — `gunicorn` declared in `pyproject.toml` dependencies (the Dockerfile `CMD` now + resolves correctly). +- **N6** — `tests/integration/conftest.py` keeps `db.create_all()` for now; commented TODO to + switch over to Alembic once the playbook owns the audit role. +- **M8** — Initial migration docstring no longer mentions `ttp_version`. + +Verification on the latest commit: `ruff check`, `ruff format --check`, `mypy --strict`, and +`pytest tests/unit` all pass; 56 unit tests green. diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index a9eea82..30bfac7 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -37,6 +37,9 @@ def app(postgres_dsn: str, monkeypatch: pytest.MonkeyPatch): application = create_app() with application.app_context(): + # TODO (N6 follow-up, sprint 1): run Alembic migrations instead of + # db.create_all() so the integration tests exercise the real schema + # including the audit_log role grants and the F11 seed. db.create_all() yield application db.session.remove() diff --git a/tasks/todo.md b/tasks/todo.md index d23c5ca..1adfe00 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -16,9 +16,10 @@ Repo skeleton + foundational modules. Nothing that depends on PR1/PR2/PR3. evidence, report, soc_session, c2_credential, audit_log). - [x] B0.4 — `C2Connector` ABC + dataclasses + `payload_type` enum + factory keyed on `c2_type`. Mythic payload map populated; Home stays empty until PR2. -- [x] B0.5 — Jinja2 SandboxedEnvironment, `regex_extract` filter (google-re2 with `re` - fallback) per D-011 semantics (fail-loud no-match), `{{ outputs.text }}` / - `{{ outputs.blob() }}` accessors with 10 MB cap. +- [x] B0.5 — Jinja2 SandboxedEnvironment, `regex_extract` filter (`google-re2` hard + dependency per D-011, raises `RuntimeError` at boot if absent — no `re` fallback), + fail-loud no-match, `{{ outputs.text }}` / `{{ outputs.blob() }}` accessors + reading gzip-compressed blobs with 10 MB cap. - [x] B0.6 — bcrypt password helpers + SOC opaque token (256-bit url-safe, bcrypt-hashed) + group-based RBAC matrix matching F11 + `@require_perm` decorator. - [x] B0.7 — Flat CRUD blueprints for engagements / hosts / TTPs / scenarios (incl. step @@ -26,6 +27,43 @@ Repo skeleton + foundational modules. Nothing that depends on PR1/PR2/PR3. - [x] B0.8 — pytest baseline: unit tests passing, integration scaffold ready (testcontainers Postgres + `/healthz` smoke). +## Backend follow-ups (sprint 1+) + +Tracked from code-review verdict on `feature/backend-skeleton` @ 12d131c: + +### MINOR (8) — to schedule + +- **M1** — Replace `parse_uuid` integer-ish lookup with `werkzeug` UUID converter on + the routes (``) to avoid the 404 on malformed strings being hidden by + the 400 path. +- **M2** — Add OpenAPI generation (Pydantic 2 + `flask-pydantic-openapi` or hand-rolled). +- **M3** — Wire `flask-limiter` for `/auth/local/login` (NF-network). +- **M4** — Replace string-based `Engagement.status` setter with a typed transition method. +- **M5** — Introduce per-engagement read view that pre-joins `engagement_member` for + RT operator dashboards (current per-request join is fine for v1 traffic, but + re-evaluate at scale). +- **M6** — `mimic-cli user create` does not handle the SOC user-type (intended, but + document and gate explicitly with a clean error message). +- **M7** — Add a `mimic-cli` `engagement add-member --role rt_operator` shortcut so + the F11 scoping in MA6 is reachable from the CLI without manual SQL. +- **M8** — _(fixed in MA1 follow-up commit)_ Initial migration docstring no longer + references `ttp_version`. + +### NIT (6) — opportunistic + +- **N1** — Sort imports inside `mimic.db.models.__init__` alphabetically for diff + stability. +- **N2** — Extract the `_engagement_or_404` duplicated body into a shared helper. +- **N3** — Replace the inline `Permission.TTP_PROMOTE not in perms` check in `ttps.py` + with a second `@require_perm`-style decorator. +- **N4** — _(fixed)_ `gunicorn` added to `pyproject.toml` dependencies. +- **N5** — Replace bare `getattr(current_user, "groups", frozenset())` accesses by a + thin `current_groups()` helper. +- **N6** — `tests/integration/conftest.py` uses `db.create_all()` instead of running + Alembic. Marked with a TODO; switch over once the F11 seed must be exercised in + integration. Plan: convert to `alembic upgrade head` once the audit role + bootstrap lives in the playbook (D-010). + ## Frontend (`ux-frontend`) - [ ] F0.1 — `frontend/` Vite + React + TypeScript strict + Tailwind 4 + TanStack Query 5,