knacky/mimic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b5ea2929de94d1cef8bf2c67826cf88a5d6f41f5
mimic/backend/tests/test_simulations_techniques.py
Knacky b5ea2929de feat(backend): sprint 3 — multi-technique simulations + MITRE matrix 
- Simulation model: replace mitre_technique_id/name scalars with techniques JSON column [{id, name}]
- Alembic migration 0003: add techniques, backfill from scalars, drop old columns (reversible)
- MITRE service: add get_tactics(), lookup_name(), get_matrix() with canonical tactic order and sub-technique nesting
- serializer: enrich techniques with tactics from service at serialize time (graceful empty tactics if bundle outdated)
- simulation_workflow: PATCH now accepts technique_ids list, validates against bundle, deduplicates preserving order, auto-transitions on non-empty list
- simulations API: add GET /api/mitre/matrix endpoint (503 if bundle absent)
- test_mitre.py: updated _reset_mitre fixture, added T1059.006 sub-technique, 14 new tests for get_tactics/lookup_name/get_matrix/matrix endpoint
- test_simulations_techniques.py: 20 new tests covering AC-13.1 to AC-13.5 (create, PATCH, dedup, auto-transition, SOC blocked, migration backfill logic)

Total: 161 tests passing. ruff clean. mypy: no new errors.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-27 03:56:02 +02:00

348 lines
12 KiB
Python
Raw Blame History

"""Sprint 3 — multi-technique simulation tests (AC-13)."""
from __future__ import annotations
import json
import pathlib
import pytest
from flask.testing import FlaskClient
from backend.app.services import mitre as mitre_svc
from backend.tests.conftest import auth_headers as _h
# ---------------------------------------------------------------------------
# Minimal STIX fixture (reused from test_mitre.py pattern)
# ---------------------------------------------------------------------------
_FIXTURE_BUNDLE = {
"type": "bundle",
"objects": [
{
"type": "attack-pattern",
"name": "Command and Scripting Interpreter",
"external_references": [{"source_name": "mitre-attack", "external_id": "T1059"}],
"kill_chain_phases": [{"phase_name": "execution", "kill_chain_name": "mitre-attack"}],
},
{
"type": "attack-pattern",
"name": "PowerShell",
"external_references": [{"source_name": "mitre-attack", "external_id": "T1059.001"}],
"kill_chain_phases": [{"phase_name": "execution", "kill_chain_name": "mitre-attack"}],
},
{
"type": "attack-pattern",
"name": "Valid Accounts",
"external_references": [{"source_name": "mitre-attack", "external_id": "T1078"}],
"kill_chain_phases": [
{"phase_name": "initial-access", "kill_chain_name": "mitre-attack"},
{"phase_name": "persistence", "kill_chain_name": "mitre-attack"},
],
},
],
}
@pytest.fixture(autouse=True)
def _reset_mitre():
original_loaded = mitre_svc.mitre_loaded
original_index = list(mitre_svc._index)
original_tactics = dict(mitre_svc._tactics_by_technique)
original_names = dict(mitre_svc._name_by_id)
original_matrix = list(mitre_svc._matrix)
yield
mitre_svc.mitre_loaded = original_loaded
mitre_svc._index = original_index
mitre_svc._tactics_by_technique = original_tactics
mitre_svc._name_by_id = original_names
mitre_svc._matrix = original_matrix
@pytest.fixture()
def bundle_file(tmp_path: pathlib.Path) -> pathlib.Path:
p = tmp_path / "enterprise-attack.json"
p.write_text(json.dumps(_FIXTURE_BUNDLE), encoding="utf-8")
return p
@pytest.fixture()
def loaded_bundle(bundle_file: pathlib.Path) -> pathlib.Path:
mitre_svc.load_bundle(bundle_file)
return bundle_file
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_engagement(client: FlaskClient, token: str) -> dict:
resp = client.post(
"/api/engagements",
headers=_h(token),
json={"name": "Op Sprint3", "start_date": "2026-06-01"},
)
assert resp.status_code == 201
return resp.get_json()
def _make_sim(client: FlaskClient, token: str, eid: int) -> dict:
resp = client.post(
f"/api/engagements/{eid}/simulations",
headers=_h(token),
json={"name": "Technique Test"},
)
assert resp.status_code == 201
return resp.get_json()
def _patch(client: FlaskClient, token: str, sid: int, payload: dict):
return client.patch(f"/api/simulations/{sid}", headers=_h(token), json=payload)
# ---------------------------------------------------------------------------
# AC-13.1 — new simulation has techniques = []
# ---------------------------------------------------------------------------
def test_new_simulation_has_empty_techniques(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
assert sim["techniques"] == []
# ---------------------------------------------------------------------------
# AC-13.3 — serializer enriches techniques with tactics
# ---------------------------------------------------------------------------
def test_techniques_enriched_with_tactics(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
_patch(client, redteam_token, sim["id"], {"technique_ids": ["T1078"]})
resp = client.get(f"/api/simulations/{sim['id']}", headers=_h(redteam_token))
assert resp.status_code == 200
techs = resp.get_json()["techniques"]
assert len(techs) == 1
assert techs[0]["id"] == "T1078"
assert "initial-access" in techs[0]["tactics"]
assert "persistence" in techs[0]["tactics"]
def test_techniques_with_unknown_id_returns_empty_tactics(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
"""If a technique was removed from the bundle after save, tactics gracefully = []."""
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
# Bypass service, write directly an id not in the bundle.
from backend.app.extensions import db
from backend.app.models.simulation import Simulation
with client.application.app_context():
s = db.session.get(Simulation, sim["id"])
s.techniques = [{"id": "T0000", "name": "Removed Technique"}]
db.session.commit()
resp = client.get(f"/api/simulations/{sim['id']}", headers=_h(redteam_token))
techs = resp.get_json()["techniques"]
assert techs[0]["tactics"] == []
# ---------------------------------------------------------------------------
# AC-13.4 — PATCH technique_ids
# ---------------------------------------------------------------------------
def test_patch_technique_ids_sets_techniques(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059", "T1078"]})
assert resp.status_code == 200
techs = resp.get_json()["techniques"]
assert len(techs) == 2
ids = [t["id"] for t in techs]
assert "T1059" in ids
assert "T1078" in ids
def test_patch_technique_ids_resolves_name(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059"]})
assert resp.status_code == 200
tech = resp.get_json()["techniques"][0]
assert tech["name"] == "Command and Scripting Interpreter"
def test_patch_technique_ids_unknown_returns_400(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T9999"]})
assert resp.status_code == 400
assert "unknown technique id: T9999" in resp.get_json()["error"]
def test_patch_technique_ids_partial_unknown_rejected(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
# One valid, one unknown — whole request rejected.
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059", "T9999"]})
assert resp.status_code == 400
def test_patch_technique_ids_includes_subtechnique(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059.001"]})
assert resp.status_code == 200
techs = resp.get_json()["techniques"]
assert techs[0]["id"] == "T1059.001"
assert techs[0]["name"] == "PowerShell"
def test_patch_technique_ids_replaces_list(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
_patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059"]})
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1078"]})
assert resp.status_code == 200
ids = [t["id"] for t in resp.get_json()["techniques"]]
assert ids == ["T1078"]
def test_patch_technique_ids_empty_clears_list(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
_patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059"]})
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": []})
assert resp.status_code == 200
assert resp.get_json()["techniques"] == []
def test_patch_technique_ids_not_list_returns_400(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": "T1059"})
assert resp.status_code == 400
# ---------------------------------------------------------------------------
# Dedup (spec-reviewer note: AC-13.4)
# ---------------------------------------------------------------------------
def test_patch_technique_ids_deduplicates(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(
client, redteam_token, sim["id"], {"technique_ids": ["T1059", "T1078", "T1059"]}
)
assert resp.status_code == 200
techs = resp.get_json()["techniques"]
assert len(techs) == 2
# Order preserved: T1059 first.
assert techs[0]["id"] == "T1059"
assert techs[1]["id"] == "T1078"
# ---------------------------------------------------------------------------
# AC-13.5 — auto-transition on technique_ids
# ---------------------------------------------------------------------------
def test_technique_ids_non_empty_triggers_auto_transition(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
assert sim["status"] == "pending"
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": ["T1059"]})
assert resp.status_code == 200
assert resp.get_json()["status"] == "in_progress"
def test_technique_ids_empty_does_not_trigger_auto_transition(
client: FlaskClient, redteam_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
resp = _patch(client, redteam_token, sim["id"], {"technique_ids": []})
assert resp.status_code == 200
assert resp.get_json()["status"] == "pending"
# ---------------------------------------------------------------------------
# SOC cannot patch technique_ids (it's a redteam field)
# ---------------------------------------------------------------------------
def test_soc_cannot_patch_technique_ids(
client: FlaskClient, redteam_token: str, soc_token: str, loaded_bundle
) -> None:
eng = _make_engagement(client, redteam_token)
sim = _make_sim(client, redteam_token, eng["id"])
# Advance to review_required so SOC can touch the simulation at all.
client.post(
f"/api/simulations/{sim['id']}/transition",
headers=_h(redteam_token),
json={"to": "review_required"},
)
resp = _patch(client, soc_token, sim["id"], {"technique_ids": ["T1059"]})
assert resp.status_code == 403
# ---------------------------------------------------------------------------
# Migration backfill test (inline, no Alembic runner needed)
# ---------------------------------------------------------------------------
def test_migration_backfill_logic() -> None:
"""Verify the backfill logic used in upgrade(): scalar → [{id, name}]."""
import json as _json
def _backfill(tech_id, tech_name):
if tech_id:
return _json.loads(_json.dumps([{"id": tech_id, "name": tech_name or ""}]))
return []
assert _backfill("T1059", "Command and Scripting Interpreter") == [
{"id": "T1059", "name": "Command and Scripting Interpreter"}
]
assert _backfill(None, None) == []
assert _backfill("T1059", None) == [{"id": "T1059", "name": ""}]
Reference in New Issue View Git Blame Copy Permalink