feat(m1): DB schema, migrations, diag visibility

23 tables + alembic_version covering the v1 data model:
- Auth/RBAC (8): users, groups, permissions, user_groups, group_permissions,
  invitations, invitation_groups, refresh_tokens.
- MITRE (4): mitre_tactics, mitre_techniques, mitre_subtechniques + the
  technique↔tactic many-to-many.
- Templates (4): test_templates, test_template_mitre_tags (3 nullable FKs +
  CHECK exactly_one_mitre_fk), scenario_templates, scenario_template_tests
  (UUID PK + UNIQUE(scenario_id, position) so a test can appear at multiple
  positions).
- Missions (6): missions, mission_members, mission_scenarios, mission_tests,
  mission_test_mitre_tags (deliberately denormalised — copies external_id +
  name + url, no FK to mitre_* — so a re-sync of the catalogue can't purge
  historical tags), mission_categories.
- Evidence/settings/notifications (5): evidence_files, settings (JSONB
  value), detection_levels, notifications.

SQLAlchemy 2.x with Mapped[]/mapped_column(), pk_/fk_/ck_/uq_/ix_ naming
convention. Reusable mixins (UuidPkMixin, TimestampMixin, SoftDeleteMixin —
no auto __table_args__ since classes silently clobber the mixin's).

Soft delete: deleted_at + partial indexes ix_<table>_active WHERE deleted_at
IS NULL on 9 tables (users, groups, test_templates, scenario_templates,
missions, mission_scenarios, mission_tests, mission_categories,
evidence_files). Notifications gets ix_..._unread WHERE read_at IS NULL.

CHECK constraints for status / state / opsec_level / mitre_kind enums.

New API endpoint GET /api/v1/diag/db: returns alembic_revision (short hash)
and the public-schema table_count. 503 with {"reachable": false} on a DB
outage. Database card on the SPA home consumes it.

Test stage in backend/Dockerfile (--target test): runtime + dev extras +
tests/. New make test-api spins an ephemeral pytest container against the
live DB on the compose network. backend/tests/test_schema.py: 8 integration
tests (tables, FK pairs, CHECK constraints, partial indexes, alembic-at-head,
negative INSERT proving the exactly_one_mitre_fk CHECK fires).

e2e/tests/m1-db.spec.ts: 4 Playwright tests covering the diag endpoint
contract + the Database card + footer/roadmap labels.

DoD: make clean && make up && make migrate → 23 tables, 32 FKs, 9 CHECKs,
make test-api → 9 passed, make e2e → 12 passed.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Knacky
2026-05-11 06:16:24 +02:00
parent f1fdf27012
commit e995853f0d
25 changed files with 2367 additions and 0 deletions

View File

@@ -0,0 +1,234 @@
"""M1 schema integration test.
Asserts that the migration has produced the expected tables, FK relations,
CHECK constraints, partial indexes, and that the alembic_version row is at head.
Skips automatically when Postgres is unreachable.
"""
from __future__ import annotations
import pytest
from sqlalchemy import inspect, text
EXPECTED_TABLES = {
# Auth / RBAC
"users",
"groups",
"permissions",
"user_groups",
"group_permissions",
"invitations",
"invitation_groups",
"refresh_tokens",
# MITRE
"mitre_tactics",
"mitre_techniques",
"mitre_subtechniques",
"mitre_technique_tactics",
# Templates
"test_templates",
"test_template_mitre_tags",
"scenario_templates",
"scenario_template_tests",
# Missions
"missions",
"mission_members",
"mission_scenarios",
"mission_tests",
"mission_test_mitre_tags",
"mission_categories",
# Evidence / settings / notifications
"evidence_files",
"settings",
"detection_levels",
"notifications",
# Alembic bookkeeping
"alembic_version",
}
# Tables that MUST carry a `deleted_at` column (soft delete).
SOFT_DELETE_TABLES = {
"users",
"groups",
"test_templates",
"scenario_templates",
"missions",
"mission_scenarios",
"mission_tests",
"mission_categories",
"evidence_files",
}
# Tables that MUST carry the standard `created_at` + `updated_at` pair.
TIMESTAMP_TABLES = {
"users",
"groups",
"test_templates",
"scenario_templates",
"missions",
"mission_scenarios",
"mission_tests",
"mission_categories",
"evidence_files",
}
# Spot-checked FK pairs (child_table, child_col, parent_table).
EXPECTED_FKS = {
("evidence_files", "mission_test_id", "mission_tests"),
("evidence_files", "uploaded_by_user_id", "users"),
("mission_members", "mission_id", "missions"),
("mission_members", "user_id", "users"),
("mission_scenarios", "mission_id", "missions"),
("mission_tests", "scenario_id", "mission_scenarios"),
("mission_tests", "detection_level_id", "detection_levels"),
("group_permissions", "group_id", "groups"),
("group_permissions", "permission_id", "permissions"),
("user_groups", "user_id", "users"),
("user_groups", "group_id", "groups"),
("refresh_tokens", "user_id", "users"),
("notifications", "user_id", "users"),
("mitre_subtechniques", "technique_id", "mitre_techniques"),
}
# CHECK constraint names we expect to see (namespace 'public' only).
# `mission_test_mitre_tags` deliberately lacks the exactly_one_mitre_fk check
# because it is denormalised — see app/models/mission.py docstring.
EXPECTED_CHECKS = {
"ck_missions_status_valid",
"ck_missions_visibility_mode_valid",
"ck_mission_tests_state_valid",
"ck_mission_tests_snapshot_opsec_level_valid",
"ck_test_templates_opsec_level_valid",
"ck_mission_members_role_hint_valid",
"ck_test_template_mitre_tags_mitre_kind_valid",
"ck_test_template_mitre_tags_exactly_one_mitre_fk",
"ck_mission_test_mitre_tags_mitre_kind_valid",
}
@pytest.fixture(scope="module")
def insp(db_engine_or_skip):
return inspect(db_engine_or_skip)
def test_all_expected_tables_exist(insp):
actual = set(insp.get_table_names(schema="public"))
missing = EXPECTED_TABLES - actual
assert not missing, f"missing tables: {sorted(missing)}"
def test_soft_delete_columns_present(insp):
for tbl in sorted(SOFT_DELETE_TABLES):
cols = {c["name"] for c in insp.get_columns(tbl)}
assert "deleted_at" in cols, f"{tbl} missing deleted_at"
def test_standard_timestamp_columns_present(insp):
for tbl in sorted(TIMESTAMP_TABLES):
cols = {c["name"] for c in insp.get_columns(tbl)}
assert "created_at" in cols, f"{tbl} missing created_at"
assert "updated_at" in cols, f"{tbl} missing updated_at"
def test_partial_index_for_soft_delete(db_engine_or_skip):
"""Each soft-delete table must carry an `ix_<table>_active` partial index."""
with db_engine_or_skip.connect() as conn:
rows = conn.execute(
text(
"SELECT indexname FROM pg_indexes "
"WHERE schemaname='public' AND indexdef ILIKE '%deleted_at IS NULL%'"
)
).all()
names = {r[0] for r in rows}
for tbl in SOFT_DELETE_TABLES:
assert f"ix_{tbl}_active" in names, f"{tbl}: partial index missing — got {names}"
def test_expected_foreign_keys(insp):
all_fks = set()
for tbl in EXPECTED_TABLES:
if tbl == "alembic_version":
continue
for fk in insp.get_foreign_keys(tbl):
for col in fk["constrained_columns"]:
all_fks.add((tbl, col, fk["referred_table"]))
for triple in EXPECTED_FKS:
assert triple in all_fks, f"missing FK: {triple}"
def test_expected_check_constraints(db_engine_or_skip):
with db_engine_or_skip.connect() as conn:
rows = conn.execute(
text(
"SELECT conname FROM pg_constraint "
"WHERE contype='c' AND connamespace = 'public'::regnamespace"
)
).all()
names = {r[0] for r in rows}
missing = EXPECTED_CHECKS - names
assert not missing, f"missing CHECK constraints: {sorted(missing)}"
def test_alembic_at_head(db_engine_or_skip):
"""The DB must be at the latest migration after `make migrate`."""
with db_engine_or_skip.connect() as conn:
rev = conn.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
assert rev, "alembic_version is empty — migrate didn't run"
assert len(rev) >= 8, f"unexpected alembic version: {rev}"
def test_exactly_one_mitre_fk_check_enforced(db_engine_or_skip):
"""Inserting a tag with two non-null FKs must raise (CHECK constraint)."""
import uuid
from sqlalchemy.exc import IntegrityError
with db_engine_or_skip.begin() as conn:
# Seed a minimal test_template + tactic + technique to reference.
tmpl_id = uuid.uuid4()
tactic_id = uuid.uuid4()
technique_id = uuid.uuid4()
conn.execute(
text(
"INSERT INTO test_templates (id, name, opsec_level) "
"VALUES (:id, 'tmp', 'low')"
),
{"id": tmpl_id},
)
conn.execute(
text(
"INSERT INTO mitre_tactics (id, external_id, short_name, name) "
"VALUES (:id, 'TA0099', 'tmp', 'tmp')"
),
{"id": tactic_id},
)
conn.execute(
text(
"INSERT INTO mitre_techniques (id, external_id, name) "
"VALUES (:id, 'T9999', 'tmp')"
),
{"id": technique_id},
)
# Now try to insert a violating row — must fail.
with pytest.raises(IntegrityError):
with db_engine_or_skip.begin() as conn:
conn.execute(
text(
"INSERT INTO test_template_mitre_tags "
"(id, test_template_id, mitre_kind, tactic_id, technique_id) "
"VALUES (:id, :tmpl, 'tactic', :tac, :tech)"
),
{
"id": uuid.uuid4(),
"tmpl": tmpl_id,
"tac": tactic_id,
"tech": technique_id,
},
)
# Cleanup so the test is rerunnable.
with db_engine_or_skip.begin() as conn:
conn.execute(text("DELETE FROM mitre_techniques WHERE id = :id"), {"id": technique_id})
conn.execute(text("DELETE FROM mitre_tactics WHERE id = :id"), {"id": tactic_id})
conn.execute(text("DELETE FROM test_templates WHERE id = :id"), {"id": tmpl_id})