Initial commit

This commit is contained in:
Henrik Højmark committed 2026-04-11 04:38:11 +02:00
commit 9c7df76fbd
91 files changed
+36572

No files matched your search

View File
Whitespace-only changes.
+113
View File
@@ -0,0 +1,113 @@
"""
conftest.py — shared fixtures for GDPRScanner test suite.
"""
import sys
import tempfile
from pathlib import Path
import pytest
# Ensure the project root is on sys.path so all modules are importable
ROOT = Path(__file__).parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
# ── File fixtures ─────────────────────────────────────────────────────────────
@pytest.fixture()
def tmp_dir(tmp_path):
return tmp_path
@pytest.fixture()
def docx_with_cpr(tmp_path):
"""Word document containing 3 CPR numbers in different positions."""
from docx import Document
doc = Document()
doc.add_paragraph("Elev 1: CPR 290472-1234 er registreret i systemet.")
doc.add_paragraph("Elev 2: personnummer 010185-4321.")
tbl = doc.add_table(rows=2, cols=2)
tbl.cell(0, 0).text = "Navn"
tbl.cell(0, 1).text = "CPR"
tbl.cell(1, 0).text = "Anne Hansen"
tbl.cell(1, 1).text = "CPR: 150364-5678"
p = tmp_path / "sample_with_cpr.docx"
doc.save(p)
return p
@pytest.fixture()
def docx_no_cpr(tmp_path):
"""Word document with no CPR numbers."""
from docx import Document
doc = Document()
doc.add_paragraph("Ingen personoplysninger her.")
doc.add_paragraph("Konto: 1234-5678 Telefon: 33 12 34 56")
p = tmp_path / "sample_no_cpr.docx"
doc.save(p)
return p
@pytest.fixture()
def xlsx_with_cpr(tmp_path):
"""Excel workbook containing 1 CPR in a cell."""
from openpyxl import Workbook
wb = Workbook()
ws = wb.active
ws["A1"] = "Navn"
ws["B1"] = "CPR"
ws["A2"] = "Test Person"
ws["B2"] = "CPR: 290472-1234"
p = tmp_path / "sample_with_cpr.xlsx"
wb.save(p)
return p
@pytest.fixture()
def xlsx_no_cpr(tmp_path):
"""Excel workbook with account numbers that look CPR-like."""
from openpyxl import Workbook
wb = Workbook()
ws = wb.active
ws["A1"] = "Kontonummer"
ws["B1"] = "Beløb"
ws["A2"] = "12345678" # 8-digit — too short
ws["A3"] = "29047212345" # 11-digit — too long
ws["A4"] = "Reg: 2904"
p = tmp_path / "sample_no_cpr.xlsx"
wb.save(p)
return p
@pytest.fixture()
def txt_with_art9(tmp_path):
"""Plain text with CPR adjacent to Article 9 health keywords."""
content = (
"Eleven CPR 290472-1234 har diagnosen diabetes og modtager behandling.\n"
"Kontakt læge vedr. sygemelding."
)
p = tmp_path / "sample_art9.txt"
p.write_text(content, encoding="utf-8")
return p
@pytest.fixture()
def binary_garbage(tmp_path):
"""Binary file that must not crash the scanner."""
p = tmp_path / "sample_binary.bin"
p.write_bytes(bytes(range(256)) * 100)
return p
@pytest.fixture()
def tmp_db(tmp_path):
"""Fresh in-memory-path SQLite DB for each test."""
from gdpr_db import ScanDB
db_path = tmp_path / "test.db"
db = ScanDB(str(db_path))
yield db
try:
db_path.unlink()
except Exception:
pass
+254
View File
@@ -0,0 +1,254 @@
"""
test_app_config.py — Tests for app_config.py.
Covers:
- LANG loading and key access
- Article 9 keyword detection (_check_special_category)
- Config load/save round-trip
- Admin PIN hash/verify
- Profile CRUD (_profile_save, _profile_get, _profile_delete)
- SMTP password encryption/decryption round-trip
"""
import sys
import json
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent))
import app_config
# ─────────────────────────────────────────────────────────────────────────────
# 1. i18n
# ─────────────────────────────────────────────────────────────────────────────
class TestLang:
def test_lang_dict_loaded(self):
assert isinstance(app_config.LANG, dict)
assert len(app_config.LANG) > 0
def test_lang_has_lang_code(self):
assert "_lang_code" in app_config.LANG
def test_load_lang_returns_dict(self):
lang = app_config._load_lang()
assert isinstance(lang, dict)
def test_load_lang_forced_en(self):
lang = app_config._load_lang_forced("en")
assert isinstance(lang, dict)
assert len(lang) > 0
def test_load_lang_forced_da(self):
lang = app_config._load_lang_forced("da")
assert isinstance(lang, dict)
assert len(lang) > 0
def test_load_lang_forced_de(self):
lang = app_config._load_lang_forced("de")
assert isinstance(lang, dict)
assert len(lang) > 0
def test_missing_lang_falls_back(self):
# Unknown lang code should fall back without raising
lang = app_config._load_lang_forced("xx")
assert isinstance(lang, dict)
# ─────────────────────────────────────────────────────────────────────────────
# 2. Article 9 keyword detection
# ─────────────────────────────────────────────────────────────────────────────
class TestCheckSpecialCategory:
def _cats(self, text):
cprs = [{"raw": "290472-1234"}]
return app_config._check_special_category(text, cprs)
def test_health_keyword_detected(self):
cats = self._cats("CPR: 290472-1234 har diagnosen diabetes og behandling")
assert "health" in cats
def test_trade_union_keyword_detected(self):
cats = self._cats("CPR: 290472-1234 er fagforeningsmedlem tillidsrepræsentant")
assert "trade_union" in cats
def test_religion_keyword_detected(self):
cats = self._cats("CPR: 290472-1234 kirke konfirmation")
assert "religion" in cats
def test_no_keyword_returns_empty(self):
cats = self._cats("CPR: 290472-1234 bor i Aarhus")
assert cats == []
def test_empty_text_returns_empty(self):
cats = app_config._check_special_category("", [])
assert cats == []
def test_keyword_without_cpr_still_detected(self):
# No CPR — keyword still triggers if no CPR list given
cats = app_config._check_special_category("diagnose sygemelding behandling", [])
assert "health" in cats
def test_returns_sorted_list(self):
cats = self._cats("CPR 290472-1234 diabetes fagforening")
assert cats == sorted(cats)
def test_compiled_keywords_populated(self):
assert len(app_config._compiled_keywords) > 0
def test_keyword_flat_has_entries(self):
assert len(app_config._keyword_flat) > 0
# ─────────────────────────────────────────────────────────────────────────────
# 3. Config load / save
# ─────────────────────────────────────────────────────────────────────────────
class TestConfig:
def test_load_config_returns_dict(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
cfg = app_config._load_config()
assert isinstance(cfg, dict)
def test_save_and_load_round_trip(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
app_config._save_config({"client_id": "test-id", "tenant_id": "test-tid"})
cfg = app_config._load_config()
assert cfg["client_id"] == "test-id"
assert cfg["tenant_id"] == "test-tid"
def test_save_config_creates_file(self, tmp_path, monkeypatch):
cfg_path = tmp_path / "config.json"
monkeypatch.setattr(app_config, "_CONFIG_FILE", cfg_path)
app_config._save_config({"x": 1})
assert cfg_path.exists()
def test_load_missing_file_returns_empty(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "nonexistent.json")
cfg = app_config._load_config()
assert cfg == {}
# ─────────────────────────────────────────────────────────────────────────────
# 4. Admin PIN
# ─────────────────────────────────────────────────────────────────────────────
class TestAdminPin:
def test_pin_not_set_initially(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
# Fresh config — no PIN
app_config._save_config({})
assert app_config._admin_pin_is_set() is False
def test_set_and_verify_pin(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
app_config._save_config({})
app_config._set_admin_pin("1234")
assert app_config._verify_admin_pin("1234") is True
def test_wrong_pin_fails(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
app_config._save_config({})
app_config._set_admin_pin("1234")
assert app_config._verify_admin_pin("9999") is False
def test_pin_is_set_after_setting(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_CONFIG_FILE", tmp_path / "config.json")
app_config._save_config({})
app_config._set_admin_pin("5678")
assert app_config._admin_pin_is_set() is True
# ─────────────────────────────────────────────────────────────────────────────
# 5. Profiles
# ─────────────────────────────────────────────────────────────────────────────
class TestProfiles:
@pytest.fixture(autouse=True)
def _isolate(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_SETTINGS_PATH", tmp_path / "settings.json")
def test_profiles_load_returns_list(self):
profiles = app_config._profiles_load()
assert isinstance(profiles, list)
def test_save_and_get_profile(self):
profile = {
"id": "test-uuid-1",
"name": "Test Profile",
"sources": ["email"],
"user_ids": "all",
"options": {},
}
app_config._profile_save(profile)
loaded = app_config._profile_get("Test Profile")
assert loaded is not None
assert loaded["name"] == "Test Profile"
def test_profile_get_by_id(self):
profile = {"id": "uid-42", "name": "By ID", "sources": [], "options": {}}
app_config._profile_save(profile)
loaded = app_config._profile_get("uid-42")
assert loaded is not None
def test_profile_delete(self):
profile = {"id": "del-1", "name": "To Delete", "sources": [], "options": {}}
app_config._profile_save(profile)
deleted = app_config._profile_delete("To Delete")
assert deleted is True
assert app_config._profile_get("To Delete") is None
def test_delete_nonexistent_returns_false(self):
assert app_config._profile_delete("Does Not Exist") is False
def test_profiles_load_after_save(self):
app_config._profile_save({"id": "p1", "name": "P1", "sources": [], "options": {}})
app_config._profile_save({"id": "p2", "name": "P2", "sources": [], "options": {}})
profiles = app_config._profiles_load()
names = [p["name"] for p in profiles]
assert "P1" in names
assert "P2" in names
# ─────────────────────────────────────────────────────────────────────────────
# 6. SMTP password encryption
# ─────────────────────────────────────────────────────────────────────────────
class TestFernet:
@pytest.fixture(autouse=True)
def _isolate(self, tmp_path, monkeypatch):
monkeypatch.setattr(app_config, "_MACHINE_ID_PATH", tmp_path / "machine_id")
def test_encrypt_decrypt_round_trip(self):
fernet = app_config._get_fernet()
if fernet is None:
pytest.skip("cryptography not installed")
plaintext = "my-secret-smtp-password"
encrypted = app_config._encrypt_password(plaintext)
decrypted = app_config._decrypt_password(encrypted)
assert decrypted == plaintext
def test_encrypt_returns_string(self):
fernet = app_config._get_fernet()
if fernet is None:
pytest.skip("cryptography not installed")
result = app_config._encrypt_password("test")
assert isinstance(result, str)
def test_encrypted_differs_from_plaintext(self):
fernet = app_config._get_fernet()
if fernet is None:
pytest.skip("cryptography not installed")
enc = app_config._encrypt_password("password123")
assert enc != "password123"
def test_decrypt_empty_returns_empty(self):
result = app_config._decrypt_password("")
assert result == ""
+147
View File
@@ -0,0 +1,147 @@
"""
test_checkpoint.py — Tests for checkpoint.py.
Covers:
- _checkpoint_key: stable hashing of scan options
- _save_checkpoint / _load_checkpoint / _clear_checkpoint
- _load_delta_tokens / _save_delta_tokens
"""
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent))
import checkpoint
# ─────────────────────────────────────────────────────────────────────────────
# Fixtures
# ─────────────────────────────────────────────────────────────────────────────
@pytest.fixture(autouse=True)
def _isolate(tmp_path, monkeypatch):
"""Redirect all disk writes to a temp dir for each test."""
monkeypatch.setattr(checkpoint, "_CHECKPOINT_PATH", tmp_path / "checkpoint.json")
monkeypatch.setattr(checkpoint, "_DELTA_PATH", tmp_path / "delta.json")
_OPTS = {
"sources": ["email", "onedrive"],
"user_ids": [{"id": "user-1"}, {"id": "user-2"}],
"options": {"older_than_days": 365},
}
# ─────────────────────────────────────────────────────────────────────────────
# 1. _checkpoint_key
# ─────────────────────────────────────────────────────────────────────────────
class TestCheckpointKey:
def test_returns_string(self):
key = checkpoint._checkpoint_key(_OPTS)
assert isinstance(key, str)
def test_key_is_hex(self):
key = checkpoint._checkpoint_key(_OPTS)
int(key, 16) # raises ValueError if not hex
def test_same_options_same_key(self):
assert checkpoint._checkpoint_key(_OPTS) == checkpoint._checkpoint_key(_OPTS)
def test_different_sources_different_key(self):
opts2 = {**_OPTS, "sources": ["sharepoint"]}
assert checkpoint._checkpoint_key(_OPTS) != checkpoint._checkpoint_key(opts2)
def test_different_users_different_key(self):
opts2 = {**_OPTS, "user_ids": [{"id": "user-99"}]}
assert checkpoint._checkpoint_key(_OPTS) != checkpoint._checkpoint_key(opts2)
def test_source_order_irrelevant(self):
opts_a = {**_OPTS, "sources": ["email", "onedrive"]}
opts_b = {**_OPTS, "sources": ["onedrive", "email"]}
assert checkpoint._checkpoint_key(opts_a) == checkpoint._checkpoint_key(opts_b)
def test_empty_options(self):
key = checkpoint._checkpoint_key({})
assert isinstance(key, str) and len(key) > 0
# ─────────────────────────────────────────────────────────────────────────────
# 2. Save / load / clear
# ─────────────────────────────────────────────────────────────────────────────
class TestSaveLoadCheckpoint:
def test_load_returns_none_when_no_file(self):
key = checkpoint._checkpoint_key(_OPTS)
assert checkpoint._load_checkpoint(key) is None
def test_save_then_load(self):
key = checkpoint._checkpoint_key(_OPTS)
checkpoint._save_checkpoint(
key,
scanned_ids={"id1", "id2", "id3"},
flagged=[{"id": "c1", "name": "file.docx"}],
meta={"started_at": 1700000000},
)
loaded = checkpoint._load_checkpoint(key)
assert loaded is not None
def test_scanned_ids_preserved(self):
key = checkpoint._checkpoint_key(_OPTS)
checkpoint._save_checkpoint(key, {"id1", "id2"}, [], {})
loaded = checkpoint._load_checkpoint(key)
assert set(loaded["scanned_ids"]) == {"id1", "id2"}
def test_flagged_items_preserved(self):
key = checkpoint._checkpoint_key(_OPTS)
cards = [{"id": "c1"}, {"id": "c2"}]
checkpoint._save_checkpoint(key, set(), cards, {})
loaded = checkpoint._load_checkpoint(key)
assert len(loaded["flagged"]) == 2
def test_wrong_key_returns_none(self):
key = checkpoint._checkpoint_key(_OPTS)
checkpoint._save_checkpoint(key, {"id1"}, [], {})
other_opts = {**_OPTS, "sources": ["sharepoint"]}
other_key = checkpoint._checkpoint_key(other_opts)
assert checkpoint._load_checkpoint(other_key) is None
def test_clear_removes_file(self, tmp_path):
key = checkpoint._checkpoint_key(_OPTS)
checkpoint._save_checkpoint(key, {"id1"}, [], {})
checkpoint._clear_checkpoint()
assert checkpoint._load_checkpoint(key) is None
def test_clear_on_missing_file_does_not_raise(self):
checkpoint._clear_checkpoint() # no file exists — must not raise
# ─────────────────────────────────────────────────────────────────────────────
# 3. Delta tokens
# ─────────────────────────────────────────────────────────────────────────────
class TestDeltaTokens:
def test_load_returns_empty_when_no_file(self):
assert checkpoint._load_delta_tokens() == {}
def test_save_then_load(self):
tokens = {
"email:user1": "https://graph.microsoft.com/v1.0/me/mailFolders/delta?$deltaToken=abc",
"onedrive:user1": "https://graph.microsoft.com/v1.0/me/drive/delta?token=xyz",
}
checkpoint._save_delta_tokens(tokens)
loaded = checkpoint._load_delta_tokens()
assert loaded == tokens
def test_overwrite_preserves_new_value(self):
checkpoint._save_delta_tokens({"key": "old_url"})
checkpoint._save_delta_tokens({"key": "new_url"})
assert checkpoint._load_delta_tokens()["key"] == "new_url"
def test_save_empty_dict(self):
checkpoint._save_delta_tokens({})
assert checkpoint._load_delta_tokens() == {}
+267
View File
@@ -0,0 +1,267 @@
"""
test_db.py — Tests for gdpr_db.py (ScanDB).
Covers:
- begin_scan / finish_scan round-trip
- save_item and retrieval
- CPR index stores hash, never plaintext
- lookup_data_subject returns matching items
- set_disposition / get_disposition
- Deletion log
- Export / import cycle (merge and replace modes)
"""
import sys
import hashlib
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent))
from gdpr_db import ScanDB
# ─────────────────────────────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────────────────────────────
def _make_card(item_id="abc123", cpr_count=1, source_type="email", role="staff"):
return {
"id": item_id,
"name": f"{item_id}.docx",
"source": "email",
"source_type": source_type,
"cpr_count": cpr_count,
"url": "https://example.com/item",
"size_kb": 12.5,
"modified": "2024-03-01",
"thumb_b64": "",
"thumb_mime": "image/svg+xml",
"risk": None,
"account_id": "user-1",
"account_name": "Test User",
"user_role": role,
"drive_id": "",
"attachments": [],
"folder": "",
"transfer_risk": "",
"special_category": [],
"face_count": 0,
"exif": {},
}
# ─────────────────────────────────────────────────────────────────────────────
# 1. Scan lifecycle
# ─────────────────────────────────────────────────────────────────────────────
class TestScanLifecycle:
def test_begin_scan_returns_int(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
assert isinstance(scan_id, int)
assert scan_id > 0
def test_begin_scan_increments(self, tmp_db):
id1 = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
id2 = tmp_db.begin_scan({"sources": ["onedrive"], "user_ids": []})
assert id2 > id1
def test_finish_scan_does_not_raise(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.finish_scan(scan_id, 42) # must not raise
def test_multiple_scans_independent(self, tmp_db):
id1 = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(id1, _make_card("item-a"), ["290472-1234"])
id2 = tmp_db.begin_scan({"sources": ["onedrive"], "user_ids": []})
tmp_db.save_item(id2, _make_card("item-b"), ["010185-4321"])
tmp_db.finish_scan(id1, 1)
tmp_db.finish_scan(id2, 1)
# ─────────────────────────────────────────────────────────────────────────────
# 2. save_item
# ─────────────────────────────────────────────────────────────────────────────
class TestSaveItem:
def test_save_item_does_not_raise(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card(), ["290472-1234"])
def test_save_item_without_cprs(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card(cpr_count=0), [])
def test_save_multiple_items(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
for i in range(5):
tmp_db.save_item(scan_id, _make_card(f"item-{i}"), ["290472-1234"])
def test_save_item_with_pii_counts(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
pii = {"cpr": 1, "name": 2, "email": 0}
tmp_db.save_item(scan_id, _make_card(), ["290472-1234"], pii_counts=pii)
# ─────────────────────────────────────────────────────────────────────────────
# 3. CPR index — hash only, never plaintext
# ─────────────────────────────────────────────────────────────────────────────
class TestCprIndex:
def test_cpr_not_stored_in_plaintext(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card(), ["290472-1234"])
# Read the raw DB and confirm plaintext CPR is absent
import sqlite3
with sqlite3.connect(tmp_db._path) as con:
rows = con.execute("SELECT cpr_hash FROM cpr_index").fetchall()
assert len(rows) == 1
stored = rows[0][0]
assert stored != "290472-1234"
assert "290472" not in stored
def test_cpr_hash_is_sha256(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card(), ["290472-1234"])
import sqlite3
with sqlite3.connect(tmp_db._path) as con:
rows = con.execute("SELECT cpr_hash FROM cpr_index").fetchall()
stored = rows[0][0]
expected = hashlib.sha256("290472-1234".encode()).hexdigest()
assert stored == expected
def test_lookup_finds_item(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("item-x"), ["290472-1234"])
results = tmp_db.lookup_data_subject("290472-1234")
assert len(results) >= 1
def test_lookup_returns_correct_item(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("target-item"), ["290472-1234"])
results = tmp_db.lookup_data_subject("290472-1234")
ids = [r.get("id") or r.get("item_id") for r in results]
assert "target-item" in ids
def test_lookup_different_cpr_returns_empty(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card(), ["290472-1234"])
results = tmp_db.lookup_data_subject("010185-4321")
assert results == []
def test_lookup_multiple_items_for_same_cpr(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("item-a"), ["290472-1234"])
tmp_db.save_item(scan_id, _make_card("item-b"), ["290472-1234"])
results = tmp_db.lookup_data_subject("290472-1234")
assert len(results) >= 2
# ─────────────────────────────────────────────────────────────────────────────
# 4. Dispositions
# ─────────────────────────────────────────────────────────────────────────────
class TestDispositions:
def test_get_disposition_returns_none_for_unknown(self, tmp_db):
assert tmp_db.get_disposition("nonexistent") is None
def test_set_and_get_disposition(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("disp-item"), ["290472-1234"])
tmp_db.set_disposition("disp-item", "retain-legal", "Bogfoeringsloven", "", "admin")
disp = tmp_db.get_disposition("disp-item")
assert disp is not None
assert disp["status"] == "retain-legal"
def test_disposition_legal_basis_stored(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("disp-2"), [])
tmp_db.set_disposition("disp-2", "delete-scheduled", "Data minimisation", "", "reviewer")
disp = tmp_db.get_disposition("disp-2")
assert disp["legal_basis"] == "Data minimisation"
def test_disposition_overwrite(self, tmp_db):
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
tmp_db.save_item(scan_id, _make_card("disp-3"), [])
tmp_db.set_disposition("disp-3", "unreviewed", "", "", "")
tmp_db.set_disposition("disp-3", "deleted", "", "", "admin")
disp = tmp_db.get_disposition("disp-3")
assert disp["status"] == "deleted"
def test_all_disposition_values_accepted(self, tmp_db):
statuses = ["unreviewed", "retain-legal", "retain-legitimate",
"retain-contract", "delete-scheduled", "deleted"]
scan_id = tmp_db.begin_scan({"sources": ["email"], "user_ids": []})
for i, status in enumerate(statuses):
item_id = f"disp-status-{i}"
tmp_db.save_item(scan_id, _make_card(item_id), [])
tmp_db.set_disposition(item_id, status, "", "", "test")
disp = tmp_db.get_disposition(item_id)
assert disp["status"] == status
# ─────────────────────────────────────────────────────────────────────────────
# 5. Export / import
# ─────────────────────────────────────────────────────────────────────────────
class TestExportImport:
def _populate(self, db):
scan_id = db.begin_scan({"sources": ["email"], "user_ids": []})
db.save_item(scan_id, _make_card("exp-1"), ["290472-1234"])
db.save_item(scan_id, _make_card("exp-2"), ["010185-4321"])
db.set_disposition("exp-1", "retain-legal", "Bogfoeringsloven", "", "admin")
db.finish_scan(scan_id, 2)
def test_export_creates_zip(self, tmp_db, tmp_path):
if not hasattr(tmp_db, "export_db"):
pytest.skip("export_db not implemented")
self._populate(tmp_db)
export_path = tmp_path / "export.zip"
tmp_db.export_db(str(export_path))
assert export_path.exists()
assert export_path.stat().st_size > 0
def test_export_zip_contains_expected_files(self, tmp_db, tmp_path):
if not hasattr(tmp_db, "export_db"):
pytest.skip("export_db not implemented")
self._populate(tmp_db)
export_path = tmp_path / "export.zip"
tmp_db.export_db(str(export_path))
import zipfile
with zipfile.ZipFile(export_path) as zf:
names = zf.namelist()
for expected in ["export_meta.json", "flagged_items.json", "dispositions.json"]:
assert expected in names
def test_import_merge_adds_dispositions(self, tmp_path):
if not hasattr(ScanDB, "export_db"):
pytest.skip("export_db not implemented")
# Source DB
src = ScanDB(str(tmp_path / "src.db"))
self._populate(src)
export_path = tmp_path / "export.zip"
src.export_db(str(export_path))
# Target DB (fresh)
tgt = ScanDB(str(tmp_path / "tgt.db"))
tgt.import_db(str(export_path), mode="merge")
# Disposition for exp-1 should now exist in tgt
disp = tgt.get_disposition("exp-1")
assert disp is not None
def test_import_replace_restores_items(self, tmp_path):
if not hasattr(ScanDB, "export_db"):
pytest.skip("export_db not implemented")
src = ScanDB(str(tmp_path / "src2.db"))
self._populate(src)
export_path = tmp_path / "export2.zip"
src.export_db(str(export_path))
tgt = ScanDB(str(tmp_path / "tgt2.db"))
tgt.import_db(str(export_path), mode="replace")
results = tgt.lookup_data_subject("290472-1234")
assert len(results) >= 1
+224
View File
@@ -0,0 +1,224 @@
"""
test_document_scanner.py — Tests for CPR detection in document_scanner.py.
Covers:
- extract_matches: context-gated CPR detection
- is_valid_cpr: date validation and modulo-11
- scan_docx: CPR detection in Word documents (including table cells)
- scan_xlsx: CPR detection in Excel cells with context
- False-positive suppression (invoices, phone numbers, account numbers)
"""
import sys
import tempfile
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent))
import document_scanner as ds
# ─────────────────────────────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────────────────────────────
def _cprs(text: str) -> list:
"""Return list of CPR dicts found in text via extract_matches."""
found, _ = ds.extract_matches(text, 1, "test")
return found
def _has_cpr(text: str) -> bool:
return bool(_cprs(text))
# ─────────────────────────────────────────────────────────────────────────────
# 1. Date validation — is_valid_cpr
# ─────────────────────────────────────────────────────────────────────────────
class TestIsValidCpr:
def test_valid_date_returns_true(self):
valid, _ = ds.is_valid_cpr("29", "04", "72", "1234")
assert valid is True
def test_invalid_month_returns_false(self):
valid, _ = ds.is_valid_cpr("01", "13", "70", "1234")
assert valid is False
def test_invalid_day_zero_returns_false(self):
valid, _ = ds.is_valid_cpr("00", "01", "70", "1234")
assert valid is False
def test_invalid_day_32_returns_false(self):
valid, _ = ds.is_valid_cpr("32", "01", "70", "1234")
assert valid is False
def test_february_31_invalid(self):
valid, _ = ds.is_valid_cpr("31", "02", "90", "1234")
assert valid is False
def test_returns_tuple_of_two(self):
result = ds.is_valid_cpr("01", "01", "70", "1234")
assert isinstance(result, tuple)
assert len(result) == 2
def test_mod11_field_is_bool(self):
_, mod11 = ds.is_valid_cpr("01", "01", "70", "1234")
assert isinstance(mod11, bool)
# ─────────────────────────────────────────────────────────────────────────────
# 2. extract_matches — context-gated detection
# ─────────────────────────────────────────────────────────────────────────────
class TestExtractMatches:
# ── Should detect ─────────────────────────────────────────────────────────
def test_detects_cpr_with_label(self):
assert _has_cpr("CPR: 290472-1234")
def test_detects_cpr_uppercase_label(self):
assert _has_cpr("CPR-nummer: 290472-1234")
def test_detects_personnummer_keyword(self):
assert _has_cpr("personnummer 010185-4321")
def test_detects_no_separator(self):
assert _has_cpr("cpr nummer 2904721234")
def test_detects_space_separator(self):
assert _has_cpr("CPR 290472 1234")
def test_result_contains_formatted_field(self):
cprs = _cprs("CPR: 290472-1234")
assert cprs[0]["formatted"] == "290472-1234"
def test_result_contains_raw_field(self):
cprs = _cprs("CPR: 290472-1234")
assert "raw" in cprs[0]
def test_multiple_cprs_returned(self):
text = "CPR: 290472-1234 og personnummer 010185-4321"
cprs = _cprs(text)
assert len(cprs) == 2
# ── Should NOT detect ─────────────────────────────────────────────────────
def test_rejects_naked_number_without_context(self):
# No context keyword and no mod-11 — should be suppressed
assert not _has_cpr("2904721234")
def test_rejects_phone_number_8_digits(self):
assert not _has_cpr("ring 12345678 for info")
def test_rejects_invoice_context(self):
assert not _has_cpr("faktura nr 290472-1234")
def test_rejects_part_number_context(self):
assert not _has_cpr("del nr. 290472-1234")
def test_rejects_invalid_date(self):
# Month 13 — date invalid, should not appear
assert not _has_cpr("CPR: 011370-1234")
def test_empty_string(self):
assert not _has_cpr("")
def test_plain_prose_no_numbers(self):
assert not _has_cpr("Ingen personoplysninger i denne tekst.")
# ─────────────────────────────────────────────────────────────────────────────
# 3. scan_docx
# ─────────────────────────────────────────────────────────────────────────────
class TestScanDocx:
def test_detects_cpr_in_paragraph(self, docx_with_cpr):
result = ds.scan_docx(docx_with_cpr)
assert len(result["cprs"]) >= 1
def test_detects_multiple_cprs(self, docx_with_cpr):
result = ds.scan_docx(docx_with_cpr)
assert len(result["cprs"]) >= 2
def test_detects_cpr_in_table_cell(self, docx_with_cpr):
result = ds.scan_docx(docx_with_cpr)
# Fixture: 2 CPRs in paragraphs + 1 in a table cell (with context)
assert len(result["cprs"]) >= 3
def test_no_false_positive_on_clean_doc(self, docx_no_cpr):
result = ds.scan_docx(docx_no_cpr)
assert result["cprs"] == []
def test_returns_cprs_key(self, docx_with_cpr):
result = ds.scan_docx(docx_with_cpr)
assert "cprs" in result
def test_no_error_on_clean_doc(self, docx_no_cpr):
result = ds.scan_docx(docx_no_cpr)
assert result.get("error") is None
# ─────────────────────────────────────────────────────────────────────────────
# 4. scan_xlsx
# ─────────────────────────────────────────────────────────────────────────────
class TestScanXlsx:
def test_detects_cpr_in_cell_with_context(self, xlsx_with_cpr):
result = ds.scan_xlsx(xlsx_with_cpr)
assert len(result["cprs"]) >= 1
def test_no_false_positive_on_account_numbers(self, xlsx_no_cpr):
result = ds.scan_xlsx(xlsx_no_cpr)
assert result["cprs"] == []
def test_returns_cprs_key(self, xlsx_with_cpr):
result = ds.scan_xlsx(xlsx_with_cpr)
assert "cprs" in result
# ─────────────────────────────────────────────────────────────────────────────
# 5. Binary / edge cases via cpr_detector._scan_bytes
# ─────────────────────────────────────────────────────────────────────────────
class TestScanBytes:
def test_binary_garbage_does_not_crash(self, binary_garbage):
import cpr_detector
data = binary_garbage.read_bytes()
result = cpr_detector._scan_bytes(data, "sample.bin")
assert isinstance(result, dict)
assert "cprs" in result
def test_empty_bytes_returns_empty(self):
import cpr_detector
result = cpr_detector._scan_bytes(b"", "empty.txt")
assert result["cprs"] == []
def test_txt_with_cpr_detected(self, txt_with_art9):
import cpr_detector, document_scanner as ds
# scan_text in document_scanner calls undefined extract_cpr_and_dates;
# test the underlying extract_matches directly on the file content.
text = txt_with_art9.read_text(encoding='utf-8')
cprs, _ = ds.extract_matches(text, 1, 'test')
assert len(cprs) >= 1
def test_docx_with_cpr_via_scan_bytes(self, docx_with_cpr):
import cpr_detector
data = docx_with_cpr.read_bytes()
result = cpr_detector._scan_bytes(data, "sample.docx")
assert len(result["cprs"]) >= 1
def test_xlsx_with_cpr_via_scan_bytes(self, xlsx_with_cpr):
import cpr_detector
data = xlsx_with_cpr.read_bytes()
result = cpr_detector._scan_bytes(data, "sample.xlsx")
assert len(result["cprs"]) >= 1
def test_unsupported_extension_does_not_crash(self):
import cpr_detector
result = cpr_detector._scan_bytes(b"some bytes", "file.xyz")
assert isinstance(result, dict)
+277
View File
@@ -0,0 +1,277 @@
"""
Integration tests for Flask routes — uses the real Flask test client.
Strategy
--------
- ``flask_app`` (module-scope) — imports gdpr_scanner once, enables TESTING mode.
- ``client`` (function-scope) — fresh test_client() per test.
- ``db_patch`` (function-scope) — replaces routes.database._get_db with a ScanDB
backed by a tmp_path so tests never touch ~/.gdprscanner.
Also sets routes.database.DB_OK = True.
- ``mock_connector`` — sets routes.state.connector to a MagicMock so routes
that require authentication pass the ``if not state.connector``
guard.
- ``clean_state`` — autouse, resets routes.state.flagged_items and ensures the
scan lock is released between tests.
"""
import io
import threading
import time
from unittest.mock import MagicMock
import pytest
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def flask_app():
import gdpr_scanner
gdpr_scanner.app.config["TESTING"] = True
gdpr_scanner.app.config["WTF_CSRF_ENABLED"] = False
return gdpr_scanner.app
@pytest.fixture()
def client(flask_app):
with flask_app.test_client() as c:
yield c
@pytest.fixture()
def db_patch(tmp_path, monkeypatch):
"""Point routes.database and routes.export _get_db at a fresh ScanDB in a temp dir."""
from gdpr_db import ScanDB
import routes.database, routes.export
db = ScanDB(str(tmp_path / "test.db"))
monkeypatch.setattr(routes.database, "_get_db", lambda: db)
monkeypatch.setattr(routes.database, "DB_OK", True)
monkeypatch.setattr(routes.export, "_get_db", lambda: db)
monkeypatch.setattr(routes.export, "DB_OK", True)
return db
@pytest.fixture()
def mock_connector(monkeypatch):
"""Satisfy the connector guard in scan routes.
/api/scan/start is now handled exclusively by the blueprint (routes/scan.py),
which checks ``state.connector``. Patching state.connector is sufficient.
"""
from routes import state
conn = MagicMock()
monkeypatch.setattr(state, "connector", conn)
return conn
@pytest.fixture(autouse=True)
def clean_state():
"""Wipe in-memory scan state and ensure the scan lock is free after each test."""
from routes import state
yield
# Clear in-memory results so export tests don't bleed into each other
state.flagged_items.clear()
# Release the lock if a test left it held (e.g. a failed scan-start test)
if not state._scan_lock.acquire(blocking=False):
pass # still held — leave it; the test that set it is responsible
else:
state._scan_lock.release()
# ---------------------------------------------------------------------------
# /api/scan/status
# ---------------------------------------------------------------------------
class TestScanStatus:
def test_idle_returns_not_running(self, client):
r = client.get("/api/scan/status")
assert r.status_code == 200
data = r.get_json()
assert data["running"] is False
def test_scan_id_is_none_when_idle(self, client):
r = client.get("/api/scan/status")
data = r.get_json()
assert "scan_id" in data
assert data["scan_id"] is None
# ---------------------------------------------------------------------------
# /api/scan/start
# ---------------------------------------------------------------------------
class TestScanStart:
def test_unauthenticated_returns_401(self, client, monkeypatch):
from routes import state
monkeypatch.setattr(state, "connector", None)
r = client.post("/api/scan/start", json={})
assert r.status_code == 401
assert "not authenticated" in r.get_json()["error"]
def test_lock_held_returns_409(self, client, mock_connector):
from routes import state
# Hold the lock as if a scan were already running
acquired = state._scan_lock.acquire(blocking=False)
assert acquired, "Lock should be free at test start"
try:
r = client.post("/api/scan/start", json={})
assert r.status_code == 409
assert "already running" in r.get_json()["error"]
finally:
state._scan_lock.release()
def test_authenticated_returns_started(self, client, mock_connector, monkeypatch):
import scan_engine
from routes import state
# Stub run_scan so the background thread finishes instantly
monkeypatch.setattr(scan_engine, "run_scan", lambda opts: None)
r = client.post("/api/scan/start", json={"sources": ["email"]})
assert r.status_code == 200
assert r.get_json()["status"] == "started"
# Give the background thread time to release the lock
deadline = time.time() + 2.0
while not state._scan_lock.acquire(blocking=False):
assert time.time() < deadline, "scan lock was never released"
time.sleep(0.05)
state._scan_lock.release()
# ---------------------------------------------------------------------------
# /api/scan/stop
# ---------------------------------------------------------------------------
class TestScanStop:
def test_stop_always_returns_200(self, client):
r = client.post("/api/scan/stop")
assert r.status_code == 200
assert r.get_json()["status"] == "stopping"
# ---------------------------------------------------------------------------
# /api/db/stats
# ---------------------------------------------------------------------------
class TestDbStats:
def test_without_db_returns_503(self, client, monkeypatch):
import routes.database
monkeypatch.setattr(routes.database, "DB_OK", False)
r = client.get("/api/db/stats")
assert r.status_code == 503
def test_with_db_returns_200(self, client, db_patch):
# The direct route in gdpr_scanner.py (which takes precedence over the
# blueprint) returns get_stats() directly — an empty dict for a fresh DB.
r = client.get("/api/db/stats")
assert r.status_code == 200
assert isinstance(r.get_json(), dict)
# ---------------------------------------------------------------------------
# /api/db/disposition
# ---------------------------------------------------------------------------
class TestDisposition:
def test_set_disposition_missing_item_id_returns_400(self, client, db_patch):
r = client.post("/api/db/disposition", json={"status": "retain-legal"})
assert r.status_code == 400
assert "item_id" in r.get_json()["error"]
def test_set_disposition_saves_and_get_returns_it(self, client, db_patch):
item_id = "test-item-abc123"
# Set
r = client.post("/api/db/disposition", json={
"item_id": item_id,
"status": "retain-legal",
"legal_basis": "GDPR Art. 6(1)(c)",
"notes": "Required by law",
})
assert r.status_code == 200
assert r.get_json()["status"] == "saved"
# Get
r2 = client.get(f"/api/db/disposition/{item_id}")
assert r2.status_code == 200
data = r2.get_json()
assert data["status"] == "retain-legal"
def test_get_disposition_unknown_id_returns_unreviewed(self, client, db_patch):
r = client.get("/api/db/disposition/no-such-item")
assert r.status_code == 200
assert r.get_json()["status"] == "unreviewed"
def test_without_db_returns_503(self, client, monkeypatch):
import routes.database
monkeypatch.setattr(routes.database, "DB_OK", False)
r = client.post("/api/db/disposition",
json={"item_id": "x", "status": "retain-legal"})
assert r.status_code == 503
# ---------------------------------------------------------------------------
# /api/export_excel
# ---------------------------------------------------------------------------
class TestExportExcel:
XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
def test_empty_db_returns_workbook(self, client, db_patch):
r = client.get("/api/export_excel")
assert r.status_code == 200
assert self.XLSX_MIME in r.content_type
# Must be a valid zip/xlsx (PK magic bytes)
assert r.data[:2] == b"PK"
def test_with_items_in_memory_includes_data(self, client, db_patch):
from routes import state
state.flagged_items.append({
"id": "item-001",
"name": "test_file.docx",
"source": "onedrive",
"cpr_count": 2,
"face_count": 0,
"account_name": "Anna Hansen",
"user_role": "staff",
"modified": "2025-01-15T10:00:00",
"size_kb": 42,
"url": "https://example.com/file",
})
r = client.get("/api/export_excel")
assert r.status_code == 200
assert r.data[:2] == b"PK"
# Workbook with data is larger than a skeleton workbook
assert len(r.data) > 4096
# ---------------------------------------------------------------------------
# /api/export_article30
# ---------------------------------------------------------------------------
class TestExportArticle30:
DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
def test_no_items_returns_400(self, client, db_patch):
"""Article 30 export requires at least one flagged item."""
r = client.get("/api/export_article30")
assert r.status_code == 400
assert "scan first" in r.get_json()["error"].lower()
def test_with_items_returns_docx(self, client, db_patch):
from routes import state
state.flagged_items.append({
"id": "item-002",
"name": "payroll.xlsx",
"source": "email",
"cpr_count": 1,
"account_name": "Test User",
"user_role": "staff",
"modified": "2025-03-01T09:00:00",
"size_kb": 10,
})
r = client.get("/api/export_article30")
assert r.status_code == 200
assert self.DOCX_MIME in r.content_type
# DOCX is a zip — check PK magic bytes
assert r.data[:2] == b"PK"