feat: interface PIN, bulk disposition tagging, Google Drive delta scan, OCR memory fixes

- Interface PIN: optional session-level auth gate for the main scanner UI
  (Settings → Security → Interface PIN). Salted SHA-256 in config.json,
  rate-limited (5 attempts/5 min per IP). /view and viewer auth exempt.
  New /login page, before_request hook, GET/POST/DELETE /api/interface/pin,
  POST /api/interface/pin/verify, POST /api/interface/logout.

- Bulk disposition tagging: Select mode (filter bar "Vælg" button) reveals
  per-card checkboxes. Bulk tag bar at bottom of grid; POST /api/db/disposition/bulk.
  Disposition stats bar (total · unreviewed · retain · delete · % reviewed)
  updates after every save.

- Google Drive delta scan: uses Drive Changes API when delta is enabled.
  Per-user token stored as gdrive:{email} in delta.json. Load-then-merge
  save avoids racing with concurrent M365 token writes.

- PDF OCR OOM fix: render one page at a time with convert_from_path
  (first_page=N, last_page=N). Added _ocr_mem_ok() psutil guard (500 MB
  threshold) before each page render across scan_pdf, redact_fitz_pdf,
  redact_pdf.

- Email test message translation fix: routes/email.py returns structured
  {ok, method, recipients} instead of a hardcoded English string;
  scheduler.js builds the translated message client-side.

- Docs: CHANGELOG, README, TODO, MANUAL-EN, MANUAL-DA all updated.
  Lang files (en/da/de) extended with bulk, interface PIN, and SMTP keys.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
StyxX65andClaude Sonnet 4.6 committed 2026-04-18 18:46:45 +02:00
1 parent b2bfa40f27
commit d8083eb0c0
26 files changed
+1059 -128

No files matched your search

+3
View File
@@ -14,6 +14,9 @@ All three scan engines must include `"source": "m365"` / `"google"` / `"file"` i
## Circular import prohibition
`scan_engine.py` and `gdpr_scanner.py` must not import each other. `scan_engine` imports from `sse`, `checkpoint`, `app_config`, `cpr_detector`; `gdpr_scanner` imports scan functions from `scan_engine`.
## `_scan_bytes` injection
`scan_engine.py` declares stub versions of `_scan_bytes` / `_scan_bytes_timeout` at module level. `gdpr_scanner.py` replaces them with the real `cpr_detector` implementations at startup. `routes/google_scan.py` pulls them from `gdpr_scanner` via `__getattr__`. Never import these directly in blueprint or engine modules — that breaks the circular-import barrier.
## Gotchas
- **`_load_settings()` return** — does NOT include `file_sources`. Returns only: sources, user_ids, options, retention_years, fiscal_year_end, email_to.
+20
View File
@@ -143,6 +143,26 @@ def db_set_disposition():
return jsonify({"status": "saved"})
@bp.route("/api/db/disposition/bulk", methods=["POST"])
def db_set_disposition_bulk():
"""Set the same disposition on multiple items at once.
Body: {item_ids: [...], status, legal_basis?, notes?, reviewed_by?}
"""
if not DB_OK: return jsonify({"error": "database not available"}), 503
data = request.get_json() or {}
item_ids = data.get("item_ids", [])
status = data.get("status", "")
if not item_ids or not status:
return jsonify({"error": "item_ids and status required"}), 400
db = _get_db()
for iid in item_ids:
db.set_disposition(iid, status,
legal_basis=data.get("legal_basis", ""),
notes=data.get("notes", ""),
reviewed_by=data.get("reviewed_by", ""))
return jsonify({"saved": len(item_ids)})
@bp.route("/api/db/disposition/<item_id>")
def db_get_disposition(item_id):
"""Get the current disposition for an item."""
+3 -4
View File
@@ -147,8 +147,7 @@ def smtp_test():
if state.connector and state.connector.is_authenticated():
try:
_send_email_graph(subject, body_html, recipients)
return jsonify({"ok": True,
"message": f"Test email sent via Microsoft Graph to {', '.join(recipients)}"})
return jsonify({"ok": True, "method": "graph", "recipients": recipients})
except Exception as graph_err:
graph_error_str = str(graph_err)
else:
@@ -193,8 +192,8 @@ def smtp_test():
if username and password:
server.login(username, password)
server.sendmail(from_addr, recipients, msg.as_string())
suffix = " (⚠ Graph also failed — Mail.Send permission not granted)" if graph_error_str else ""
return jsonify({"ok": True, "message": f"Test email sent via SMTP to {', '.join(recipients)}{suffix}"})
return jsonify({"ok": True, "method": "smtp", "recipients": recipients,
"graph_also_failed": bool(graph_error_str)})
except Exception as smtp_err:
err_str = str(smtp_err)
_h = host.lower()
+46 -9
View File
@@ -140,6 +140,11 @@ def _run_google_scan(options: dict):
max_file_mb = float(scan_opts.get("max_file_mb", 50.0))
scan_body = bool(scan_opts.get("scan_body", True))
scan_att = bool(scan_opts.get("scan_attachments", True))
delta_enabled = bool(scan_opts.get("delta", False))
from checkpoint import _load_delta_tokens, _save_delta_tokens
_drive_delta_tokens: dict = _load_delta_tokens() if delta_enabled else {}
_new_drive_tokens: dict = {}
# Resolve users: explicit list → Admin SDK → fall back to SA email itself
_user_role_map: dict = {} # email → role
@@ -283,12 +288,35 @@ def _run_google_scan(options: dict):
# ── Google Drive ──────────────────────────────────────────────────────
if "gdrive" in sources:
try:
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive"})
for meta, data in conn.iter_drive_files(
user_email,
max_files=max_files,
max_file_mb=max_file_mb,
):
delta_key = f"gdrive:{user_email}"
saved_token = _drive_delta_tokens.get(delta_key) if delta_enabled else None
if delta_enabled and saved_token:
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive (delta)"})
try:
drive_items, new_token = conn.get_drive_changes(
user_email, saved_token,
max_files=max_files, max_file_mb=max_file_mb,
)
_new_drive_tokens[delta_key] = new_token
except Exception as delta_err:
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive (delta token invalid — full scan)"})
logger.warning("[gdrive delta] %s: %s — falling back to full scan", user_email, delta_err)
drive_items = list(conn.iter_drive_files(user_email, max_files=max_files, max_file_mb=max_file_mb))
try:
_new_drive_tokens[delta_key] = conn.get_drive_start_token(user_email)
except Exception:
pass
else:
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive"})
drive_items = list(conn.iter_drive_files(user_email, max_files=max_files, max_file_mb=max_file_mb))
if delta_enabled:
try:
_new_drive_tokens[delta_key] = conn.get_drive_start_token(user_email)
except Exception:
pass
for meta, data in drive_items:
if _check_abort():
return
total_scanned += 1
@@ -306,7 +334,7 @@ def _run_google_scan(options: dict):
except Exception as e:
broadcast("scan_error", {"file": meta.get("name", ""), "error": str(e)})
continue
cprs = result.get("cprs", [])
cprs = result.get("cprs", [])
pii_counts = result.get("pii_counts")
if cprs or (pii_counts and any(pii_counts.values())):
_broadcast_card(meta, cprs, pii_counts)
@@ -315,11 +343,20 @@ def _run_google_scan(options: dict):
except Exception as e:
broadcast("scan_error", {"file": f"Drive/{user_email}", "error": str(e)})
if delta_enabled and _new_drive_tokens:
try:
current_tokens = _load_delta_tokens()
_save_delta_tokens({**current_tokens, **_new_drive_tokens})
except Exception as e:
logger.warning("[gdrive delta] token save failed: %s", e)
elapsed = _time.monotonic() - t_start
broadcast("google_scan_done", {
"flagged_count": total_flagged,
"total_scanned": total_scanned,
"flagged_count": total_flagged,
"total_scanned": total_scanned,
"elapsed_seconds": round(elapsed, 1),
"delta": delta_enabled and bool(_new_drive_tokens),
"delta_sources": len(_new_drive_tokens),
})
if _db and _db_scan_id:
try:
+42
View File
@@ -14,6 +14,10 @@ from app_config import (
set_viewer_pin,
verify_viewer_pin,
clear_viewer_pin,
get_interface_pin_hash,
set_interface_pin,
verify_interface_pin,
clear_interface_pin,
)
bp = Blueprint("viewer", __name__)
@@ -161,6 +165,44 @@ def pin_clear():
return jsonify({"ok": True})
# ── Interface PIN management endpoints ───────────────────────────────────────
@bp.route("/api/interface/pin", methods=["GET"])
def interface_pin_status():
"""Return whether an interface PIN is currently set."""
return jsonify({"pin_set": bool(get_interface_pin_hash())})
@bp.route("/api/interface/pin", methods=["POST"])
def interface_pin_set():
"""Set or change the interface PIN.
Body: {pin: "...", current_pin: "..."}
current_pin required only when a PIN is already set.
"""
body = request.get_json(silent=True) or {}
new_pin = str(body.get("pin", "")).strip()
if not new_pin:
return jsonify({"error": "pin required"}), 400
if not new_pin.isdigit() or not (4 <= len(new_pin) <= 8):
return jsonify({"error": "PIN must be 4–8 digits"}), 400
if get_interface_pin_hash():
if not verify_interface_pin(str(body.get("current_pin", "")).strip()):
return jsonify({"error": "current PIN is incorrect"}), 403
set_interface_pin(new_pin)
return jsonify({"ok": True})
@bp.route("/api/interface/pin", methods=["DELETE"])
def interface_pin_clear():
"""Remove the interface PIN. Requires current PIN if one is set."""
body = request.get_json(silent=True) or {}
if get_interface_pin_hash():
if not verify_interface_pin(str(body.get("current_pin", "")).strip()):
return jsonify({"error": "current PIN is incorrect"}), 403
clear_interface_pin()
return jsonify({"ok": True})
@bp.route("/api/viewer/pin/verify", methods=["POST"])
def pin_verify():
"""Verify a PIN submission and set a viewer session on success."""