feat: interface PIN, bulk disposition tagging, Google Drive delta scan, OCR memory fixes
- Interface PIN: optional session-level auth gate for the main scanner UI
(Settings → Security → Interface PIN). Salted SHA-256 in config.json,
rate-limited (5 attempts/5 min per IP). /view and viewer auth exempt.
New /login page, before_request hook, GET/POST/DELETE /api/interface/pin,
POST /api/interface/pin/verify, POST /api/interface/logout.
- Bulk disposition tagging: Select mode (filter bar "Vælg" button) reveals
per-card checkboxes. Bulk tag bar at bottom of grid; POST /api/db/disposition/bulk.
Disposition stats bar (total · unreviewed · retain · delete · % reviewed)
updates after every save.
- Google Drive delta scan: uses Drive Changes API when delta is enabled.
Per-user token stored as gdrive:{email} in delta.json. Load-then-merge
save avoids racing with concurrent M365 token writes.
- PDF OCR OOM fix: render one page at a time with convert_from_path
(first_page=N, last_page=N). Added _ocr_mem_ok() psutil guard (500 MB
threshold) before each page render across scan_pdf, redact_fitz_pdf,
redact_pdf.
- Email test message translation fix: routes/email.py returns structured
{ok, method, recipients} instead of a hardcoded English string;
scheduler.js builds the translated message client-side.
- Docs: CHANGELOG, README, TODO, MANUAL-EN, MANUAL-DA all updated.
Lang files (en/da/de) extended with bulk, interface PIN, and SMTP keys.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
b2bfa40f27
commit
d8083eb0c0
26 files changed
+1059
-128
No files matched your search
@@ -14,6 +14,9 @@ All three scan engines must include `"source": "m365"` / `"google"` / `"file"` i
|
||||
## Circular import prohibition
|
||||
`scan_engine.py` and `gdpr_scanner.py` must not import each other. `scan_engine` imports from `sse`, `checkpoint`, `app_config`, `cpr_detector`; `gdpr_scanner` imports scan functions from `scan_engine`.
|
||||
|
||||
## `_scan_bytes` injection
|
||||
`scan_engine.py` declares stub versions of `_scan_bytes` / `_scan_bytes_timeout` at module level. `gdpr_scanner.py` replaces them with the real `cpr_detector` implementations at startup. `routes/google_scan.py` pulls them from `gdpr_scanner` via `__getattr__`. Never import these directly in blueprint or engine modules — that breaks the circular-import barrier.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **`_load_settings()` return** — does NOT include `file_sources`. Returns only: sources, user_ids, options, retention_years, fiscal_year_end, email_to.
|
||||
|
||||
@@ -143,6 +143,26 @@ def db_set_disposition():
|
||||
return jsonify({"status": "saved"})
|
||||
|
||||
|
||||
@bp.route("/api/db/disposition/bulk", methods=["POST"])
|
||||
def db_set_disposition_bulk():
|
||||
"""Set the same disposition on multiple items at once.
|
||||
Body: {item_ids: [...], status, legal_basis?, notes?, reviewed_by?}
|
||||
"""
|
||||
if not DB_OK: return jsonify({"error": "database not available"}), 503
|
||||
data = request.get_json() or {}
|
||||
item_ids = data.get("item_ids", [])
|
||||
status = data.get("status", "")
|
||||
if not item_ids or not status:
|
||||
return jsonify({"error": "item_ids and status required"}), 400
|
||||
db = _get_db()
|
||||
for iid in item_ids:
|
||||
db.set_disposition(iid, status,
|
||||
legal_basis=data.get("legal_basis", ""),
|
||||
notes=data.get("notes", ""),
|
||||
reviewed_by=data.get("reviewed_by", ""))
|
||||
return jsonify({"saved": len(item_ids)})
|
||||
|
||||
|
||||
@bp.route("/api/db/disposition/<item_id>")
|
||||
def db_get_disposition(item_id):
|
||||
"""Get the current disposition for an item."""
|
||||
|
||||
+3
-4
@@ -147,8 +147,7 @@ def smtp_test():
|
||||
if state.connector and state.connector.is_authenticated():
|
||||
try:
|
||||
_send_email_graph(subject, body_html, recipients)
|
||||
return jsonify({"ok": True,
|
||||
"message": f"Test email sent via Microsoft Graph to {', '.join(recipients)}"})
|
||||
return jsonify({"ok": True, "method": "graph", "recipients": recipients})
|
||||
except Exception as graph_err:
|
||||
graph_error_str = str(graph_err)
|
||||
else:
|
||||
@@ -193,8 +192,8 @@ def smtp_test():
|
||||
if username and password:
|
||||
server.login(username, password)
|
||||
server.sendmail(from_addr, recipients, msg.as_string())
|
||||
suffix = " (⚠ Graph also failed — Mail.Send permission not granted)" if graph_error_str else ""
|
||||
return jsonify({"ok": True, "message": f"Test email sent via SMTP to {', '.join(recipients)}{suffix}"})
|
||||
return jsonify({"ok": True, "method": "smtp", "recipients": recipients,
|
||||
"graph_also_failed": bool(graph_error_str)})
|
||||
except Exception as smtp_err:
|
||||
err_str = str(smtp_err)
|
||||
_h = host.lower()
|
||||
|
||||
+46
-9
@@ -140,6 +140,11 @@ def _run_google_scan(options: dict):
|
||||
max_file_mb = float(scan_opts.get("max_file_mb", 50.0))
|
||||
scan_body = bool(scan_opts.get("scan_body", True))
|
||||
scan_att = bool(scan_opts.get("scan_attachments", True))
|
||||
delta_enabled = bool(scan_opts.get("delta", False))
|
||||
|
||||
from checkpoint import _load_delta_tokens, _save_delta_tokens
|
||||
_drive_delta_tokens: dict = _load_delta_tokens() if delta_enabled else {}
|
||||
_new_drive_tokens: dict = {}
|
||||
|
||||
# Resolve users: explicit list → Admin SDK → fall back to SA email itself
|
||||
_user_role_map: dict = {} # email → role
|
||||
@@ -283,12 +288,35 @@ def _run_google_scan(options: dict):
|
||||
# ── Google Drive ──────────────────────────────────────────────────────
|
||||
if "gdrive" in sources:
|
||||
try:
|
||||
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive"})
|
||||
for meta, data in conn.iter_drive_files(
|
||||
user_email,
|
||||
max_files=max_files,
|
||||
max_file_mb=max_file_mb,
|
||||
):
|
||||
delta_key = f"gdrive:{user_email}"
|
||||
saved_token = _drive_delta_tokens.get(delta_key) if delta_enabled else None
|
||||
|
||||
if delta_enabled and saved_token:
|
||||
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive (delta)"})
|
||||
try:
|
||||
drive_items, new_token = conn.get_drive_changes(
|
||||
user_email, saved_token,
|
||||
max_files=max_files, max_file_mb=max_file_mb,
|
||||
)
|
||||
_new_drive_tokens[delta_key] = new_token
|
||||
except Exception as delta_err:
|
||||
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive (delta token invalid — full scan)"})
|
||||
logger.warning("[gdrive delta] %s: %s — falling back to full scan", user_email, delta_err)
|
||||
drive_items = list(conn.iter_drive_files(user_email, max_files=max_files, max_file_mb=max_file_mb))
|
||||
try:
|
||||
_new_drive_tokens[delta_key] = conn.get_drive_start_token(user_email)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
broadcast("scan_phase", {"phase": f"{user_email} — Google Drive"})
|
||||
drive_items = list(conn.iter_drive_files(user_email, max_files=max_files, max_file_mb=max_file_mb))
|
||||
if delta_enabled:
|
||||
try:
|
||||
_new_drive_tokens[delta_key] = conn.get_drive_start_token(user_email)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for meta, data in drive_items:
|
||||
if _check_abort():
|
||||
return
|
||||
total_scanned += 1
|
||||
@@ -306,7 +334,7 @@ def _run_google_scan(options: dict):
|
||||
except Exception as e:
|
||||
broadcast("scan_error", {"file": meta.get("name", ""), "error": str(e)})
|
||||
continue
|
||||
cprs = result.get("cprs", [])
|
||||
cprs = result.get("cprs", [])
|
||||
pii_counts = result.get("pii_counts")
|
||||
if cprs or (pii_counts and any(pii_counts.values())):
|
||||
_broadcast_card(meta, cprs, pii_counts)
|
||||
@@ -315,11 +343,20 @@ def _run_google_scan(options: dict):
|
||||
except Exception as e:
|
||||
broadcast("scan_error", {"file": f"Drive/{user_email}", "error": str(e)})
|
||||
|
||||
if delta_enabled and _new_drive_tokens:
|
||||
try:
|
||||
current_tokens = _load_delta_tokens()
|
||||
_save_delta_tokens({**current_tokens, **_new_drive_tokens})
|
||||
except Exception as e:
|
||||
logger.warning("[gdrive delta] token save failed: %s", e)
|
||||
|
||||
elapsed = _time.monotonic() - t_start
|
||||
broadcast("google_scan_done", {
|
||||
"flagged_count": total_flagged,
|
||||
"total_scanned": total_scanned,
|
||||
"flagged_count": total_flagged,
|
||||
"total_scanned": total_scanned,
|
||||
"elapsed_seconds": round(elapsed, 1),
|
||||
"delta": delta_enabled and bool(_new_drive_tokens),
|
||||
"delta_sources": len(_new_drive_tokens),
|
||||
})
|
||||
if _db and _db_scan_id:
|
||||
try:
|
||||
|
||||
@@ -14,6 +14,10 @@ from app_config import (
|
||||
set_viewer_pin,
|
||||
verify_viewer_pin,
|
||||
clear_viewer_pin,
|
||||
get_interface_pin_hash,
|
||||
set_interface_pin,
|
||||
verify_interface_pin,
|
||||
clear_interface_pin,
|
||||
)
|
||||
|
||||
bp = Blueprint("viewer", __name__)
|
||||
@@ -161,6 +165,44 @@ def pin_clear():
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
# ── Interface PIN management endpoints ───────────────────────────────────────
|
||||
|
||||
@bp.route("/api/interface/pin", methods=["GET"])
|
||||
def interface_pin_status():
|
||||
"""Return whether an interface PIN is currently set."""
|
||||
return jsonify({"pin_set": bool(get_interface_pin_hash())})
|
||||
|
||||
|
||||
@bp.route("/api/interface/pin", methods=["POST"])
|
||||
def interface_pin_set():
|
||||
"""Set or change the interface PIN.
|
||||
Body: {pin: "...", current_pin: "..."}
|
||||
current_pin required only when a PIN is already set.
|
||||
"""
|
||||
body = request.get_json(silent=True) or {}
|
||||
new_pin = str(body.get("pin", "")).strip()
|
||||
if not new_pin:
|
||||
return jsonify({"error": "pin required"}), 400
|
||||
if not new_pin.isdigit() or not (4 <= len(new_pin) <= 8):
|
||||
return jsonify({"error": "PIN must be 4–8 digits"}), 400
|
||||
if get_interface_pin_hash():
|
||||
if not verify_interface_pin(str(body.get("current_pin", "")).strip()):
|
||||
return jsonify({"error": "current PIN is incorrect"}), 403
|
||||
set_interface_pin(new_pin)
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@bp.route("/api/interface/pin", methods=["DELETE"])
|
||||
def interface_pin_clear():
|
||||
"""Remove the interface PIN. Requires current PIN if one is set."""
|
||||
body = request.get_json(silent=True) or {}
|
||||
if get_interface_pin_hash():
|
||||
if not verify_interface_pin(str(body.get("current_pin", "")).strip()):
|
||||
return jsonify({"error": "current PIN is incorrect"}), 403
|
||||
clear_interface_pin()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@bp.route("/api/viewer/pin/verify", methods=["POST"])
|
||||
def pin_verify():
|
||||
"""Verify a PIN submission and set a viewer session on success."""
|
||||
|
||||
Reference in new issue
Block a user