Extended the M365 checkpoint/resume mechanism to all three scan engines. Each engine writes its own +file (checkpoint_m365.json, checkpoint_google.json, checkpoint_file_{source_id}.json) every 25 + items.

routes/google_scan.py

@@ -144,7 +144,8 @@ def _run_google_scan(options: dict):
     scan_emails   = bool(scan_opts.get("scan_emails",  False))
     scan_phones   = bool(scan_opts.get("scan_phones",  False))
 
-    from checkpoint import _load_delta_tokens, _save_delta_tokens
+    from checkpoint import (_load_delta_tokens, _save_delta_tokens,
+                            _save_checkpoint, _load_checkpoint, _clear_checkpoint)
     _drive_delta_tokens: dict = _load_delta_tokens() if delta_enabled else {}
     _new_drive_tokens:   dict = {}
 
@@ -195,6 +196,28 @@ def _run_google_scan(options: dict):
         except Exception as e:
             logger.error("[google_scan] begin_scan failed: %s", e)
 
+    # ── Checkpoint: resume from a previous interrupted Google scan ────────────
+    import hashlib as _hl, json as _js
+    _gck_prefix = "google"
+    _gck_key    = _hl.sha256(_js.dumps({
+        "emails":  sorted(user_emails),
+        "sources": sorted(sources),
+        "older_than_days": scan_opts.get("older_than_days", 0),
+    }, sort_keys=True).encode()).hexdigest()[:16]
+    _gck             = _load_checkpoint(_gck_key, prefix=_gck_prefix)
+    _g_scanned_ids:  set  = set(_gck["scanned_ids"]) if _gck else set()
+    _google_flagged: list = []  # items found by this Google scan (for checkpoint)
+    _gck_resumed = len(_g_scanned_ids)
+    if _gck:
+        from scan_engine import _with_disposition as _wd_ck
+        _google_flagged = list(_gck.get("flagged", []))
+        flagged_items.extend(_google_flagged)
+        broadcast("scan_phase", {"phase": f"Resuming — skipping {_gck_resumed} already-scanned items…"})
+        for _card in _google_flagged:
+            broadcast("scan_file_flagged", _wd_ck(_card, _db))
+    _GCHECKPOINT_SAVE_EVERY = 25
+    _g_items_since_save = 0
+
     total_flagged = 0
     total_scanned = 0
     t_start = _time.monotonic()
@@ -234,6 +257,7 @@ def _run_google_scan(options: dict):
             "exif":             {},
         }
         flagged_items.append(card)
+        _google_flagged.append(card)
         broadcast("scan_file_flagged", _with_disposition(card, _db))
         total_flagged += 1
         if _db and _db_scan_id:
@@ -265,6 +289,10 @@ def _run_google_scan(options: dict):
                 ):
                     if _check_abort():
                         return
+                    _item_id = meta.get("id", "")
+                    if _item_id in _g_scanned_ids:
+                        total_scanned += 1
+                        continue
                     total_scanned += 1
                     broadcast("scan_file", {"file": meta.get("name", "")})
                     broadcast("scan_progress", {
@@ -279,6 +307,7 @@ def _run_google_scan(options: dict):
                         result = _scan_bytes(data, meta.get("name", "msg.txt"))
                     except Exception as e:
                         broadcast("scan_error", {"file": meta.get("name", ""), "error": str(e)})
+                        _g_scanned_ids.add(_item_id)
                         continue
                     cprs       = result.get("cprs", [])
                     pii_counts = result.get("pii_counts")
@@ -288,6 +317,11 @@ def _run_google_scan(options: dict):
                         meta["_email_count"] = len(_em)
                         meta["_phone_count"] = len(_ph)
                         _broadcast_card(meta, cprs, pii_counts)
+                    _g_scanned_ids.add(_item_id)
+                    _g_items_since_save += 1
+                    if _g_items_since_save >= _GCHECKPOINT_SAVE_EVERY:
+                        _save_checkpoint(_gck_key, _g_scanned_ids, _google_flagged, {}, prefix=_gck_prefix)
+                        _g_items_since_save = 0
             except GoogleError as e:
                 broadcast("scan_error", {"file": f"Gmail/{user_email}", "error": str(e)})
             except Exception as e:
@@ -327,6 +361,10 @@ def _run_google_scan(options: dict):
                 for meta, data in drive_items:
                     if _check_abort():
                         return
+                    _item_id = meta.get("id", "")
+                    if _item_id in _g_scanned_ids:
+                        total_scanned += 1
+                        continue
                     total_scanned += 1
                     broadcast("scan_file", {"file": meta.get("name", "")})
                     broadcast("scan_progress", {
@@ -341,6 +379,7 @@ def _run_google_scan(options: dict):
                         result = _scan_bytes(data, meta.get("name", "file"))
                     except Exception as e:
                         broadcast("scan_error", {"file": meta.get("name", ""), "error": str(e)})
+                        _g_scanned_ids.add(_item_id)
                         continue
                     cprs       = result.get("cprs", [])
                     pii_counts = result.get("pii_counts")
@@ -350,6 +389,11 @@ def _run_google_scan(options: dict):
                         meta["_email_count"] = len(_em)
                         meta["_phone_count"] = len(_ph)
                         _broadcast_card(meta, cprs, pii_counts)
+                    _g_scanned_ids.add(_item_id)
+                    _g_items_since_save += 1
+                    if _g_items_since_save >= _GCHECKPOINT_SAVE_EVERY:
+                        _save_checkpoint(_gck_key, _g_scanned_ids, _google_flagged, {}, prefix=_gck_prefix)
+                        _g_items_since_save = 0
             except GoogleError as e:
                 broadcast("scan_error", {"file": f"Drive/{user_email}", "error": str(e)})
             except Exception as e:
@@ -362,6 +406,10 @@ def _run_google_scan(options: dict):
         except Exception as e:
             logger.warning("[gdrive delta] token save failed: %s", e)
 
+    from gdpr_scanner import _scan_abort as _gsa
+    if not _gsa.is_set():
+        _clear_checkpoint(prefix=_gck_prefix)
+
     elapsed = _time.monotonic() - t_start
     broadcast("google_scan_done", {
         "flagged_count":   total_flagged,

routes/scan.py

@@ -13,7 +13,7 @@ from app_config import (
 )
 from checkpoint import (
     _checkpoint_key, _load_checkpoint, _clear_checkpoint,
-    _load_delta_tokens, _DELTA_PATH,
+    _load_delta_tokens, _DELTA_PATH, _cp_path,
 )
 
 bp = Blueprint("scan", __name__)
@@ -121,28 +121,80 @@ def scan_stop():
 def scan_checkpoint_info():
     """Return info about any saved checkpoint for the given scan options.
     If check_only=true, just reports whether a scan is currently running."""
+    import hashlib, json as _json
     options = request.get_json() or {}
     if options.get("check_only"):
         acquired = state._scan_lock.acquire(blocking=False)
         if acquired:
             state._scan_lock.release()
         return jsonify({"running": not acquired})
-    key = _checkpoint_key(options)
-    cp  = _load_checkpoint(key)
-    if not cp:
+
+    engines = {}
+
+    # M365
+    if options.get("sources"):
+        key = _checkpoint_key(options)
+        cp  = _load_checkpoint(key, prefix="m365")
+        if cp:
+            engines["m365"] = {
+                "exists":        True,
+                "scanned_count": len(cp.get("scanned_ids", [])),
+                "flagged_count": len(cp.get("flagged", [])),
+                "started_at":    cp.get("meta", {}).get("started_at"),
+            }
+
+    # Google
+    google_emails  = options.get("googleUserEmails", [])
+    google_sources = options.get("googleSources", [])
+    if google_emails and google_sources:
+        gkey = hashlib.sha256(_json.dumps({
+            "emails":  sorted(google_emails),
+            "sources": sorted(google_sources),
+            "older_than_days": options.get("options", {}).get("older_than_days", 0),
+        }, sort_keys=True).encode()).hexdigest()[:16]
+        cp = _load_checkpoint(gkey, prefix="google")
+        if cp:
+            engines["google"] = {
+                "exists":        True,
+                "scanned_count": len(cp.get("scanned_ids", [])),
+                "flagged_count": len(cp.get("flagged", [])),
+                "started_at":    cp.get("meta", {}).get("started_at"),
+            }
+
+    # File sources (one checkpoint per source ID)
+    for src_id in options.get("fileSources", []):
+        fkey = _checkpoint_key({"sources": ["file"], "user_ids": [src_id], "options": {}})
+        cp   = _load_checkpoint(fkey, prefix=f"file_{src_id}")
+        if cp:
+            fe = engines.setdefault("file", {"exists": True, "scanned_count": 0, "flagged_count": 0, "started_at": None})
+            fe["scanned_count"] += len(cp.get("scanned_ids", []))
+            fe["flagged_count"]  += len(cp.get("flagged", []))
+            if not fe["started_at"]:
+                fe["started_at"] = cp.get("meta", {}).get("started_at")
+
+    if not engines:
         return jsonify({"exists": False})
+
+    started_ats = [v["started_at"] for v in engines.values() if v.get("started_at")]
     return jsonify({
         "exists":        True,
-        "scanned_count": len(cp.get("scanned_ids", [])),
-        "flagged_count": len(cp.get("flagged", [])),
-        "started_at":    cp.get("meta", {}).get("started_at"),
+        "scanned_count": sum(v.get("scanned_count", 0) for v in engines.values()),
+        "flagged_count": sum(v.get("flagged_count", 0) for v in engines.values()),
+        "started_at":    min(started_ats) if started_ats else None,
+        "engines":       engines,
     })
 
 
 @bp.route("/api/scan/clear_checkpoint", methods=["POST"])
 def scan_clear_checkpoint():
-    """Discard any saved checkpoint so the next scan starts fresh."""
-    _clear_checkpoint()
+    """Discard all saved checkpoints so the next scan starts fresh."""
+    from pathlib import Path
+    data_dir = Path.home() / ".gdprscanner"
+    for f in data_dir.glob("checkpoint_*.json"):
+        try:
+            f.unlink()
+        except Exception:
+            pass
     return jsonify({"status": "cleared"})