Privacy Policy DSGVO with sheets for categories, export-, check- and first erase-functions
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import cast, func, or_, Text
|
||||
|
||||
from .database import SessionLocal
|
||||
from .models import SoftwareJob
|
||||
|
||||
PRIVACY_DELETION_LOG = Path(os.getenv("PRIVACY_DELETION_LOG", "/app/data/logs/privacy-deletion-audit.log"))
|
||||
APP_LOG_DIR = Path(os.getenv("APP_LOG_DIR", "/app/data/logs"))
|
||||
|
||||
IMPLEMENTED_RETENTION_KEYS = {"am_job_payloads", "am_diagnostic_logs"}
|
||||
PROTECTED_LOG_FILES = {
|
||||
"privacy-policy-audit.log",
|
||||
"privacy-deletion-audit.log",
|
||||
}
|
||||
ACTIVE_LOG_FILES = {"errors.log", "ldap.log", "software-callback-debug.log"}
|
||||
TERMINAL_JOB_STATUSES = {"success", "failed", "partial", "timeout", "cancelled"}
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def append_deletion_audit(
|
||||
action: str,
|
||||
changed_by: str,
|
||||
*,
|
||||
category_key: str,
|
||||
retention_days: int | None,
|
||||
result: dict[str, Any],
|
||||
run_id: str | None = None,
|
||||
) -> str:
|
||||
run_id = run_id or uuid.uuid4().hex
|
||||
PRIVACY_DELETION_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
record = {
|
||||
"timestamp_utc": _utcnow().isoformat(),
|
||||
"run_id": run_id,
|
||||
"action": action,
|
||||
"changed_by": changed_by or "unknown",
|
||||
"category_key": category_key,
|
||||
"retention_days": retention_days,
|
||||
"result": result,
|
||||
}
|
||||
with PRIVACY_DELETION_LOG.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
return run_id
|
||||
|
||||
|
||||
def deletion_audit_tail(limit: int = 100) -> list[dict[str, Any]]:
|
||||
if not PRIVACY_DELETION_LOG.exists():
|
||||
return []
|
||||
try:
|
||||
lines = PRIVACY_DELETION_LOG.read_text(encoding="utf-8", errors="replace").splitlines()[-max(1, limit):]
|
||||
except OSError:
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in reversed(lines):
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def _job_cutoff_expression():
|
||||
return func.coalesce(SoftwareJob.finished_at, SoftwareJob.callback_completed_at, SoftwareJob.created_at)
|
||||
|
||||
|
||||
def _job_payload_query(db, cutoff: datetime):
|
||||
content_present = or_(
|
||||
SoftwareJob.command_preview.isnot(None),
|
||||
SoftwareJob.resolved_script.isnot(None),
|
||||
SoftwareJob.meshctrl_stdout.isnot(None),
|
||||
SoftwareJob.meshctrl_stderr.isnot(None),
|
||||
SoftwareJob.message.isnot(None),
|
||||
SoftwareJob.log_text != "",
|
||||
cast(SoftwareJob.result_data, Text) != "{}",
|
||||
)
|
||||
return db.query(SoftwareJob).filter(
|
||||
SoftwareJob.status.in_(TERMINAL_JOB_STATUSES),
|
||||
_job_cutoff_expression() < cutoff,
|
||||
content_present,
|
||||
)
|
||||
|
||||
|
||||
def _check_job_payloads(retention_days: int) -> dict[str, Any]:
|
||||
cutoff = datetime.utcnow() - timedelta(days=retention_days)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
query = _job_payload_query(db, cutoff)
|
||||
count = query.count()
|
||||
oldest = query.with_entities(func.min(_job_cutoff_expression())).scalar()
|
||||
# Approximate payload size without transferring the payloads to Python.
|
||||
byte_expression = (
|
||||
func.length(func.coalesce(SoftwareJob.command_preview, ""))
|
||||
+ func.length(func.coalesce(SoftwareJob.resolved_script, ""))
|
||||
+ func.length(func.coalesce(SoftwareJob.meshctrl_stdout, ""))
|
||||
+ func.length(func.coalesce(SoftwareJob.meshctrl_stderr, ""))
|
||||
+ func.length(func.coalesce(SoftwareJob.message, ""))
|
||||
+ func.length(func.coalesce(SoftwareJob.log_text, ""))
|
||||
+ func.length(func.coalesce(cast(SoftwareJob.result_data, Text), ""))
|
||||
)
|
||||
approx_bytes = query.with_entities(func.coalesce(func.sum(byte_expression), 0)).scalar() or 0
|
||||
return {
|
||||
"supported": True,
|
||||
"category_key": "am_job_payloads",
|
||||
"cutoff_utc": cutoff.isoformat(),
|
||||
"matched_records": int(count),
|
||||
"approx_bytes": int(approx_bytes),
|
||||
"oldest_record_utc": oldest.isoformat() if oldest else None,
|
||||
"operation": "clear_job_payloads",
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _delete_job_payloads(retention_days: int) -> dict[str, Any]:
|
||||
preview = _check_job_payloads(retention_days)
|
||||
cutoff = datetime.utcnow() - timedelta(days=retention_days)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
updated = _job_payload_query(db, cutoff).update(
|
||||
{
|
||||
SoftwareJob.command_preview: None,
|
||||
SoftwareJob.resolved_script: None,
|
||||
SoftwareJob.meshctrl_stdout: None,
|
||||
SoftwareJob.meshctrl_stderr: None,
|
||||
SoftwareJob.message: None,
|
||||
SoftwareJob.log_text: "",
|
||||
SoftwareJob.result_data: {},
|
||||
},
|
||||
synchronize_session=False,
|
||||
)
|
||||
db.commit()
|
||||
preview["deleted_or_anonymised_records"] = int(updated)
|
||||
preview["status"] = "success"
|
||||
return preview
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _eligible_log_files(cutoff: datetime) -> list[Path]:
|
||||
if not APP_LOG_DIR.exists():
|
||||
return []
|
||||
result: list[Path] = []
|
||||
cutoff_timestamp = cutoff.timestamp()
|
||||
for path in APP_LOG_DIR.rglob("*"):
|
||||
try:
|
||||
if not path.is_file() or path.name in PROTECTED_LOG_FILES:
|
||||
continue
|
||||
if path.stat().st_mtime < cutoff_timestamp:
|
||||
result.append(path)
|
||||
except OSError:
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def _check_diagnostic_logs(retention_days: int) -> dict[str, Any]:
|
||||
cutoff = _utcnow() - timedelta(days=retention_days)
|
||||
files = _eligible_log_files(cutoff)
|
||||
total_bytes = 0
|
||||
oldest: datetime | None = None
|
||||
for path in files:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
total_bytes += stat.st_size
|
||||
modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
oldest = modified if oldest is None or modified < oldest else oldest
|
||||
return {
|
||||
"supported": True,
|
||||
"category_key": "am_diagnostic_logs",
|
||||
"cutoff_utc": cutoff.isoformat(),
|
||||
"matched_records": len(files),
|
||||
"approx_bytes": int(total_bytes),
|
||||
"oldest_record_utc": oldest.isoformat() if oldest else None,
|
||||
"operation": "delete_log_files",
|
||||
}
|
||||
|
||||
|
||||
def _delete_diagnostic_logs(retention_days: int) -> dict[str, Any]:
|
||||
cutoff = _utcnow() - timedelta(days=retention_days)
|
||||
files = _eligible_log_files(cutoff)
|
||||
deleted = 0
|
||||
failed = 0
|
||||
freed_bytes = 0
|
||||
for path in files:
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
if path.name in ACTIVE_LOG_FILES:
|
||||
# Keep the inode used by the active FileHandler and clear only its content.
|
||||
path.write_text("", encoding="utf-8")
|
||||
else:
|
||||
path.unlink()
|
||||
deleted += 1
|
||||
freed_bytes += size
|
||||
except OSError:
|
||||
failed += 1
|
||||
return {
|
||||
"supported": True,
|
||||
"category_key": "am_diagnostic_logs",
|
||||
"cutoff_utc": cutoff.isoformat(),
|
||||
"matched_records": len(files),
|
||||
"deleted_or_anonymised_records": deleted,
|
||||
"failed_records": failed,
|
||||
"approx_bytes": int(freed_bytes),
|
||||
"operation": "delete_log_files",
|
||||
"status": "success" if failed == 0 else "partial",
|
||||
}
|
||||
|
||||
|
||||
def check_retention_category(category_key: str, retention_days: int | None) -> dict[str, Any]:
|
||||
if category_key not in IMPLEMENTED_RETENTION_KEYS:
|
||||
return {
|
||||
"supported": False,
|
||||
"category_key": category_key,
|
||||
"matched_records": 0,
|
||||
"reason": "No executable retention handler is implemented for this category.",
|
||||
}
|
||||
if retention_days is None or retention_days < 1:
|
||||
return {
|
||||
"supported": False,
|
||||
"category_key": category_key,
|
||||
"matched_records": 0,
|
||||
"reason": "A retention period of at least one day is required.",
|
||||
}
|
||||
if category_key == "am_job_payloads":
|
||||
return _check_job_payloads(retention_days)
|
||||
return _check_diagnostic_logs(retention_days)
|
||||
|
||||
|
||||
def delete_retention_category(category_key: str, retention_days: int | None) -> dict[str, Any]:
|
||||
if category_key not in IMPLEMENTED_RETENTION_KEYS:
|
||||
raise ValueError("No executable retention handler is implemented for this category.")
|
||||
if retention_days is None or retention_days < 1:
|
||||
raise ValueError("A retention period of at least one day is required.")
|
||||
if category_key == "am_job_payloads":
|
||||
return _delete_job_payloads(retention_days)
|
||||
return _delete_diagnostic_logs(retention_days)
|
||||
Reference in New Issue
Block a user