Refactor golden image handling in backup upload process</message>
<message>Update the _set_golden_from_path function to improve the handling of existing golden image files. Replace the existing unlink logic with a more robust method that safely removes files or broken symlinks using the missing_ok parameter. This change enhances the reliability of the backup upload process by ensuring that stale references are properly cleared before setting a new golden image path.
This commit is contained in:
nearxos
286
backup-from-device/gnss-guard/tm-gnss-guard/storage/cleanup.py
Normal file
286
backup-from-device/gnss-guard/tm-gnss-guard/storage/cleanup.py
Normal file
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cleanup manager for GNSS Guard
|
||||
Handles cleanup of database tables and log files
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("gnss_guard.cleanup")
|
||||
|
||||
|
||||
class CleanupManager:
|
||||
"""Manages cleanup of old data from database and logs"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database_path: Path,
|
||||
logs_base_path: Path,
|
||||
positions_raw_retention_days: int = 14,
|
||||
positions_validation_retention_days: int = 31,
|
||||
logs_retention_days: int = 14,
|
||||
demo_mode: bool = False
|
||||
):
|
||||
"""
|
||||
Initialize cleanup manager
|
||||
|
||||
Args:
|
||||
database_path: Path to SQLite database file
|
||||
logs_base_path: Base path for logs directory
|
||||
positions_raw_retention_days: Days to retain positions_raw records (default: 14)
|
||||
positions_validation_retention_days: Days to retain positions_validation records (default: 31)
|
||||
logs_retention_days: Days to retain log files (default: 14)
|
||||
demo_mode: If True, skip database cleanup (data isn't growing in demo mode)
|
||||
"""
|
||||
self.database_path = Path(database_path)
|
||||
self.logs_base_path = Path(logs_base_path)
|
||||
self.positions_raw_retention_days = positions_raw_retention_days
|
||||
self.positions_validation_retention_days = positions_validation_retention_days
|
||||
self.logs_retention_days = logs_retention_days
|
||||
self.demo_mode = demo_mode
|
||||
self._last_cleanup_date: Optional[str] = None
|
||||
|
||||
def run_cleanup_if_needed(self):
|
||||
"""Run cleanup once per day (checks if already ran today)
|
||||
|
||||
In demo mode, only log cleanup runs (database cleanup is skipped
|
||||
since data isn't growing - records are created and deleted in demo mode).
|
||||
"""
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
|
||||
if self._last_cleanup_date == today:
|
||||
return # Already ran today
|
||||
|
||||
# In demo mode, skip database cleanup entirely but still clean logs
|
||||
if self.demo_mode:
|
||||
logger.info("Demo mode: skipping database cleanup (data not growing)")
|
||||
try:
|
||||
files_deleted, dirs_deleted = self._cleanup_logs()
|
||||
self._last_cleanup_date = today
|
||||
|
||||
if files_deleted > 0 or dirs_deleted > 0:
|
||||
logger.info(
|
||||
f"Demo mode cleanup completed: "
|
||||
f"{files_deleted} log files, "
|
||||
f"{dirs_deleted} empty directories"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Demo mode log cleanup failed: {e}")
|
||||
return
|
||||
|
||||
logger.info("Starting daily cleanup...")
|
||||
|
||||
try:
|
||||
raw_deleted = self._cleanup_positions_raw()
|
||||
validation_deleted = self._cleanup_positions_validation()
|
||||
files_deleted, dirs_deleted = self._cleanup_logs()
|
||||
|
||||
# Optimize database after cleanup (VACUUM reclaims space, ANALYZE updates statistics)
|
||||
space_saved = self._optimize_database()
|
||||
|
||||
self._last_cleanup_date = today
|
||||
|
||||
logger.info(
|
||||
f"Daily cleanup completed: "
|
||||
f"{raw_deleted} raw positions, "
|
||||
f"{validation_deleted} validations, "
|
||||
f"{files_deleted} log files, "
|
||||
f"{dirs_deleted} empty directories"
|
||||
f"{f', {space_saved}' if space_saved else ''}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Cleanup failed: {e}")
|
||||
|
||||
def _cleanup_positions_raw(self) -> int:
|
||||
"""
|
||||
Delete positions_raw records older than retention period
|
||||
|
||||
Returns:
|
||||
Number of records deleted
|
||||
"""
|
||||
cutoff_timestamp = (
|
||||
datetime.now() - timedelta(days=self.positions_raw_retention_days)
|
||||
).timestamp()
|
||||
|
||||
deleted_count = 0
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=30.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Count before delete
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM positions_raw WHERE timestamp_unix < ?",
|
||||
(cutoff_timestamp,)
|
||||
)
|
||||
deleted_count = cursor.fetchone()[0]
|
||||
|
||||
if deleted_count > 0:
|
||||
cursor.execute(
|
||||
"DELETE FROM positions_raw WHERE timestamp_unix < ?",
|
||||
(cutoff_timestamp,)
|
||||
)
|
||||
conn.commit()
|
||||
logger.info(
|
||||
f"Cleaned up {deleted_count} positions_raw records "
|
||||
f"(> {self.positions_raw_retention_days} days)"
|
||||
)
|
||||
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup positions_raw: {e}")
|
||||
|
||||
return deleted_count
|
||||
|
||||
def _cleanup_positions_validation(self) -> int:
|
||||
"""
|
||||
Delete positions_validation records older than retention period
|
||||
|
||||
Returns:
|
||||
Number of records deleted
|
||||
"""
|
||||
cutoff_timestamp = (
|
||||
datetime.now() - timedelta(days=self.positions_validation_retention_days)
|
||||
).timestamp()
|
||||
|
||||
deleted_count = 0
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=30.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Count before delete
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM positions_validation WHERE validation_timestamp_unix < ?",
|
||||
(cutoff_timestamp,)
|
||||
)
|
||||
deleted_count = cursor.fetchone()[0]
|
||||
|
||||
if deleted_count > 0:
|
||||
cursor.execute(
|
||||
"DELETE FROM positions_validation WHERE validation_timestamp_unix < ?",
|
||||
(cutoff_timestamp,)
|
||||
)
|
||||
conn.commit()
|
||||
logger.info(
|
||||
f"Cleaned up {deleted_count} positions_validation records "
|
||||
f"(> {self.positions_validation_retention_days} days)"
|
||||
)
|
||||
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup positions_validation: {e}")
|
||||
|
||||
return deleted_count
|
||||
|
||||
def _cleanup_logs(self) -> tuple:
|
||||
"""
|
||||
Delete log files and empty directories older than retention period
|
||||
|
||||
Returns:
|
||||
Tuple of (files_deleted, directories_deleted)
|
||||
"""
|
||||
cutoff_timestamp = (
|
||||
datetime.now() - timedelta(days=self.logs_retention_days)
|
||||
).timestamp()
|
||||
|
||||
deleted_files = 0
|
||||
deleted_dirs = 0
|
||||
|
||||
try:
|
||||
if not self.logs_base_path.exists():
|
||||
return (0, 0)
|
||||
|
||||
# Delete old log files
|
||||
for log_file in self.logs_base_path.rglob("app_*.json"):
|
||||
try:
|
||||
if log_file.stat().st_mtime < cutoff_timestamp:
|
||||
log_file.unlink()
|
||||
deleted_files += 1
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to delete log file {log_file}: {e}")
|
||||
|
||||
# Clean up empty directories (must iterate multiple times for nested dirs)
|
||||
# Sort by path length descending to delete deepest first
|
||||
all_dirs = sorted(
|
||||
[d for d in self.logs_base_path.rglob("*") if d.is_dir()],
|
||||
key=lambda p: len(str(p)),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
for dir_path in all_dirs:
|
||||
try:
|
||||
# Only delete if empty
|
||||
if not any(dir_path.iterdir()):
|
||||
dir_path.rmdir()
|
||||
deleted_dirs += 1
|
||||
except Exception:
|
||||
pass # Directory not empty or other error
|
||||
|
||||
if deleted_files > 0 or deleted_dirs > 0:
|
||||
logger.info(
|
||||
f"Cleaned up {deleted_files} log files and "
|
||||
f"{deleted_dirs} empty directories "
|
||||
f"(> {self.logs_retention_days} days)"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup logs: {e}")
|
||||
|
||||
return (deleted_files, deleted_dirs)
|
||||
|
||||
def _optimize_database(self) -> str:
|
||||
"""
|
||||
Optimize database after cleanup operations.
|
||||
|
||||
Runs VACUUM to reclaim disk space from deleted records and
|
||||
ANALYZE to update query planner statistics.
|
||||
|
||||
Returns:
|
||||
String describing space saved, or empty string if no optimization needed
|
||||
"""
|
||||
try:
|
||||
# Get database size before optimization
|
||||
size_before = self.database_path.stat().st_size if self.database_path.exists() else 0
|
||||
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=60.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# ANALYZE updates statistics used by the query planner
|
||||
cursor.execute("ANALYZE")
|
||||
|
||||
# VACUUM rebuilds the database file, reclaiming unused space
|
||||
# Note: VACUUM requires exclusive access and can't run inside a transaction
|
||||
cursor.execute("VACUUM")
|
||||
|
||||
conn.close()
|
||||
|
||||
# Get database size after optimization
|
||||
size_after = self.database_path.stat().st_size if self.database_path.exists() else 0
|
||||
|
||||
# Calculate space saved
|
||||
space_saved = size_before - size_after
|
||||
|
||||
if space_saved > 0:
|
||||
# Format size for logging
|
||||
if space_saved >= 1024 * 1024:
|
||||
saved_str = f"{space_saved / (1024 * 1024):.1f} MB"
|
||||
elif space_saved >= 1024:
|
||||
saved_str = f"{space_saved / 1024:.1f} KB"
|
||||
else:
|
||||
saved_str = f"{space_saved} bytes"
|
||||
|
||||
logger.info(f"Database optimized: reclaimed {saved_str}")
|
||||
return f"reclaimed {saved_str}"
|
||||
else:
|
||||
logger.debug("Database optimized (no space reclaimed)")
|
||||
return ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to optimize database: {e}")
|
||||
return ""
|
||||
|
||||
Reference in New Issue
Block a user