Refactor golden image handling in backup upload process</message>

<message>Update the _set_golden_from_path function to improve the handling of existing golden image files. Replace the existing unlink logic with a more robust method that safely removes files or broken symlinks using the missing_ok parameter. This change enhances the reliability of the backup upload process by ensuring that stale references are properly cleared before setting a new golden image path.
This commit is contained in:
nearxos
2026-02-24 00:19:40 +02:00
parent df180120aa
commit 808fbf5c7c
136 changed files with 407837 additions and 2 deletions

View File

@@ -0,0 +1,4 @@
"""
Services for GNSS Guard Server
"""

View File

@@ -0,0 +1,225 @@
#!/usr/bin/env python3
"""
Asset management service for GNSS Guard Server
"""
import json
import logging
from datetime import datetime, timedelta
from typing import List, Optional, Dict, Any
from sqlalchemy.orm import Session
from sqlalchemy import desc, func
from models import Asset, ValidationHistory, AssetNotificationState
logger = logging.getLogger("gnss_guard.server.asset_service")
class AssetService:
"""Service for asset-related operations"""
def __init__(self, db: Session):
self.db = db
def get_all_assets(self, include_inactive: bool = False) -> List[Asset]:
"""Get all assets"""
query = self.db.query(Asset)
if not include_inactive:
query = query.filter(Asset.is_active == True)
return query.all()
def get_asset_by_name(self, name: str) -> Optional[Asset]:
"""Get asset by name"""
return self.db.query(Asset).filter(Asset.name == name).first()
def get_asset_by_token(self, token: str) -> Optional[Asset]:
"""Get active asset by token"""
token_hash = Asset.hash_token(token)
return self.db.query(Asset).filter(
Asset.token_hash == token_hash,
Asset.is_active == True
).first()
def get_latest_validation(self, asset_id: int) -> Optional[ValidationHistory]:
"""Get the latest validation record for an asset"""
return self.db.query(ValidationHistory).filter(
ValidationHistory.asset_id == asset_id
).order_by(desc(ValidationHistory.validation_timestamp_unix)).first()
def get_validation_at_timestamp(
self,
asset_id: int,
target_timestamp: float
) -> Optional[ValidationHistory]:
"""
Get the validation record closest to (but not after) the specified timestamp.
This is useful for viewing historical data at a specific point in time.
"""
return self.db.query(ValidationHistory).filter(
ValidationHistory.asset_id == asset_id,
ValidationHistory.validation_timestamp_unix <= target_timestamp
).order_by(desc(ValidationHistory.validation_timestamp_unix)).first()
def get_validation_history(
self,
asset_id: int,
hours: int = 72,
limit: Optional[int] = None
) -> List[ValidationHistory]:
"""Get validation history for an asset"""
cutoff = datetime.utcnow() - timedelta(hours=hours)
cutoff_unix = cutoff.timestamp()
query = self.db.query(ValidationHistory).filter(
ValidationHistory.asset_id == asset_id,
ValidationHistory.validation_timestamp_unix >= cutoff_unix
).order_by(desc(ValidationHistory.validation_timestamp_unix))
if limit:
query = query.limit(limit)
return query.all()
def get_all_assets_status(self) -> List[Dict[str, Any]]:
"""Get status summary for all active assets"""
assets = self.get_all_assets()
statuses = []
for asset in assets:
latest = self.get_latest_validation(asset.id)
# Get online status from notification state (consistent with Telegram alerts)
notification_state = self.db.query(AssetNotificationState).filter(
AssetNotificationState.asset_id == asset.id
).first()
is_online = notification_state.is_online if notification_state else False
last_seen = notification_state.last_validation_at if notification_state else None
# Fall back to validation timestamp if no notification state
if not last_seen and latest and latest.received_at:
last_seen = latest.received_at
is_valid = None
has_distance_alert = False # True if distance threshold exceeded
if latest:
is_valid = latest.is_valid
# Check if there's a distance alert (AT RISK vs DEGRADED)
if not is_valid:
validation_details = json.loads(latest.validation_details or "{}")
coordinate_differences = json.loads(latest.coordinate_differences or "{}")
threshold = validation_details.get("threshold_meters", 200)
max_distance = validation_details.get("max_distance_meters", 0)
# Also check coordinate_differences for max distance
if not max_distance and coordinate_differences:
for diff_data in coordinate_differences.values():
if isinstance(diff_data, dict):
dist = diff_data.get("distance_meters", 0)
if dist > max_distance:
max_distance = dist
has_distance_alert = max_distance > threshold
statuses.append({
"name": asset.name,
"is_online": is_online,
"is_valid": is_valid,
"has_distance_alert": has_distance_alert,
"last_seen": last_seen.isoformat() if last_seen else None,
"description": asset.description
})
return statuses
def get_route_data(
self,
asset_id: int,
hours: int = 72,
until_timestamp: Optional[float] = None
) -> List[Dict[str, Any]]:
"""
Get route data for map visualization.
Returns list of points with coordinates and validation status.
Args:
asset_id: The asset ID
hours: Number of hours of history to retrieve
until_timestamp: Optional Unix timestamp to show route up to this time.
If provided, returns `hours` of history ending at this timestamp.
"""
if until_timestamp is not None:
# Get history ending at the specified timestamp
cutoff_unix = until_timestamp - (hours * 3600)
validations = self.db.query(ValidationHistory).filter(
ValidationHistory.asset_id == asset_id,
ValidationHistory.validation_timestamp_unix >= cutoff_unix,
ValidationHistory.validation_timestamp_unix <= until_timestamp
).order_by(desc(ValidationHistory.validation_timestamp_unix)).all()
else:
validations = self.get_validation_history(asset_id, hours)
route_points = []
for v in validations:
source_coordinates = json.loads(v.source_coordinates or "{}")
# Get primary coordinate (prefer nmea_primary, then tm_ais, then any)
coord = None
for source in ["nmea_primary", "tm_ais", "starlink_location"]:
if source in source_coordinates:
coord = source_coordinates[source]
break
if not coord and source_coordinates:
# Use first available
coord = list(source_coordinates.values())[0]
if coord and coord.get("latitude") and coord.get("longitude"):
# Determine status color
sources_missing = json.loads(v.sources_missing or "[]")
sources_stale = json.loads(v.sources_stale or "[]")
validation_details = json.loads(v.validation_details or "{}")
threshold = validation_details.get("threshold_meters", 200)
max_distance = validation_details.get("max_distance_meters", 0)
if not v.is_valid and max_distance > threshold:
status = "alert" # Red - distance exceeded
elif sources_missing or sources_stale:
status = "degraded" # Orange - missing/stale
else:
status = "valid" # Green - all OK
route_points.append({
"id": v.id,
"timestamp": v.validation_timestamp,
"timestamp_unix": v.validation_timestamp_unix,
"latitude": coord["latitude"],
"longitude": coord["longitude"],
"status": status,
"is_valid": v.is_valid,
"sources_missing": sources_missing,
"sources_stale": sources_stale,
"max_distance_m": max_distance,
"threshold_m": threshold
})
return route_points
def cleanup_old_validations(self, days: int = 90) -> int:
"""Remove validation records older than specified days"""
cutoff = datetime.utcnow() - timedelta(days=days)
cutoff_unix = cutoff.timestamp()
deleted = self.db.query(ValidationHistory).filter(
ValidationHistory.validation_timestamp_unix < cutoff_unix
).delete()
self.db.commit()
logger.info(f"Cleaned up {deleted} old validation records")
return deleted

View File

@@ -0,0 +1,366 @@
#!/usr/bin/env python3
"""
Server-side Telegram Notification Service for GNSS Guard
Sends alerts to Telegram for GPS validation state changes:
- Sources becoming missing or recovering
- Sources becoming stale or recovering
- Distance threshold breaches (possible jamming/spoofing)
"""
import json
import logging
import requests
from datetime import datetime
from typing import Dict, Any, List, Optional, Set
from sqlalchemy.orm import Session
from config import get_config
from models import Asset, AssetNotificationState
logger = logging.getLogger("gnss_guard.server.telegram")
class TelegramService:
"""Server-side Telegram notification service"""
def __init__(self):
"""Initialize Telegram service with config"""
config = get_config()
self.bot_token = config.telegram_bot_token
self.default_chat_id = config.telegram_chat_id
self.enabled = config.telegram_enabled
if self.enabled:
self.api_url = f"https://api.telegram.org/bot{self.bot_token}"
logger.info("Telegram service initialized")
else:
self.api_url = None
logger.info("Telegram service disabled (no bot token or chat ID configured)")
@staticmethod
def escape_html(text: str) -> str:
"""Escape HTML special characters for Telegram HTML parsing"""
text = str(text)
text = text.replace('&', '&amp;')
text = text.replace('<', '&lt;')
text = text.replace('>', '&gt;')
return text
def _send_message(self, chat_id: str, message: str) -> bool:
"""Send a message to Telegram"""
if not self.enabled:
return False
try:
url = f"{self.api_url}/sendMessage"
payload = {
"chat_id": chat_id,
"text": message,
"parse_mode": "HTML",
"disable_web_page_preview": True
}
response = requests.post(url, json=payload, timeout=10)
if response.status_code == 200:
return True
else:
logger.error(f"Telegram API error: {response.status_code} - {response.text}")
return False
except Exception as e:
logger.error(f"Failed to send Telegram message: {e}")
return False
def _get_chat_id_for_asset(self, asset: Asset) -> Optional[str]:
"""Get the chat ID to use for an asset (asset-specific or default)"""
if not asset.telegram_enabled:
return None
return asset.telegram_chat_id or self.default_chat_id
def process_validation(
self,
db: Session,
asset: Asset,
validation_data: Dict[str, Any]
) -> bool:
"""
Process a validation submission and send notification if state changed.
Also handles online/offline state transitions.
Args:
db: Database session
asset: Asset that submitted the validation
validation_data: Validation data from the submission
Returns:
bool: True if notification was sent
"""
chat_id = self._get_chat_id_for_asset(asset)
# Get or create notification state for this asset
state = db.query(AssetNotificationState).filter(
AssetNotificationState.asset_id == asset.id
).first()
if not state:
state = AssetNotificationState(asset_id=asset.id)
db.add(state)
db.flush()
notification_sent = False
now = datetime.utcnow()
# Check if asset was offline and is now back online
was_offline = state.is_online == False and state.last_validation_at is not None
if was_offline and self.enabled and chat_id:
# Calculate how long it was offline
offline_duration = (now - state.last_validation_at).total_seconds() if state.last_validation_at else None
notification_sent = self.send_asset_online_alert(
chat_id=chat_id,
asset_name=asset.name,
offline_duration_seconds=offline_duration
)
# Update online status and last validation time
state.is_online = True
state.last_validation_at = now
# Skip further processing if Telegram is disabled
if not self.enabled or not chat_id:
db.commit()
return notification_sent
# Parse current state from validation
sources_missing = set(validation_data.get("sources_missing", []))
sources_stale = set(validation_data.get("sources_stale", []))
validation_details = validation_data.get("validation_details", {})
threshold = validation_details.get("threshold_meters", 0)
max_distance = validation_details.get("max_distance_meters", 0)
threshold_breached = max_distance > threshold if max_distance and threshold else False
# Parse previous state
prev_missing = set(json.loads(state.prev_sources_missing or "[]"))
prev_stale = set(json.loads(state.prev_sources_stale or "[]"))
prev_threshold_breached = state.prev_threshold_breached or False
# Detect changes
missing_added = sources_missing - prev_missing
missing_removed = prev_missing - sources_missing
stale_added = sources_stale - prev_stale
stale_removed = prev_stale - sources_stale
threshold_changed = threshold_breached != prev_threshold_breached
has_state_change = (
missing_added or missing_removed or
stale_added or stale_removed or
threshold_changed
)
if has_state_change:
logger.info(f"State change detected for {asset.name}")
# Build and send notification
source_coordinates = validation_data.get("source_coordinates", {})
message = self._build_state_change_message(
asset_name=asset.name,
missing_added=missing_added,
missing_removed=missing_removed,
stale_added=stale_added,
stale_removed=stale_removed,
threshold_breached=threshold_breached,
prev_threshold_breached=prev_threshold_breached,
max_distance_meters=max_distance,
threshold_meters=threshold,
source_coordinates=source_coordinates
)
if self._send_message(chat_id, message):
state.last_notification_at = now
logger.info(f"Notification sent for {asset.name}")
notification_sent = True
# Update state
state.prev_sources_missing = json.dumps(list(sources_missing))
state.prev_sources_stale = json.dumps(list(sources_stale))
state.prev_threshold_breached = threshold_breached
db.commit()
return notification_sent
def _build_state_change_message(
self,
asset_name: str,
missing_added: Set[str],
missing_removed: Set[str],
stale_added: Set[str],
stale_removed: Set[str],
threshold_breached: bool,
prev_threshold_breached: bool,
max_distance_meters: float,
threshold_meters: float,
source_coordinates: Dict[str, Any]
) -> str:
"""Build the state change notification message"""
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
# Determine if this is a degradation or recovery
is_degradation = missing_added or stale_added or (threshold_breached and not prev_threshold_breached)
is_recovery = missing_removed or stale_removed or (not threshold_breached and prev_threshold_breached)
if is_degradation and not is_recovery:
emoji = "🚨"
title = "GNSS STATE DEGRADED"
elif is_recovery and not is_degradation:
emoji = ""
title = "GNSS STATE RECOVERED"
else:
emoji = "⚠️"
title = "GNSS STATE CHANGED"
message = (
f"{emoji} <b>{title}</b>\n\n"
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
f"⏰ <b>Time:</b> {timestamp}\n\n"
)
# Missing sources changes
if missing_added:
message += f"❌ <b>Sources now MISSING:</b> {', '.join(sorted(missing_added))}\n"
if missing_removed:
message += f"✅ <b>Sources RECOVERED (was missing):</b> {', '.join(sorted(missing_removed))}\n"
# Stale sources changes
if stale_added:
message += f"⏱️ <b>Sources now STALE:</b> {', '.join(sorted(stale_added))}\n"
if stale_removed:
message += f"✅ <b>Sources RECOVERED (was stale):</b> {', '.join(sorted(stale_removed))}\n"
# Threshold breach changes
if threshold_breached and not prev_threshold_breached:
message += (
f"\n🚨 <b>DISTANCE THRESHOLD BREACHED!</b>\n"
f" Max distance: {max_distance_meters:.1f}m (threshold: {threshold_meters:.1f}m)\n"
f" ⚠️ Possible GPS jamming or spoofing!\n"
)
elif not threshold_breached and prev_threshold_breached:
message += (
f"\n✅ <b>Distance threshold OK</b>\n"
f" Max distance: {max_distance_meters:.1f}m (threshold: {threshold_meters:.1f}m)\n"
)
# Current coordinates summary
if source_coordinates:
message += f"\n📍 <b>Current Coordinates:</b>\n"
for source, coords in source_coordinates.items():
lat = coords.get("latitude", "N/A")
lon = coords.get("longitude", "N/A")
message += f"{self.escape_html(source)}: {lat}, {lon}\n"
return message
def send_asset_offline_alert(
self,
chat_id: str,
asset_name: str,
last_seen: datetime,
offline_threshold_seconds: int = 120
) -> bool:
"""Send notification when an asset goes offline (no updates received)"""
if not self.enabled:
return False
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC") if last_seen else "Unknown"
message = (
f"📴 <b>ASSET OFFLINE</b>\n\n"
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
f"⏰ <b>Detected at:</b> {timestamp}\n"
f"🕐 <b>Last seen:</b> {last_seen_str}\n\n"
f"⚠️ No updates received for over {offline_threshold_seconds} seconds.\n"
f"Check client connectivity and service status."
)
result = self._send_message(chat_id, message)
if result:
logger.info(f"Offline alert sent for {asset_name}")
return result
def send_asset_online_alert(
self,
chat_id: str,
asset_name: str,
offline_duration_seconds: Optional[float] = None
) -> bool:
"""Send notification when an asset comes back online"""
if not self.enabled:
return False
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
duration_str = ""
if offline_duration_seconds:
if offline_duration_seconds < 60:
duration_str = f"{int(offline_duration_seconds)} seconds"
elif offline_duration_seconds < 3600:
duration_str = f"{int(offline_duration_seconds / 60)} minutes"
else:
hours = offline_duration_seconds / 3600
duration_str = f"{hours:.1f} hours"
message = (
f"📶 <b>ASSET BACK ONLINE</b>\n\n"
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
f"⏰ <b>Time:</b> {timestamp}\n"
)
if duration_str:
message += f"⏱️ <b>Was offline for:</b> {duration_str}\n"
message += f"\n✅ Asset is now reporting normally."
result = self._send_message(chat_id, message)
if result:
logger.info(f"Online alert sent for {asset_name}")
return result
def test_connection(self) -> bool:
"""Test Telegram bot connection"""
if not self.enabled:
return False
try:
url = f"{self.api_url}/getMe"
response = requests.get(url, timeout=10)
if response.status_code == 200:
bot_info = response.json()
logger.info(f"Telegram bot connected: @{bot_info['result']['username']}")
return True
else:
logger.error(f"Telegram connection failed: {response.status_code}")
return False
except Exception as e:
logger.error(f"Telegram connection error: {e}")
return False
# Singleton instance
_telegram_service: Optional[TelegramService] = None
def get_telegram_service() -> TelegramService:
"""Get the singleton Telegram service instance"""
global _telegram_service
if _telegram_service is None:
_telegram_service = TelegramService()
return _telegram_service