Refactor golden image handling in backup upload process</message>
<message>Update the _set_golden_from_path function to improve the handling of existing golden image files. Replace the existing unlink logic with a more robust method that safely removes files or broken symlinks using the missing_ok parameter. This change enhances the reliability of the backup upload process by ensuring that stale references are properly cleared before setting a new golden image path.
This commit is contained in:
nearxos
@@ -0,0 +1,366 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Server-side Telegram Notification Service for GNSS Guard
|
||||
|
||||
Sends alerts to Telegram for GPS validation state changes:
|
||||
- Sources becoming missing or recovering
|
||||
- Sources becoming stale or recovering
|
||||
- Distance threshold breaches (possible jamming/spoofing)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional, Set
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from config import get_config
|
||||
from models import Asset, AssetNotificationState
|
||||
|
||||
logger = logging.getLogger("gnss_guard.server.telegram")
|
||||
|
||||
|
||||
class TelegramService:
|
||||
"""Server-side Telegram notification service"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Telegram service with config"""
|
||||
config = get_config()
|
||||
self.bot_token = config.telegram_bot_token
|
||||
self.default_chat_id = config.telegram_chat_id
|
||||
self.enabled = config.telegram_enabled
|
||||
|
||||
if self.enabled:
|
||||
self.api_url = f"https://api.telegram.org/bot{self.bot_token}"
|
||||
logger.info("Telegram service initialized")
|
||||
else:
|
||||
self.api_url = None
|
||||
logger.info("Telegram service disabled (no bot token or chat ID configured)")
|
||||
|
||||
@staticmethod
|
||||
def escape_html(text: str) -> str:
|
||||
"""Escape HTML special characters for Telegram HTML parsing"""
|
||||
text = str(text)
|
||||
text = text.replace('&', '&')
|
||||
text = text.replace('<', '<')
|
||||
text = text.replace('>', '>')
|
||||
return text
|
||||
|
||||
def _send_message(self, chat_id: str, message: str) -> bool:
|
||||
"""Send a message to Telegram"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
try:
|
||||
url = f"{self.api_url}/sendMessage"
|
||||
payload = {
|
||||
"chat_id": chat_id,
|
||||
"text": message,
|
||||
"parse_mode": "HTML",
|
||||
"disable_web_page_preview": True
|
||||
}
|
||||
|
||||
response = requests.post(url, json=payload, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Telegram API error: {response.status_code} - {response.text}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Telegram message: {e}")
|
||||
return False
|
||||
|
||||
def _get_chat_id_for_asset(self, asset: Asset) -> Optional[str]:
|
||||
"""Get the chat ID to use for an asset (asset-specific or default)"""
|
||||
if not asset.telegram_enabled:
|
||||
return None
|
||||
return asset.telegram_chat_id or self.default_chat_id
|
||||
|
||||
def process_validation(
|
||||
self,
|
||||
db: Session,
|
||||
asset: Asset,
|
||||
validation_data: Dict[str, Any]
|
||||
) -> bool:
|
||||
"""
|
||||
Process a validation submission and send notification if state changed.
|
||||
Also handles online/offline state transitions.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
asset: Asset that submitted the validation
|
||||
validation_data: Validation data from the submission
|
||||
|
||||
Returns:
|
||||
bool: True if notification was sent
|
||||
"""
|
||||
chat_id = self._get_chat_id_for_asset(asset)
|
||||
|
||||
# Get or create notification state for this asset
|
||||
state = db.query(AssetNotificationState).filter(
|
||||
AssetNotificationState.asset_id == asset.id
|
||||
).first()
|
||||
|
||||
if not state:
|
||||
state = AssetNotificationState(asset_id=asset.id)
|
||||
db.add(state)
|
||||
db.flush()
|
||||
|
||||
notification_sent = False
|
||||
now = datetime.utcnow()
|
||||
|
||||
# Check if asset was offline and is now back online
|
||||
was_offline = state.is_online == False and state.last_validation_at is not None
|
||||
|
||||
if was_offline and self.enabled and chat_id:
|
||||
# Calculate how long it was offline
|
||||
offline_duration = (now - state.last_validation_at).total_seconds() if state.last_validation_at else None
|
||||
|
||||
notification_sent = self.send_asset_online_alert(
|
||||
chat_id=chat_id,
|
||||
asset_name=asset.name,
|
||||
offline_duration_seconds=offline_duration
|
||||
)
|
||||
|
||||
# Update online status and last validation time
|
||||
state.is_online = True
|
||||
state.last_validation_at = now
|
||||
|
||||
# Skip further processing if Telegram is disabled
|
||||
if not self.enabled or not chat_id:
|
||||
db.commit()
|
||||
return notification_sent
|
||||
|
||||
# Parse current state from validation
|
||||
sources_missing = set(validation_data.get("sources_missing", []))
|
||||
sources_stale = set(validation_data.get("sources_stale", []))
|
||||
validation_details = validation_data.get("validation_details", {})
|
||||
threshold = validation_details.get("threshold_meters", 0)
|
||||
max_distance = validation_details.get("max_distance_meters", 0)
|
||||
threshold_breached = max_distance > threshold if max_distance and threshold else False
|
||||
|
||||
# Parse previous state
|
||||
prev_missing = set(json.loads(state.prev_sources_missing or "[]"))
|
||||
prev_stale = set(json.loads(state.prev_sources_stale or "[]"))
|
||||
prev_threshold_breached = state.prev_threshold_breached or False
|
||||
|
||||
# Detect changes
|
||||
missing_added = sources_missing - prev_missing
|
||||
missing_removed = prev_missing - sources_missing
|
||||
stale_added = sources_stale - prev_stale
|
||||
stale_removed = prev_stale - sources_stale
|
||||
threshold_changed = threshold_breached != prev_threshold_breached
|
||||
|
||||
has_state_change = (
|
||||
missing_added or missing_removed or
|
||||
stale_added or stale_removed or
|
||||
threshold_changed
|
||||
)
|
||||
|
||||
if has_state_change:
|
||||
logger.info(f"State change detected for {asset.name}")
|
||||
|
||||
# Build and send notification
|
||||
source_coordinates = validation_data.get("source_coordinates", {})
|
||||
|
||||
message = self._build_state_change_message(
|
||||
asset_name=asset.name,
|
||||
missing_added=missing_added,
|
||||
missing_removed=missing_removed,
|
||||
stale_added=stale_added,
|
||||
stale_removed=stale_removed,
|
||||
threshold_breached=threshold_breached,
|
||||
prev_threshold_breached=prev_threshold_breached,
|
||||
max_distance_meters=max_distance,
|
||||
threshold_meters=threshold,
|
||||
source_coordinates=source_coordinates
|
||||
)
|
||||
|
||||
if self._send_message(chat_id, message):
|
||||
state.last_notification_at = now
|
||||
logger.info(f"Notification sent for {asset.name}")
|
||||
notification_sent = True
|
||||
|
||||
# Update state
|
||||
state.prev_sources_missing = json.dumps(list(sources_missing))
|
||||
state.prev_sources_stale = json.dumps(list(sources_stale))
|
||||
state.prev_threshold_breached = threshold_breached
|
||||
|
||||
db.commit()
|
||||
|
||||
return notification_sent
|
||||
|
||||
def _build_state_change_message(
|
||||
self,
|
||||
asset_name: str,
|
||||
missing_added: Set[str],
|
||||
missing_removed: Set[str],
|
||||
stale_added: Set[str],
|
||||
stale_removed: Set[str],
|
||||
threshold_breached: bool,
|
||||
prev_threshold_breached: bool,
|
||||
max_distance_meters: float,
|
||||
threshold_meters: float,
|
||||
source_coordinates: Dict[str, Any]
|
||||
) -> str:
|
||||
"""Build the state change notification message"""
|
||||
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
# Determine if this is a degradation or recovery
|
||||
is_degradation = missing_added or stale_added or (threshold_breached and not prev_threshold_breached)
|
||||
is_recovery = missing_removed or stale_removed or (not threshold_breached and prev_threshold_breached)
|
||||
|
||||
if is_degradation and not is_recovery:
|
||||
emoji = "🚨"
|
||||
title = "GNSS STATE DEGRADED"
|
||||
elif is_recovery and not is_degradation:
|
||||
emoji = "✅"
|
||||
title = "GNSS STATE RECOVERED"
|
||||
else:
|
||||
emoji = "⚠️"
|
||||
title = "GNSS STATE CHANGED"
|
||||
|
||||
message = (
|
||||
f"{emoji} <b>{title}</b>\n\n"
|
||||
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
|
||||
f"⏰ <b>Time:</b> {timestamp}\n\n"
|
||||
)
|
||||
|
||||
# Missing sources changes
|
||||
if missing_added:
|
||||
message += f"❌ <b>Sources now MISSING:</b> {', '.join(sorted(missing_added))}\n"
|
||||
if missing_removed:
|
||||
message += f"✅ <b>Sources RECOVERED (was missing):</b> {', '.join(sorted(missing_removed))}\n"
|
||||
|
||||
# Stale sources changes
|
||||
if stale_added:
|
||||
message += f"⏱️ <b>Sources now STALE:</b> {', '.join(sorted(stale_added))}\n"
|
||||
if stale_removed:
|
||||
message += f"✅ <b>Sources RECOVERED (was stale):</b> {', '.join(sorted(stale_removed))}\n"
|
||||
|
||||
# Threshold breach changes
|
||||
if threshold_breached and not prev_threshold_breached:
|
||||
message += (
|
||||
f"\n🚨 <b>DISTANCE THRESHOLD BREACHED!</b>\n"
|
||||
f" Max distance: {max_distance_meters:.1f}m (threshold: {threshold_meters:.1f}m)\n"
|
||||
f" ⚠️ Possible GPS jamming or spoofing!\n"
|
||||
)
|
||||
elif not threshold_breached and prev_threshold_breached:
|
||||
message += (
|
||||
f"\n✅ <b>Distance threshold OK</b>\n"
|
||||
f" Max distance: {max_distance_meters:.1f}m (threshold: {threshold_meters:.1f}m)\n"
|
||||
)
|
||||
|
||||
# Current coordinates summary
|
||||
if source_coordinates:
|
||||
message += f"\n📍 <b>Current Coordinates:</b>\n"
|
||||
for source, coords in source_coordinates.items():
|
||||
lat = coords.get("latitude", "N/A")
|
||||
lon = coords.get("longitude", "N/A")
|
||||
message += f" • {self.escape_html(source)}: {lat}, {lon}\n"
|
||||
|
||||
return message
|
||||
|
||||
def send_asset_offline_alert(
|
||||
self,
|
||||
chat_id: str,
|
||||
asset_name: str,
|
||||
last_seen: datetime,
|
||||
offline_threshold_seconds: int = 120
|
||||
) -> bool:
|
||||
"""Send notification when an asset goes offline (no updates received)"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
last_seen_str = last_seen.strftime("%Y-%m-%d %H:%M:%S UTC") if last_seen else "Unknown"
|
||||
|
||||
message = (
|
||||
f"📴 <b>ASSET OFFLINE</b>\n\n"
|
||||
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
|
||||
f"⏰ <b>Detected at:</b> {timestamp}\n"
|
||||
f"🕐 <b>Last seen:</b> {last_seen_str}\n\n"
|
||||
f"⚠️ No updates received for over {offline_threshold_seconds} seconds.\n"
|
||||
f"Check client connectivity and service status."
|
||||
)
|
||||
|
||||
result = self._send_message(chat_id, message)
|
||||
if result:
|
||||
logger.info(f"Offline alert sent for {asset_name}")
|
||||
return result
|
||||
|
||||
def send_asset_online_alert(
|
||||
self,
|
||||
chat_id: str,
|
||||
asset_name: str,
|
||||
offline_duration_seconds: Optional[float] = None
|
||||
) -> bool:
|
||||
"""Send notification when an asset comes back online"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
duration_str = ""
|
||||
if offline_duration_seconds:
|
||||
if offline_duration_seconds < 60:
|
||||
duration_str = f"{int(offline_duration_seconds)} seconds"
|
||||
elif offline_duration_seconds < 3600:
|
||||
duration_str = f"{int(offline_duration_seconds / 60)} minutes"
|
||||
else:
|
||||
hours = offline_duration_seconds / 3600
|
||||
duration_str = f"{hours:.1f} hours"
|
||||
|
||||
message = (
|
||||
f"📶 <b>ASSET BACK ONLINE</b>\n\n"
|
||||
f"📍 <b>Asset:</b> {self.escape_html(asset_name)}\n"
|
||||
f"⏰ <b>Time:</b> {timestamp}\n"
|
||||
)
|
||||
|
||||
if duration_str:
|
||||
message += f"⏱️ <b>Was offline for:</b> {duration_str}\n"
|
||||
|
||||
message += f"\n✅ Asset is now reporting normally."
|
||||
|
||||
result = self._send_message(chat_id, message)
|
||||
if result:
|
||||
logger.info(f"Online alert sent for {asset_name}")
|
||||
return result
|
||||
|
||||
def test_connection(self) -> bool:
|
||||
"""Test Telegram bot connection"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
try:
|
||||
url = f"{self.api_url}/getMe"
|
||||
response = requests.get(url, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
bot_info = response.json()
|
||||
logger.info(f"Telegram bot connected: @{bot_info['result']['username']}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Telegram connection failed: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Telegram connection error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_telegram_service: Optional[TelegramService] = None
|
||||
|
||||
|
||||
def get_telegram_service() -> TelegramService:
|
||||
"""Get the singleton Telegram service instance"""
|
||||
global _telegram_service
|
||||
if _telegram_service is None:
|
||||
_telegram_service = TelegramService()
|
||||
return _telegram_service
|
||||
|
||||
Reference in New Issue
Block a user