Refactor golden image handling in backup upload process</message>
<message>Update the _set_golden_from_path function to improve the handling of existing golden image files. Replace the existing unlink logic with a more robust method that safely removes files or broken symlinks using the missing_ok parameter. This change enhances the reliability of the backup upload process by ensuring that stale references are properly cleared before setting a new golden image path.
This commit is contained in:
nearxos
@@ -0,0 +1,4 @@
|
||||
"""
|
||||
Services for GNSS Guard client
|
||||
"""
|
||||
|
||||
258
backup-from-device/gnss-guard/tm-gnss-guard/services/buzzer.py
Normal file
258
backup-from-device/gnss-guard/tm-gnss-guard/services/buzzer.py
Normal file
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Buzzer Service for reTerminal DM4
|
||||
Controls the hardware buzzer using the Linux LED subsystem
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("gnss_guard.buzzer")
|
||||
|
||||
# Buzzer control path (Linux LED subsystem)
|
||||
BUZZER_PATH = '/sys/class/leds/usr-buzzer/brightness'
|
||||
|
||||
|
||||
class BuzzerService:
|
||||
"""
|
||||
Service to control the hardware buzzer on reTerminal DM4.
|
||||
|
||||
The buzzer is controlled via the Linux LED subsystem:
|
||||
- Write "1" to turn ON
|
||||
- Write "0" to turn OFF
|
||||
|
||||
Supports alarm patterns (on/off cycling) that run in a background thread.
|
||||
"""
|
||||
|
||||
def __init__(self, on_duration: float = 1.0, off_duration: float = 1.0):
|
||||
"""
|
||||
Initialize the buzzer service.
|
||||
|
||||
Args:
|
||||
on_duration: Duration in seconds for buzzer ON during alarm pattern
|
||||
off_duration: Duration in seconds for buzzer OFF during alarm pattern
|
||||
"""
|
||||
self.on_duration = on_duration
|
||||
self.off_duration = off_duration
|
||||
|
||||
# Alarm state
|
||||
self._alarm_active = False
|
||||
self._alarm_acknowledged = False
|
||||
self._alarm_thread: Optional[threading.Thread] = None
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
# Check if buzzer is available
|
||||
self._buzzer_available = os.path.exists(BUZZER_PATH)
|
||||
if not self._buzzer_available:
|
||||
logger.warning(f"Buzzer not available at {BUZZER_PATH} - running in simulation mode")
|
||||
else:
|
||||
logger.info(f"Buzzer service initialized (path: {BUZZER_PATH})")
|
||||
# Ensure buzzer is off on startup
|
||||
self.buzzer_off()
|
||||
|
||||
def _write_buzzer(self, value: str) -> bool:
|
||||
"""
|
||||
Write value to buzzer control file.
|
||||
|
||||
Args:
|
||||
value: "1" for ON, "0" for OFF
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
if not self._buzzer_available:
|
||||
logger.debug(f"Buzzer simulation: {'ON' if value == '1' else 'OFF'}")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Use sudo tee to write to the sysfs file (requires sudo permissions)
|
||||
result = subprocess.run(
|
||||
['sudo', 'tee', BUZZER_PATH],
|
||||
input=value,
|
||||
text=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=2.0
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.error(f"Failed to write to buzzer: {result.stderr}")
|
||||
return False
|
||||
return True
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Timeout writing to buzzer")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing to buzzer: {e}")
|
||||
return False
|
||||
|
||||
def buzzer_on(self) -> bool:
|
||||
"""Turn buzzer ON"""
|
||||
return self._write_buzzer('1')
|
||||
|
||||
def buzzer_off(self) -> bool:
|
||||
"""Turn buzzer OFF"""
|
||||
return self._write_buzzer('0')
|
||||
|
||||
def get_status(self) -> str:
|
||||
"""
|
||||
Get current buzzer status.
|
||||
|
||||
Returns:
|
||||
"ON", "OFF", or "UNKNOWN"
|
||||
"""
|
||||
if not self._buzzer_available:
|
||||
return "SIMULATED"
|
||||
|
||||
try:
|
||||
with open(BUZZER_PATH, 'r') as f:
|
||||
value = f.read().strip()
|
||||
return "ON" if value in ['1', '255'] else "OFF"
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading buzzer status: {e}")
|
||||
return "UNKNOWN"
|
||||
|
||||
def _alarm_loop(self):
|
||||
"""
|
||||
Background thread loop for alarm pattern (1 second on, 1 second off).
|
||||
Runs until alarm is acknowledged or stopped.
|
||||
"""
|
||||
logger.info("Alarm pattern started")
|
||||
|
||||
while not self._stop_event.is_set() and not self._alarm_acknowledged:
|
||||
# Buzzer ON
|
||||
self.buzzer_on()
|
||||
|
||||
# Wait for on_duration or until stopped
|
||||
if self._stop_event.wait(self.on_duration):
|
||||
break
|
||||
if self._alarm_acknowledged:
|
||||
break
|
||||
|
||||
# Buzzer OFF
|
||||
self.buzzer_off()
|
||||
|
||||
# Wait for off_duration or until stopped
|
||||
if self._stop_event.wait(self.off_duration):
|
||||
break
|
||||
|
||||
# Ensure buzzer is off when alarm stops
|
||||
self.buzzer_off()
|
||||
self._alarm_active = False
|
||||
logger.info("Alarm pattern stopped")
|
||||
|
||||
def start_alarm(self) -> bool:
|
||||
"""
|
||||
Start the alarm pattern (1 second on, 1 second off).
|
||||
|
||||
Returns:
|
||||
True if alarm started, False if already running
|
||||
"""
|
||||
if self._alarm_active and self._alarm_thread and self._alarm_thread.is_alive():
|
||||
logger.debug("Alarm already active")
|
||||
return False
|
||||
|
||||
# Reset state
|
||||
self._alarm_acknowledged = False
|
||||
self._alarm_active = True
|
||||
self._stop_event.clear()
|
||||
|
||||
# Start alarm thread
|
||||
self._alarm_thread = threading.Thread(target=self._alarm_loop, daemon=True)
|
||||
self._alarm_thread.start()
|
||||
|
||||
logger.info("Alarm started")
|
||||
return True
|
||||
|
||||
def stop_alarm(self) -> bool:
|
||||
"""
|
||||
Stop the alarm pattern.
|
||||
|
||||
Returns:
|
||||
True if alarm was stopped, False if not running
|
||||
"""
|
||||
if not self._alarm_active:
|
||||
return False
|
||||
|
||||
self._stop_event.set()
|
||||
|
||||
# Wait for thread to finish
|
||||
if self._alarm_thread and self._alarm_thread.is_alive():
|
||||
self._alarm_thread.join(timeout=3.0)
|
||||
|
||||
# Ensure buzzer is off
|
||||
self.buzzer_off()
|
||||
self._alarm_active = False
|
||||
|
||||
logger.info("Alarm stopped")
|
||||
return True
|
||||
|
||||
def acknowledge_alarm(self) -> bool:
|
||||
"""
|
||||
Acknowledge the alarm, stopping the buzzer.
|
||||
|
||||
Returns:
|
||||
True if alarm was acknowledged, False if no alarm active
|
||||
"""
|
||||
if not self._alarm_active:
|
||||
logger.debug("No active alarm to acknowledge")
|
||||
return False
|
||||
|
||||
self._alarm_acknowledged = True
|
||||
self._stop_event.set()
|
||||
|
||||
# Wait for thread to finish
|
||||
if self._alarm_thread and self._alarm_thread.is_alive():
|
||||
self._alarm_thread.join(timeout=3.0)
|
||||
|
||||
# Ensure buzzer is off
|
||||
self.buzzer_off()
|
||||
self._alarm_active = False
|
||||
|
||||
logger.info("Alarm acknowledged")
|
||||
return True
|
||||
|
||||
def is_alarm_active(self) -> bool:
|
||||
"""Check if alarm is currently active"""
|
||||
return self._alarm_active
|
||||
|
||||
def is_alarm_acknowledged(self) -> bool:
|
||||
"""Check if current alarm has been acknowledged"""
|
||||
return self._alarm_acknowledged
|
||||
|
||||
def reset_acknowledged(self):
|
||||
"""
|
||||
Reset the acknowledged state.
|
||||
Called when status returns to healthy, allowing new alarms to trigger.
|
||||
"""
|
||||
self._alarm_acknowledged = False
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the buzzer service, ensuring buzzer is off"""
|
||||
self.stop_alarm()
|
||||
self.buzzer_off()
|
||||
logger.info("Buzzer service shutdown")
|
||||
|
||||
|
||||
# Global buzzer service instance (singleton pattern)
|
||||
_buzzer_instance: Optional[BuzzerService] = None
|
||||
|
||||
|
||||
def get_buzzer_service(on_duration: float = 1.0, off_duration: float = 1.0) -> BuzzerService:
|
||||
"""
|
||||
Get or create the global buzzer service instance.
|
||||
|
||||
Args:
|
||||
on_duration: Duration in seconds for buzzer ON during alarm
|
||||
off_duration: Duration in seconds for buzzer OFF during alarm
|
||||
|
||||
Returns:
|
||||
BuzzerService instance
|
||||
"""
|
||||
global _buzzer_instance
|
||||
if _buzzer_instance is None:
|
||||
_buzzer_instance = BuzzerService(on_duration, off_duration)
|
||||
return _buzzer_instance
|
||||
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Server Sync Service for GNSS Guard Client
|
||||
|
||||
Syncs validation data to the central GNSS Guard Server.
|
||||
Features:
|
||||
- Immediate sync on each validation
|
||||
- Offline queue for failed syncs
|
||||
- Batch catchup for queued records
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger("gnss_guard.server_sync")
|
||||
|
||||
|
||||
class ServerSync:
|
||||
"""
|
||||
Syncs validation data to the central GNSS Guard Server.
|
||||
|
||||
Features:
|
||||
- Sends validation results to server after each iteration
|
||||
- Queues failed requests for retry
|
||||
- Batch sends queued records on successful connection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database_path: Path,
|
||||
server_url: str,
|
||||
server_token: str,
|
||||
asset_name: str,
|
||||
batch_size: int = 100,
|
||||
max_queue_size: int = 1000
|
||||
):
|
||||
"""
|
||||
Initialize server sync service.
|
||||
|
||||
Args:
|
||||
database_path: Path to SQLite database (for sync queue)
|
||||
server_url: Base URL of GNSS Guard Server
|
||||
server_token: Authentication token for this asset
|
||||
asset_name: Name of this asset
|
||||
batch_size: Max records to send in batch catchup
|
||||
max_queue_size: Max records to keep in queue
|
||||
"""
|
||||
self.database_path = database_path
|
||||
self.server_url = server_url.rstrip('/')
|
||||
self.server_token = server_token
|
||||
self.asset_name = asset_name
|
||||
self.batch_size = batch_size
|
||||
self.max_queue_size = max_queue_size
|
||||
|
||||
# Request timeout (seconds)
|
||||
self.timeout = 10
|
||||
|
||||
# Initialize sync queue table
|
||||
self._init_sync_queue_table()
|
||||
|
||||
logger.info(f"Server sync initialized for asset '{asset_name}' -> {server_url}")
|
||||
|
||||
def _init_sync_queue_table(self):
|
||||
"""Create sync_queue table if it doesn't exist"""
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS sync_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
validation_timestamp_unix REAL NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
attempts INTEGER DEFAULT 0,
|
||||
last_attempt_at TEXT,
|
||||
UNIQUE(validation_timestamp_unix)
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_sync_queue_timestamp
|
||||
ON sync_queue(validation_timestamp_unix)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.debug("Sync queue table initialized")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize sync queue table: {e}")
|
||||
|
||||
def _get_headers(self) -> Dict[str, str]:
|
||||
"""Get request headers with authentication"""
|
||||
return {
|
||||
"Authorization": f"Bearer {self.server_token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
def sync_validation(self, validation_result: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Sync a validation result to the server.
|
||||
|
||||
If sync fails, the record is queued for later retry.
|
||||
If sync succeeds, attempt to send any queued records.
|
||||
|
||||
Args:
|
||||
validation_result: Validation result from CoordinateValidator
|
||||
|
||||
Returns:
|
||||
bool: True if sync succeeded, False if queued
|
||||
"""
|
||||
# Prepare payload
|
||||
payload = {
|
||||
"validation_timestamp": validation_result.get("validation_timestamp"),
|
||||
"validation_timestamp_unix": validation_result.get("validation_timestamp_unix"),
|
||||
"is_valid": validation_result.get("is_valid", False),
|
||||
"sources_missing": validation_result.get("sources_missing", []),
|
||||
"sources_stale": validation_result.get("sources_stale", []),
|
||||
"coordinate_differences": validation_result.get("coordinate_differences", {}),
|
||||
"source_coordinates": validation_result.get("source_coordinates", {}),
|
||||
"validation_details": validation_result.get("validation_details", {}),
|
||||
}
|
||||
|
||||
# Try to send
|
||||
success = self._send_validation(payload)
|
||||
|
||||
if success:
|
||||
# On success, try to send queued records
|
||||
self._process_queue()
|
||||
else:
|
||||
# On failure, queue the record
|
||||
self._queue_record(payload)
|
||||
|
||||
return success
|
||||
|
||||
def _send_validation(self, payload: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Send a single validation record to the server.
|
||||
|
||||
Args:
|
||||
payload: Validation data to send
|
||||
|
||||
Returns:
|
||||
bool: True if successful
|
||||
"""
|
||||
try:
|
||||
url = f"{self.server_url}/api/v1/validation"
|
||||
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=self._get_headers(),
|
||||
timeout=self.timeout
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
logger.debug(f"Validation synced to server")
|
||||
return True
|
||||
elif response.status_code == 401:
|
||||
logger.error(f"Server auth failed - check SERVER_TOKEN")
|
||||
return False
|
||||
else:
|
||||
logger.warning(f"Server returned {response.status_code}: {response.text[:200]}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
logger.warning(f"Server request timed out")
|
||||
return False
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.warning(f"Cannot connect to server at {self.server_url}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Server sync error: {e}")
|
||||
return False
|
||||
|
||||
def _send_batch(self, records: List[Dict[str, Any]]) -> bool:
|
||||
"""
|
||||
Send a batch of validation records to the server.
|
||||
|
||||
Args:
|
||||
records: List of validation payloads
|
||||
|
||||
Returns:
|
||||
bool: True if successful
|
||||
"""
|
||||
try:
|
||||
url = f"{self.server_url}/api/v1/validation/batch"
|
||||
|
||||
response = requests.post(
|
||||
url,
|
||||
json={"records": records},
|
||||
headers=self._get_headers(),
|
||||
timeout=self.timeout * 3 # Longer timeout for batch
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
result = response.json()
|
||||
logger.info(f"Batch sync: {result.get('saved', 0)} saved, {result.get('skipped', 0)} skipped")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Batch sync failed: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Batch sync error: {e}")
|
||||
return False
|
||||
|
||||
def _queue_record(self, payload: Dict[str, Any]):
|
||||
"""
|
||||
Add a validation record to the sync queue.
|
||||
|
||||
Args:
|
||||
payload: Validation data to queue
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check queue size and remove oldest if full
|
||||
cursor.execute("SELECT COUNT(*) FROM sync_queue")
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
if count >= self.max_queue_size:
|
||||
# Remove oldest records to make room
|
||||
remove_count = count - self.max_queue_size + 10
|
||||
cursor.execute("""
|
||||
DELETE FROM sync_queue
|
||||
WHERE id IN (
|
||||
SELECT id FROM sync_queue
|
||||
ORDER BY validation_timestamp_unix ASC
|
||||
LIMIT ?
|
||||
)
|
||||
""", (remove_count,))
|
||||
logger.warning(f"Sync queue full, removed {remove_count} oldest records")
|
||||
|
||||
# Insert new record
|
||||
cursor.execute("""
|
||||
INSERT OR IGNORE INTO sync_queue
|
||||
(validation_timestamp_unix, payload, created_at)
|
||||
VALUES (?, ?, ?)
|
||||
""", (
|
||||
payload["validation_timestamp_unix"],
|
||||
json.dumps(payload),
|
||||
datetime.utcnow().isoformat()
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.debug(f"Queued validation record for later sync")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to queue record: {e}")
|
||||
|
||||
def _process_queue(self):
|
||||
"""Process queued records after successful connection"""
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get queued records (oldest first)
|
||||
cursor.execute("""
|
||||
SELECT id, payload FROM sync_queue
|
||||
ORDER BY validation_timestamp_unix ASC
|
||||
LIMIT ?
|
||||
""", (self.batch_size,))
|
||||
|
||||
rows = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
if not rows:
|
||||
return
|
||||
|
||||
logger.info(f"Processing {len(rows)} queued records")
|
||||
|
||||
# Parse payloads
|
||||
records = []
|
||||
record_ids = []
|
||||
for row_id, payload_json in rows:
|
||||
try:
|
||||
records.append(json.loads(payload_json))
|
||||
record_ids.append(row_id)
|
||||
except json.JSONDecodeError:
|
||||
record_ids.append(row_id) # Still mark for deletion if corrupt
|
||||
|
||||
if not records:
|
||||
return
|
||||
|
||||
# Send batch
|
||||
if self._send_batch(records):
|
||||
# Remove sent records from queue
|
||||
self._remove_from_queue(record_ids)
|
||||
else:
|
||||
# Update attempt count
|
||||
self._update_attempt_count(record_ids)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing queue: {e}")
|
||||
|
||||
def _remove_from_queue(self, record_ids: List[int]):
|
||||
"""Remove successfully sent records from queue"""
|
||||
if not record_ids:
|
||||
return
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
placeholders = ','.join('?' * len(record_ids))
|
||||
cursor.execute(f"DELETE FROM sync_queue WHERE id IN ({placeholders})", record_ids)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.debug(f"Removed {len(record_ids)} records from sync queue")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to remove records from queue: {e}")
|
||||
|
||||
def _update_attempt_count(self, record_ids: List[int]):
|
||||
"""Update attempt count for failed records"""
|
||||
if not record_ids:
|
||||
return
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
now = datetime.utcnow().isoformat()
|
||||
placeholders = ','.join('?' * len(record_ids))
|
||||
cursor.execute(f"""
|
||||
UPDATE sync_queue
|
||||
SET attempts = attempts + 1, last_attempt_at = ?
|
||||
WHERE id IN ({placeholders})
|
||||
""", [now] + record_ids)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update attempt count: {e}")
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current sync queue status.
|
||||
|
||||
Returns:
|
||||
Dictionary with queue stats
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM sync_queue")
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute("SELECT MIN(validation_timestamp_unix), MAX(validation_timestamp_unix) FROM sync_queue")
|
||||
oldest, newest = cursor.fetchone()
|
||||
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
"queued_count": count,
|
||||
"oldest_timestamp": oldest,
|
||||
"newest_timestamp": newest,
|
||||
"queue_full": count >= self.max_queue_size
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get queue status: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def force_sync(self) -> bool:
|
||||
"""
|
||||
Force a sync of all queued records.
|
||||
|
||||
Returns:
|
||||
bool: True if all records synced successfully
|
||||
"""
|
||||
logger.info("Starting forced sync of queued records")
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(self.database_path), timeout=5.0)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM sync_queue")
|
||||
total = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
if total == 0:
|
||||
logger.info("No records to sync")
|
||||
return True
|
||||
|
||||
synced = 0
|
||||
while True:
|
||||
# Check if queue is empty
|
||||
status = self.get_queue_status()
|
||||
if status.get("queued_count", 0) == 0:
|
||||
break
|
||||
|
||||
# Process a batch
|
||||
before_count = status["queued_count"]
|
||||
self._process_queue()
|
||||
|
||||
# Check if we made progress
|
||||
after_status = self.get_queue_status()
|
||||
if after_status.get("queued_count", 0) >= before_count:
|
||||
# No progress, connection likely failed
|
||||
logger.warning("Sync stalled, connection issue")
|
||||
break
|
||||
|
||||
synced += before_count - after_status.get("queued_count", 0)
|
||||
|
||||
logger.info(f"Force sync completed: {synced}/{total} records synced")
|
||||
return synced == total
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Force sync error: {e}")
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user