feat(jlcpcb): warn when downloaded catalog is stale (#199)
Per review feedback: CDFER's upstream scraper pipeline has stalled for weeks (broken cart API + a bug in their scraper), so a "fresh download" can still be old data. Compute the catalog age from the source Last-Modified header, expose catalog_age_days, and emit a stale=True + warning when older than 14 days that points users to source='yaqwsx' (fresh, needs 7z) or source='official'. Surface the warning in the MCP tool output. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -412,6 +412,29 @@ def _download_official(target: Path, force: bool, progress: ProgressFn) -> Dict[
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Warn if the catalog is older than this (CDFER's pipeline has stalled for
|
||||||
|
# weeks at a time, so a "fresh download" can still be stale data).
|
||||||
|
STALE_AFTER_DAYS = 14
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_age_days(last_modified: Optional[str]) -> Optional[int]:
|
||||||
|
"""Parse an HTTP Last-Modified date and return its age in whole days."""
|
||||||
|
if not last_modified:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from email.utils import parsedate_to_datetime
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
dt = parsedate_to_datetime(last_modified)
|
||||||
|
if dt is None:
|
||||||
|
return None
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=timezone.utc)
|
||||||
|
return max(0, (datetime.now(timezone.utc) - dt).days)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _result(
|
def _result(
|
||||||
source: str,
|
source: str,
|
||||||
target: Path,
|
target: Path,
|
||||||
@@ -421,7 +444,7 @@ def _result(
|
|||||||
last_modified: Optional[str],
|
last_modified: Optional[str],
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
db_size_mb = round(target.stat().st_size / (1024 * 1024), 2) if target.exists() else 0
|
db_size_mb = round(target.stat().st_size / (1024 * 1024), 2) if target.exists() else 0
|
||||||
return {
|
result: Dict[str, Any] = {
|
||||||
"success": True,
|
"success": True,
|
||||||
"source": source,
|
"source": source,
|
||||||
"total_parts": total,
|
"total_parts": total,
|
||||||
@@ -431,6 +454,19 @@ def _result(
|
|||||||
"db_path": str(target),
|
"db_path": str(target),
|
||||||
"catalog_last_modified": last_modified,
|
"catalog_last_modified": last_modified,
|
||||||
}
|
}
|
||||||
|
age_days = _catalog_age_days(last_modified)
|
||||||
|
if age_days is not None:
|
||||||
|
result["catalog_age_days"] = age_days
|
||||||
|
if age_days >= STALE_AFTER_DAYS:
|
||||||
|
warning = (
|
||||||
|
f"Catalog from '{source}' is ~{age_days} days old (dated {last_modified}). "
|
||||||
|
"Its upstream pipeline may have stalled. For fresher data, re-run with "
|
||||||
|
"source='yaqwsx' (needs a 7z CLI) or source='official' (needs JLCPCB API creds)."
|
||||||
|
)
|
||||||
|
result["stale"] = True
|
||||||
|
result["warning"] = warning
|
||||||
|
logger.warning(warning)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -41,13 +41,18 @@ provider. Re-run with force=true to refresh.`,
|
|||||||
`✓ Successfully downloaded JLCPCB parts database\n\n` +
|
`✓ Successfully downloaded JLCPCB parts database\n\n` +
|
||||||
`Source: ${result.source}\n` +
|
`Source: ${result.source}\n` +
|
||||||
(result.catalog_last_modified
|
(result.catalog_last_modified
|
||||||
? `Catalog dated: ${result.catalog_last_modified}\n`
|
? `Catalog dated: ${result.catalog_last_modified}` +
|
||||||
|
(typeof result.catalog_age_days === "number"
|
||||||
|
? ` (~${result.catalog_age_days} days old)`
|
||||||
|
: "") +
|
||||||
|
`\n`
|
||||||
: "") +
|
: "") +
|
||||||
`Total parts: ${result.total_parts}\n` +
|
`Total parts: ${result.total_parts}\n` +
|
||||||
`Basic parts: ${result.basic_parts}\n` +
|
`Basic parts: ${result.basic_parts}\n` +
|
||||||
`Extended parts: ${result.extended_parts}\n` +
|
`Extended parts: ${result.extended_parts}\n` +
|
||||||
`Database size: ${result.db_size_mb} MB\n` +
|
`Database size: ${result.db_size_mb} MB\n` +
|
||||||
`Database path: ${result.db_path}`,
|
`Database path: ${result.db_path}` +
|
||||||
|
(result.warning ? `\n\n⚠ ${result.warning}` : ""),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -170,3 +170,40 @@ def test_download_database_prefer_cdfer_converts(tmp_path, monkeypatch):
|
|||||||
assert result["basic_parts"] == 1
|
assert result["basic_parts"] == 1
|
||||||
assert result["catalog_last_modified"].startswith("Thu, 02 Apr 2026")
|
assert result["catalog_last_modified"].startswith("Thu, 02 Apr 2026")
|
||||||
assert (tmp_path / "jlcpcb_parts.db").exists()
|
assert (tmp_path / "jlcpcb_parts.db").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def _http_date(days_ago: int) -> str:
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from email.utils import format_datetime
|
||||||
|
|
||||||
|
return format_datetime(datetime.now(timezone.utc) - timedelta(days=days_ago))
|
||||||
|
|
||||||
|
|
||||||
|
def test_result_flags_stale_catalog_and_leaves_fresh_alone(tmp_path, monkeypatch):
|
||||||
|
"""A catalog older than STALE_AFTER_DAYS must be flagged; a fresh one must not."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
jlcpcb_downloader.PlatformHelper, "get_data_dir", staticmethod(lambda: tmp_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fake_cdfer_dated(http_date):
|
||||||
|
def _impl(cache_dir, progress=None):
|
||||||
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
src = cache_dir / "cdfer.sqlite3"
|
||||||
|
_make_cdfer_like_source(src)
|
||||||
|
return src, http_date
|
||||||
|
|
||||||
|
return _impl
|
||||||
|
|
||||||
|
# Stale: 60 days old
|
||||||
|
monkeypatch.setattr(jlcpcb_downloader, "download_cdfer", _fake_cdfer_dated(_http_date(60)))
|
||||||
|
stale = jlcpcb_downloader.download_database(prefer_source="cdfer")
|
||||||
|
assert stale.get("stale") is True
|
||||||
|
assert "warning" in stale
|
||||||
|
assert stale["catalog_age_days"] >= jlcpcb_downloader.STALE_AFTER_DAYS
|
||||||
|
|
||||||
|
# Fresh: 1 day old
|
||||||
|
monkeypatch.setattr(jlcpcb_downloader, "download_cdfer", _fake_cdfer_dated(_http_date(1)))
|
||||||
|
fresh = jlcpcb_downloader.download_database(prefer_source="cdfer")
|
||||||
|
assert fresh.get("stale") is not True
|
||||||
|
assert "warning" not in fresh
|
||||||
|
assert fresh["catalog_age_days"] <= 1
|
||||||
|
|||||||
Reference in New Issue
Block a user