fix(jlcpcb): download prebuilt catalog instead of broken JLCSearch offset loop (#199)

The download_jlcpcb_database tool paged the community JLCSearch API with an
offset parameter, but that endpoint is a search front-end that ignores offset
and returns the same first 100 parts on every page, so a full catalog download
was impossible.

Add commands/jlcpcb_downloader.py with a layered strategy that reuses prebuilt
catalogs the whole ecosystem already trusts:
  - CDFER single-file SQLite (primary; no 7z/zip, reliable on Windows)
  - yaqwsx/jlcparts split 7z (fallback; only if a 7z CLI is present)
  - official JLCPCB API (optional; cursor pagination, if credentials set)

Conversion reads CDFER's v_components view (or sniffs the largest table for
yaqwsx), C-prefixes integer lcsc, derives library_type from basic/preferred,
maps mfr->mfr_part, and normalizes price JSON to the manager's [{qty,price}]
shape. Rewire _handle_download_jlcpcb_database to use it (closing/reopening the
manager connection so the on-disk db can be rewritten on Windows). Remove the
broken offset loop from jlcsearch.py (client kept for interactive lookups).
Reduce download_jlcpcb.py to a thin CLI wrapper and update the TS tool schema.

Verified end-to-end against live CDFER: 616k parts downloaded + converted in
~40s, FTS search and price-break parsing correct. New unit tests cover the
conversion and source fall-through; no network in tests.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
mixelpixx
2026-05-24 11:53:12 -04:00
parent d765bfec78
commit b21b1410eb
6 changed files with 773 additions and 376 deletions

View File

@@ -6262,12 +6262,18 @@ print("ok")
# JLCPCB API handlers
def _handle_download_jlcpcb_database(self, params: Dict[str, Any]) -> Dict[str, Any]:
"""Download JLCPCB parts database from JLCSearch API"""
"""Download the JLCPCB parts catalog from a prebuilt source (issue #199).
Layered strategy (see commands.jlcpcb_downloader): CDFER single-file
SQLite (primary, no 7z) -> yaqwsx split-7z (fallback) -> official JLCPCB
API (optional, if credentials set). Replaces the broken JLCSearch
offset-pagination download.
"""
from commands import jlcpcb_downloader
try:
force = params.get("force", False)
# Check if database exists
import os
prefer_source = params.get("source") # optional: cdfer|yaqwsx|official
stats = self.jlcpcb_parts.get_database_stats()
if stats["total_parts"] > 0 and not force:
@@ -6277,36 +6283,38 @@ print("ok")
"stats": stats,
}
logger.info("Downloading JLCPCB parts database from JLCSearch...")
# The prebuilt paths recreate jlcpcb_parts.db on disk, so the open
# manager connection must be released first (Windows file locking),
# then reopened on the freshly written database.
self.jlcpcb_parts.close()
# Download parts from JLCSearch public API (no auth required)
parts = self.jlcsearch_client.download_all_components(
callback=lambda total, msg: logger.info(f"{msg}")
result = jlcpcb_downloader.download_database(
force=force,
prefer_source=prefer_source,
progress=lambda msg: logger.info(msg),
)
# Import into database
logger.info(f"Importing {len(parts)} parts into database...")
self.jlcpcb_parts.import_jlcsearch_parts(
parts, progress_callback=lambda curr, total, msg: logger.info(msg)
)
# Reopen the manager on the new database regardless of outcome.
self.jlcpcb_parts = JLCPCBPartsManager()
# Get final stats
if not result.get("success"):
return result
# Refresh counts from the reopened manager (authoritative).
stats = self.jlcpcb_parts.get_database_stats()
# Calculate database size
db_size_mb = os.path.getsize(self.jlcpcb_parts.db_path) / (1024 * 1024)
return {
"success": True,
"total_parts": stats["total_parts"],
"basic_parts": stats["basic_parts"],
"extended_parts": stats["extended_parts"],
"db_size_mb": round(db_size_mb, 2),
"db_path": stats["db_path"],
}
result["total_parts"] = stats["total_parts"]
result["basic_parts"] = stats["basic_parts"]
result["extended_parts"] = stats["extended_parts"]
result["db_path"] = stats["db_path"]
return result
except Exception as e:
logger.error(f"Error downloading JLCPCB database: {e}", exc_info=True)
# Best-effort: ensure the manager is usable after a failure.
try:
self.jlcpcb_parts = JLCPCBPartsManager()
except Exception:
pass
return {
"success": False,
"message": f"Failed to download database: {str(e)}",