fix(jlcpcb): download prebuilt catalog instead of broken JLCSearch offset loop (#199)
The download_jlcpcb_database tool paged the community JLCSearch API with an
offset parameter, but that endpoint is a search front-end that ignores offset
and returns the same first 100 parts on every page, so a full catalog download
was impossible.
Add commands/jlcpcb_downloader.py with a layered strategy that reuses prebuilt
catalogs the whole ecosystem already trusts:
- CDFER single-file SQLite (primary; no 7z/zip, reliable on Windows)
- yaqwsx/jlcparts split 7z (fallback; only if a 7z CLI is present)
- official JLCPCB API (optional; cursor pagination, if credentials set)
Conversion reads CDFER's v_components view (or sniffs the largest table for
yaqwsx), C-prefixes integer lcsc, derives library_type from basic/preferred,
maps mfr->mfr_part, and normalizes price JSON to the manager's [{qty,price}]
shape. Rewire _handle_download_jlcpcb_database to use it (closing/reopening the
manager connection so the on-disk db can be rewritten on Windows). Remove the
broken offset loop from jlcsearch.py (client kept for interactive lookups).
Reduce download_jlcpcb.py to a thin CLI wrapper and update the TS tool schema.
Verified end-to-end against live CDFER: 616k parts downloaded + converted in
~40s, FTS search and price-break parsing correct. New unit tests cover the
conversion and source fall-through; no network in tests.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6262,12 +6262,18 @@ print("ok")
|
||||
# JLCPCB API handlers
|
||||
|
||||
def _handle_download_jlcpcb_database(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Download JLCPCB parts database from JLCSearch API"""
|
||||
"""Download the JLCPCB parts catalog from a prebuilt source (issue #199).
|
||||
|
||||
Layered strategy (see commands.jlcpcb_downloader): CDFER single-file
|
||||
SQLite (primary, no 7z) -> yaqwsx split-7z (fallback) -> official JLCPCB
|
||||
API (optional, if credentials set). Replaces the broken JLCSearch
|
||||
offset-pagination download.
|
||||
"""
|
||||
from commands import jlcpcb_downloader
|
||||
|
||||
try:
|
||||
force = params.get("force", False)
|
||||
|
||||
# Check if database exists
|
||||
import os
|
||||
prefer_source = params.get("source") # optional: cdfer|yaqwsx|official
|
||||
|
||||
stats = self.jlcpcb_parts.get_database_stats()
|
||||
if stats["total_parts"] > 0 and not force:
|
||||
@@ -6277,36 +6283,38 @@ print("ok")
|
||||
"stats": stats,
|
||||
}
|
||||
|
||||
logger.info("Downloading JLCPCB parts database from JLCSearch...")
|
||||
# The prebuilt paths recreate jlcpcb_parts.db on disk, so the open
|
||||
# manager connection must be released first (Windows file locking),
|
||||
# then reopened on the freshly written database.
|
||||
self.jlcpcb_parts.close()
|
||||
|
||||
# Download parts from JLCSearch public API (no auth required)
|
||||
parts = self.jlcsearch_client.download_all_components(
|
||||
callback=lambda total, msg: logger.info(f"{msg}")
|
||||
result = jlcpcb_downloader.download_database(
|
||||
force=force,
|
||||
prefer_source=prefer_source,
|
||||
progress=lambda msg: logger.info(msg),
|
||||
)
|
||||
|
||||
# Import into database
|
||||
logger.info(f"Importing {len(parts)} parts into database...")
|
||||
self.jlcpcb_parts.import_jlcsearch_parts(
|
||||
parts, progress_callback=lambda curr, total, msg: logger.info(msg)
|
||||
)
|
||||
# Reopen the manager on the new database regardless of outcome.
|
||||
self.jlcpcb_parts = JLCPCBPartsManager()
|
||||
|
||||
# Get final stats
|
||||
if not result.get("success"):
|
||||
return result
|
||||
|
||||
# Refresh counts from the reopened manager (authoritative).
|
||||
stats = self.jlcpcb_parts.get_database_stats()
|
||||
|
||||
# Calculate database size
|
||||
db_size_mb = os.path.getsize(self.jlcpcb_parts.db_path) / (1024 * 1024)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"total_parts": stats["total_parts"],
|
||||
"basic_parts": stats["basic_parts"],
|
||||
"extended_parts": stats["extended_parts"],
|
||||
"db_size_mb": round(db_size_mb, 2),
|
||||
"db_path": stats["db_path"],
|
||||
}
|
||||
result["total_parts"] = stats["total_parts"]
|
||||
result["basic_parts"] = stats["basic_parts"]
|
||||
result["extended_parts"] = stats["extended_parts"]
|
||||
result["db_path"] = stats["db_path"]
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading JLCPCB database: {e}", exc_info=True)
|
||||
# Best-effort: ensure the manager is usable after a failure.
|
||||
try:
|
||||
self.jlcpcb_parts = JLCPCBPartsManager()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"Failed to download database: {str(e)}",
|
||||
|
||||
Reference in New Issue
Block a user