fix(jlcpcb): download prebuilt catalog instead of broken JLCSearch offset loop (#199)
The download_jlcpcb_database tool paged the community JLCSearch API with an
offset parameter, but that endpoint is a search front-end that ignores offset
and returns the same first 100 parts on every page, so a full catalog download
was impossible.
Add commands/jlcpcb_downloader.py with a layered strategy that reuses prebuilt
catalogs the whole ecosystem already trusts:
- CDFER single-file SQLite (primary; no 7z/zip, reliable on Windows)
- yaqwsx/jlcparts split 7z (fallback; only if a 7z CLI is present)
- official JLCPCB API (optional; cursor pagination, if credentials set)
Conversion reads CDFER's v_components view (or sniffs the largest table for
yaqwsx), C-prefixes integer lcsc, derives library_type from basic/preferred,
maps mfr->mfr_part, and normalizes price JSON to the manager's [{qty,price}]
shape. Rewire _handle_download_jlcpcb_database to use it (closing/reopening the
manager connection so the on-disk db can be rewritten on Windows). Remove the
broken offset loop from jlcsearch.py (client kept for interactive lookups).
Reduce download_jlcpcb.py to a thin CLI wrapper and update the TS tool schema.
Verified end-to-end against live CDFER: 616k parts downloaded + converted in
~40s, FTS search and price-break parsing correct. New unit tests cover the
conversion and source fall-through; no network in tests.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
mixelpixx
501
python/commands/jlcpcb_downloader.py
Normal file
501
python/commands/jlcpcb_downloader.py
Normal file
@@ -0,0 +1,501 @@
|
||||
"""
|
||||
JLCPCB parts catalog downloader (issue #199).
|
||||
|
||||
The old approach paged the community JLCSearch API with an ``offset`` parameter,
|
||||
but that endpoint is a search front-end that ignores ``offset`` and returns the
|
||||
same first 100 parts on every page, so a full catalog download was impossible.
|
||||
|
||||
This module replaces that with a layered download of a *prebuilt* catalog and
|
||||
converts it into the schema expected by ``JLCPCBPartsManager``
|
||||
(``get_data_dir()/jlcpcb_parts.db``):
|
||||
|
||||
1. CDFER (primary) -- https://github.com/CDFER/jlcpcb-parts-database
|
||||
A single, raw, uncompressed SQLite file on GitHub Pages. No 7z/zip
|
||||
needed, so this is the reliable cross-platform (incl. Windows) path.
|
||||
2. yaqwsx/jlcparts (fallback) -- https://github.com/yaqwsx/jlcparts
|
||||
The canonical upstream, published as a split 7z archive. Used only if
|
||||
CDFER fails AND a 7z CLI is available.
|
||||
3. Official JLCPCB API (optional) -- requires JLCPCB_APP_ID / JLCPCB_API_KEY /
|
||||
JLCPCB_API_SECRET. Uses cursor pagination (works correctly).
|
||||
|
||||
Data is MIT-licensed via CDFER and yaqwsx/jlcparts; the underlying catalog facts
|
||||
originate from JLCPCB / LCSC and are subject to their terms.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from utils.platform_helper import PlatformHelper
|
||||
|
||||
logger = logging.getLogger("kicad_interface")
|
||||
|
||||
CDFER_SQLITE_URL = "https://cdfer.github.io/jlcpcb-parts-database/jlcpcb-components.sqlite3"
|
||||
YAQWSX_BASE_URL = "https://yaqwsx.github.io/jlcparts/data"
|
||||
YAQWSX_MAX_VOLUMES = 30
|
||||
|
||||
ProgressFn = Optional[Callable[[str], None]]
|
||||
|
||||
|
||||
def _progress(progress: ProgressFn, message: str) -> None:
|
||||
if progress:
|
||||
try:
|
||||
progress(message)
|
||||
except Exception: # pragma: no cover - progress must never break a download
|
||||
pass
|
||||
logger.info(message)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conversion (shared by the CDFER and yaqwsx paths)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normalize_price_json(raw: Any) -> str:
|
||||
"""Normalize a source price value into the manager's ``[{"qty","price"}]`` shape.
|
||||
|
||||
Handles both CDFER/jlcparts JSON arrays (``[{"qFrom","qTo","price"}, ...]``)
|
||||
and a bare scalar price. Returns a JSON string (``"[]"`` when no price).
|
||||
"""
|
||||
if raw is None or raw == "":
|
||||
return json.dumps([])
|
||||
|
||||
# JSON array string (CDFER / jlcparts store price as TEXT JSON)
|
||||
if isinstance(raw, str) and raw.strip().startswith("["):
|
||||
try:
|
||||
arr = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return json.dumps([])
|
||||
out: List[Dict[str, Any]] = []
|
||||
for item in arr if isinstance(arr, list) else []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
price = item.get("price")
|
||||
if price is None:
|
||||
continue
|
||||
qty = item.get("qFrom", item.get("qty", 1)) or 1
|
||||
out.append({"qty": qty, "price": price})
|
||||
return json.dumps(out)
|
||||
|
||||
# Scalar numeric price
|
||||
try:
|
||||
price_val = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return json.dumps([])
|
||||
return json.dumps([{"qty": 1, "price": price_val}]) if price_val else json.dumps([])
|
||||
|
||||
|
||||
def _library_type(is_basic: Any, is_preferred: Any) -> str:
|
||||
if is_basic:
|
||||
return "Basic"
|
||||
if is_preferred:
|
||||
return "Preferred"
|
||||
return "Extended"
|
||||
|
||||
|
||||
def _pick_source_relation(src: sqlite3.Connection) -> str:
|
||||
"""Choose which table/view to read from the source database.
|
||||
|
||||
Prefers CDFER's denormalized ``v_components`` view (which exposes category,
|
||||
subcategory and manufacturer names directly). Falls back to the largest
|
||||
table for yaqwsx-style schemas.
|
||||
"""
|
||||
names = [
|
||||
r[0]
|
||||
for r in src.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type IN ('table','view')"
|
||||
).fetchall()
|
||||
]
|
||||
if "v_components" in names:
|
||||
return "v_components"
|
||||
if "components" in names:
|
||||
return "components"
|
||||
# Fallback: largest table
|
||||
best, best_count = None, -1
|
||||
for name in names:
|
||||
try:
|
||||
count = src.execute(f"SELECT COUNT(*) FROM [{name}]").fetchone()[0]
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
if count > best_count:
|
||||
best, best_count = name, count
|
||||
if not best:
|
||||
raise RuntimeError("No usable table/view found in source database")
|
||||
return best
|
||||
|
||||
|
||||
def convert_source_sqlite(
|
||||
source_path: Path, target_path: Path, progress: ProgressFn = None
|
||||
) -> Dict[str, int]:
|
||||
"""Convert a prebuilt source SQLite (CDFER or yaqwsx) into the MCP schema.
|
||||
|
||||
Writes a fresh ``target_path`` (deleting any existing file) with the
|
||||
``components`` table + FTS index that ``JLCPCBPartsManager`` expects.
|
||||
Returns ``{"total","basic","extended"}`` counts.
|
||||
"""
|
||||
source_path = Path(source_path)
|
||||
target_path = Path(target_path)
|
||||
if not source_path.exists():
|
||||
raise FileNotFoundError(f"source database not found: {source_path}")
|
||||
|
||||
src = sqlite3.connect(str(source_path))
|
||||
src.row_factory = sqlite3.Row
|
||||
try:
|
||||
relation = _pick_source_relation(src)
|
||||
_progress(progress, f"Converting from source relation '{relation}'...")
|
||||
|
||||
if target_path.exists():
|
||||
target_path.unlink()
|
||||
|
||||
dst = sqlite3.connect(str(target_path))
|
||||
try:
|
||||
dst.execute("""
|
||||
CREATE TABLE components (
|
||||
lcsc TEXT PRIMARY KEY,
|
||||
category TEXT,
|
||||
subcategory TEXT,
|
||||
mfr_part TEXT,
|
||||
package TEXT,
|
||||
solder_joints INTEGER,
|
||||
manufacturer TEXT,
|
||||
library_type TEXT,
|
||||
description TEXT,
|
||||
datasheet TEXT,
|
||||
stock INTEGER,
|
||||
price_json TEXT,
|
||||
last_updated INTEGER
|
||||
)
|
||||
""")
|
||||
dst.execute("CREATE INDEX idx_category ON components(category, subcategory)")
|
||||
dst.execute("CREATE INDEX idx_package ON components(package)")
|
||||
dst.execute("CREATE INDEX idx_manufacturer ON components(manufacturer)")
|
||||
dst.execute("CREATE INDEX idx_library_type ON components(library_type)")
|
||||
dst.execute("CREATE INDEX idx_mfr_part ON components(mfr_part)")
|
||||
|
||||
now = int(time.time())
|
||||
batch: List[Tuple] = []
|
||||
count = 0
|
||||
|
||||
insert_sql = """
|
||||
INSERT OR REPLACE INTO components
|
||||
(lcsc, category, subcategory, mfr_part, package, solder_joints,
|
||||
manufacturer, library_type, description, datasheet, stock,
|
||||
price_json, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
|
||||
for row in src.execute(f"SELECT * FROM [{relation}]"):
|
||||
rd = dict(row)
|
||||
|
||||
lcsc = rd.get("lcsc") or rd.get("LCSC_Part") or rd.get("lcsc_id")
|
||||
if lcsc is None:
|
||||
continue
|
||||
if isinstance(lcsc, int):
|
||||
lcsc = f"C{lcsc}"
|
||||
elif not str(lcsc).startswith("C"):
|
||||
lcsc = f"C{lcsc}"
|
||||
|
||||
mfr_part = rd.get("mfr") or rd.get("MFR_Part") or rd.get("mfr_part") or ""
|
||||
package = rd.get("package") or rd.get("Package") or ""
|
||||
manufacturer = rd.get("manufacturer") or rd.get("Manufacturer") or ""
|
||||
description = rd.get("description") or rd.get("Description") or ""
|
||||
stock = rd.get("stock") or rd.get("Stock") or 0
|
||||
category = rd.get("category") or rd.get("First Category") or ""
|
||||
subcategory = rd.get("subcategory") or rd.get("Second Category") or ""
|
||||
datasheet = rd.get("datasheet") or rd.get("url") or ""
|
||||
joints = rd.get("joints") or rd.get("solder_joints") or 0
|
||||
|
||||
lib_type = _library_type(
|
||||
rd.get("basic") or rd.get("is_basic") or rd.get("Basic"),
|
||||
rd.get("preferred") or rd.get("is_preferred") or rd.get("Preferred"),
|
||||
)
|
||||
price_json = normalize_price_json(rd.get("price") or rd.get("Price"))
|
||||
|
||||
batch.append(
|
||||
(
|
||||
str(lcsc),
|
||||
category,
|
||||
subcategory,
|
||||
mfr_part,
|
||||
package,
|
||||
int(joints) if joints else 0,
|
||||
manufacturer,
|
||||
lib_type,
|
||||
description,
|
||||
datasheet,
|
||||
int(stock) if stock else 0,
|
||||
price_json,
|
||||
now,
|
||||
)
|
||||
)
|
||||
|
||||
if len(batch) >= 10000:
|
||||
dst.executemany(insert_sql, batch)
|
||||
count += len(batch)
|
||||
batch = []
|
||||
if count % 100000 == 0:
|
||||
_progress(progress, f"Converted {count:,} parts...")
|
||||
|
||||
if batch:
|
||||
dst.executemany(insert_sql, batch)
|
||||
count += len(batch)
|
||||
|
||||
_progress(progress, "Building full-text search index...")
|
||||
dst.execute("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS components_fts USING fts5(
|
||||
lcsc, description, mfr_part, manufacturer,
|
||||
content=components
|
||||
)
|
||||
""")
|
||||
dst.execute("INSERT INTO components_fts(components_fts) VALUES('rebuild')")
|
||||
dst.commit()
|
||||
|
||||
total = dst.execute("SELECT COUNT(*) FROM components").fetchone()[0]
|
||||
basic = dst.execute(
|
||||
"SELECT COUNT(*) FROM components WHERE library_type='Basic'"
|
||||
).fetchone()[0]
|
||||
extended = dst.execute(
|
||||
"SELECT COUNT(*) FROM components WHERE library_type='Extended'"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
dst.close()
|
||||
finally:
|
||||
src.close()
|
||||
|
||||
return {"total": total, "basic": basic, "extended": extended}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source downloads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _head_last_modified(url: str) -> Optional[str]:
|
||||
try:
|
||||
import requests
|
||||
|
||||
resp = requests.head(url, allow_redirects=True, timeout=30)
|
||||
if resp.ok:
|
||||
return resp.headers.get("Last-Modified")
|
||||
except Exception as exc: # network/dns/etc.
|
||||
logger.debug(f"HEAD {url} failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path, Optional[str]]:
|
||||
"""Stream-download CDFER's single uncompressed SQLite. Returns (path, last_modified)."""
|
||||
import requests
|
||||
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = cache_dir / "cdfer.sqlite3"
|
||||
last_modified = _head_last_modified(CDFER_SQLITE_URL)
|
||||
_progress(
|
||||
progress,
|
||||
"Downloading CDFER prebuilt SQLite (~1.5 GB)"
|
||||
+ (f" [catalog dated {last_modified}]" if last_modified else "")
|
||||
+ "...",
|
||||
)
|
||||
|
||||
with requests.get(CDFER_SQLITE_URL, stream=True, timeout=60) as resp:
|
||||
resp.raise_for_status()
|
||||
last_modified = last_modified or resp.headers.get("Last-Modified")
|
||||
# NOTE: GitHub Pages' Content-Length can understate the real transfer
|
||||
# size (compressed transfer accounting), so we report MB downloaded
|
||||
# rather than a misleading percentage.
|
||||
written = 0
|
||||
next_mark = 50 * 1024 * 1024 # log every ~50 MB
|
||||
with open(dest, "wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
fh.write(chunk)
|
||||
written += len(chunk)
|
||||
if written >= next_mark:
|
||||
_progress(progress, f"Downloaded {written // (1024 * 1024)} MB...")
|
||||
next_mark += 50 * 1024 * 1024
|
||||
|
||||
if dest.stat().st_size < 1_000_000:
|
||||
raise RuntimeError("CDFER download too small to be valid")
|
||||
return dest, last_modified
|
||||
|
||||
|
||||
def _find_7z() -> Optional[str]:
|
||||
for cmd in ("7z", "7zz", "7za"):
|
||||
if shutil.which(cmd):
|
||||
return cmd
|
||||
return None
|
||||
|
||||
|
||||
def download_yaqwsx(cache_dir: Path, progress: ProgressFn = None) -> Path:
|
||||
"""Download + extract the yaqwsx split 7z archive. Requires a 7z CLI."""
|
||||
import subprocess
|
||||
|
||||
seven_zip = _find_7z()
|
||||
if not seven_zip:
|
||||
raise RuntimeError("yaqwsx fallback requires a 7z CLI (7z/7zz/7za) which was not found")
|
||||
if not shutil.which("curl"):
|
||||
raise RuntimeError("yaqwsx fallback requires curl which was not found")
|
||||
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
_progress(progress, "Downloading yaqwsx split archive (~421 MB)...")
|
||||
|
||||
def _curl(url: str, dst: Path) -> bool:
|
||||
return (
|
||||
subprocess.run(["curl", "-L", "-f", "-o", str(dst), "--progress-bar", url]).returncode
|
||||
== 0
|
||||
)
|
||||
|
||||
for i in range(1, YAQWSX_MAX_VOLUMES + 1):
|
||||
part = f"cache.z{i:02d}"
|
||||
dst = cache_dir / part
|
||||
if dst.exists() and dst.stat().st_size > 1000:
|
||||
continue
|
||||
if not _curl(f"{YAQWSX_BASE_URL}/{part}", dst):
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
break
|
||||
|
||||
zip_dst = cache_dir / "cache.zip"
|
||||
if not (zip_dst.exists() and zip_dst.stat().st_size > 1000):
|
||||
if not _curl(f"{YAQWSX_BASE_URL}/cache.zip", zip_dst):
|
||||
raise RuntimeError("failed to download yaqwsx cache.zip")
|
||||
|
||||
_progress(progress, f"Extracting archive with {seven_zip}...")
|
||||
result = subprocess.run(
|
||||
[seven_zip, "x", "-y", "-o" + str(cache_dir), str(zip_dst)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"7z extraction failed: {result.stderr[:300]}")
|
||||
|
||||
sqlite_path = cache_dir / "cache.sqlite3"
|
||||
if not sqlite_path.exists():
|
||||
raise RuntimeError("yaqwsx archive did not yield cache.sqlite3")
|
||||
return sqlite_path
|
||||
|
||||
|
||||
def _download_official(target: Path, force: bool, progress: ProgressFn) -> Dict[str, Any]:
|
||||
"""Optional fallback: official JLCPCB API (cursor pagination). Requires creds."""
|
||||
from commands.jlcpcb import JLCPCBClient
|
||||
from commands.jlcpcb_parts import JLCPCBPartsManager
|
||||
|
||||
client = JLCPCBClient()
|
||||
if not (client.app_id and client.access_key and client.secret_key):
|
||||
raise RuntimeError("official API credentials not set")
|
||||
|
||||
_progress(progress, "Downloading via official JLCPCB API...")
|
||||
parts = client.download_full_database(
|
||||
callback=lambda page, total, msg: _progress(progress, msg)
|
||||
)
|
||||
if force and target.exists():
|
||||
target.unlink()
|
||||
|
||||
mgr = JLCPCBPartsManager(db_path=str(target))
|
||||
try:
|
||||
mgr.import_parts(parts, progress_callback=lambda c, t, m: _progress(progress, m))
|
||||
stats = mgr.get_database_stats()
|
||||
finally:
|
||||
mgr.close()
|
||||
return _result(
|
||||
"official-api",
|
||||
target,
|
||||
stats["total_parts"],
|
||||
stats["basic_parts"],
|
||||
stats["extended_parts"],
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _result(
|
||||
source: str,
|
||||
target: Path,
|
||||
total: int,
|
||||
basic: int,
|
||||
extended: int,
|
||||
last_modified: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
db_size_mb = round(target.stat().st_size / (1024 * 1024), 2) if target.exists() else 0
|
||||
return {
|
||||
"success": True,
|
||||
"source": source,
|
||||
"total_parts": total,
|
||||
"basic_parts": basic,
|
||||
"extended_parts": extended,
|
||||
"db_size_mb": db_size_mb,
|
||||
"db_path": str(target),
|
||||
"catalog_last_modified": last_modified,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def download_database(
|
||||
force: bool = False,
|
||||
prefer_source: Optional[str] = None,
|
||||
progress: ProgressFn = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Download the JLCPCB catalog into ``jlcpcb_parts.db`` using a layered strategy.
|
||||
|
||||
Order: CDFER (primary) -> yaqwsx (if 7z available) -> official API (if creds).
|
||||
``prefer_source`` (``"cdfer"``/``"yaqwsx"``/``"official"``) forces a single source.
|
||||
Returns a result dict (see ``_result``) or ``{"success": False, "message": ...}``.
|
||||
|
||||
NOTE: callers that hold an open ``JLCPCBPartsManager`` on the target file must
|
||||
close it before calling (the prebuilt paths recreate the file) and reopen after.
|
||||
"""
|
||||
data_dir = PlatformHelper.get_data_dir()
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = data_dir / "jlcpcb_parts.db"
|
||||
cache_dir = data_dir / "jlcparts_cache"
|
||||
|
||||
order = [prefer_source] if prefer_source else ["cdfer", "yaqwsx", "official"]
|
||||
errors: List[str] = []
|
||||
|
||||
for src in order:
|
||||
try:
|
||||
if src == "cdfer":
|
||||
src_path, last_mod = download_cdfer(cache_dir, progress)
|
||||
stats = convert_source_sqlite(src_path, target, progress)
|
||||
_safe_rmtree(cache_dir)
|
||||
return _result(
|
||||
"cdfer", target, stats["total"], stats["basic"], stats["extended"], last_mod
|
||||
)
|
||||
if src == "yaqwsx":
|
||||
src_path = download_yaqwsx(cache_dir, progress)
|
||||
stats = convert_source_sqlite(src_path, target, progress)
|
||||
_safe_rmtree(cache_dir)
|
||||
return _result(
|
||||
"yaqwsx", target, stats["total"], stats["basic"], stats["extended"], None
|
||||
)
|
||||
if src == "official":
|
||||
return _download_official(target, force, progress)
|
||||
errors.append(f"{src}: unknown source")
|
||||
except Exception as exc:
|
||||
logger.warning(f"JLCPCB source '{src}' failed: {exc}")
|
||||
errors.append(f"{src}: {exc}")
|
||||
continue
|
||||
|
||||
return {
|
||||
"success": False,
|
||||
"message": "All JLCPCB download sources failed. "
|
||||
"Install a 7z CLI for the yaqwsx fallback, or set JLCPCB_APP_ID/"
|
||||
"JLCPCB_API_KEY/JLCPCB_API_SECRET for the official API.",
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def _safe_rmtree(path: Path) -> None:
|
||||
try:
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
except OSError as exc: # pragma: no cover - cleanup is best-effort
|
||||
logger.debug(f"cache cleanup failed for {path}: {exc}")
|
||||
@@ -137,59 +137,12 @@ class JLCSearchClient:
|
||||
logger.error(f"Failed to get part C{lcsc_number}: {e}")
|
||||
return None
|
||||
|
||||
def download_all_components(
|
||||
self, callback: Optional[Callable[[int, str], None]] = None, batch_size: int = 100
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Download all components from jlcsearch database
|
||||
|
||||
Note: tscircuit API has a hard-coded 100 result limit per request.
|
||||
Full catalog download requires ~25,000 paginated requests (~40-60 minutes).
|
||||
|
||||
Args:
|
||||
callback: Optional progress callback function(parts_count, status_msg)
|
||||
batch_size: Number of parts per batch (max 100 due to API limit)
|
||||
|
||||
Returns:
|
||||
List of all parts
|
||||
"""
|
||||
all_parts = []
|
||||
offset = 0
|
||||
|
||||
logger.info("Starting full jlcsearch parts database download...")
|
||||
|
||||
while True:
|
||||
try:
|
||||
batch = self.search_components("components", limit=batch_size, offset=offset)
|
||||
|
||||
# Stop if no results returned (end of catalog)
|
||||
if not batch or len(batch) == 0:
|
||||
break
|
||||
|
||||
all_parts.extend(batch)
|
||||
offset += len(batch)
|
||||
|
||||
if callback:
|
||||
callback(len(all_parts), f"Downloaded {len(all_parts)} parts...")
|
||||
else:
|
||||
logger.info(f"Downloaded {len(all_parts)} parts so far...")
|
||||
|
||||
# Continue pagination - API returns exactly 100 results per page until exhausted
|
||||
# Only stop when we get 0 results (handled above)
|
||||
|
||||
# Rate limiting - be nice to the API
|
||||
time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading parts at offset {offset}: {e}")
|
||||
if len(all_parts) > 0:
|
||||
logger.warning(f"Partial download available: {len(all_parts)} parts")
|
||||
return all_parts
|
||||
else:
|
||||
raise
|
||||
|
||||
logger.info(f"Download complete: {len(all_parts)} parts retrieved")
|
||||
return all_parts
|
||||
# NOTE: bulk catalog download was removed (issue #199). The JLCSearch
|
||||
# endpoint is a *search front-end* that ignores the ``offset`` parameter,
|
||||
# so offset-paged "download everything" loops returned the same first 100
|
||||
# parts forever. Full-catalog download now uses a prebuilt source via
|
||||
# ``commands.jlcpcb_downloader.download_database()``. This client remains
|
||||
# for interactive/parametric lookups only (search_components, etc.).
|
||||
|
||||
|
||||
def test_jlcsearch_connection() -> bool:
|
||||
|
||||
Reference in New Issue
Block a user