diff --git a/python/commands/jlcpcb_downloader.py b/python/commands/jlcpcb_downloader.py index 39a2911..e5d8fbc 100644 --- a/python/commands/jlcpcb_downloader.py +++ b/python/commands/jlcpcb_downloader.py @@ -98,19 +98,57 @@ def _library_type(is_basic: Any, is_preferred: Any) -> str: return "Extended" +def _relation_names(src: sqlite3.Connection) -> List[str]: + # Include temp objects so the v_components view built by _ensure_components_view + # (a TEMP view) is visible to _pick_source_relation. + return [ + r[0] + for r in src.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table','view') " + "UNION SELECT name FROM sqlite_temp_master WHERE type IN ('table','view')" + ).fetchall() + ] + + +def _ensure_components_view(src: sqlite3.Connection) -> None: + """Create a denormalized ``v_components`` view for yaqwsx-style sources. + + CDFER ships a ``v_components`` view, but yaqwsx's raw ``cache.sqlite3`` (the + FULL catalog) stores category/manufacturer as IDs in sibling ``categories`` + and ``manufacturers`` tables. Without this join the full-catalog convert + would leave category/subcategory/manufacturer blank. We build the same view + shape CDFER exposes so the rest of the converter is source-agnostic. + """ + names = _relation_names(src) + if "v_components" in names: + return + if not ({"components", "categories", "manufacturers"} <= set(names)): + return + comp_cols = {r[1] for r in src.execute("PRAGMA table_info([components])").fetchall()} + if not ({"category_id", "manufacturer_id"} <= comp_cols): + return + try: + src.execute(""" + CREATE TEMP VIEW v_components AS + SELECT c.*, cat.category AS category, cat.subcategory AS subcategory, + m.name AS manufacturer + FROM components c + LEFT JOIN categories cat ON cat.id = c.category_id + LEFT JOIN manufacturers m ON m.id = c.manufacturer_id + """) + logger.info("Built v_components join view for yaqwsx-style source") + except sqlite3.Error as exc: # pragma: no cover - defensive + logger.debug(f"could not build v_components view: {exc}") + + def _pick_source_relation(src: sqlite3.Connection) -> str: """Choose which table/view to read from the source database. - Prefers CDFER's denormalized ``v_components`` view (which exposes category, - subcategory and manufacturer names directly). Falls back to the largest - table for yaqwsx-style schemas. + Prefers a denormalized ``v_components`` view (CDFER ships one; for yaqwsx we + build an equivalent in ``_ensure_components_view``). Falls back to the + largest table. """ - names = [ - r[0] - for r in src.execute( - "SELECT name FROM sqlite_master WHERE type IN ('table','view')" - ).fetchall() - ] + names = _relation_names(src) if "v_components" in names: return "v_components" if "components" in names: @@ -146,6 +184,7 @@ def convert_source_sqlite( src = sqlite3.connect(str(source_path)) src.row_factory = sqlite3.Row try: + _ensure_components_view(src) relation = _pick_source_relation(src) _progress(progress, f"Converting from source relation '{relation}'...") @@ -287,8 +326,16 @@ def _head_last_modified(url: str) -> Optional[str]: return None -def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path, Optional[str]]: - """Stream-download CDFER's single uncompressed SQLite. Returns (path, last_modified).""" +def download_cdfer( + cache_dir: Path, progress: ProgressFn = None, max_retries: int = 5 +) -> Tuple[Path, Optional[str]]: + """Stream-download CDFER's single uncompressed SQLite, with resume + retry. + + Stalls/drops were a recurring complaint, so this uses a read timeout and, on + a network error, retries while *resuming* from the partial file via an HTTP + Range request (GitHub Pages supports ranged GETs) instead of restarting. + Returns (path, last_modified). + """ import requests cache_dir.mkdir(parents=True, exist_ok=True) @@ -301,23 +348,48 @@ def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path, + "...", ) - with requests.get(CDFER_SQLITE_URL, stream=True, timeout=60) as resp: - resp.raise_for_status() - last_modified = last_modified or resp.headers.get("Last-Modified") - # NOTE: GitHub Pages' Content-Length can understate the real transfer - # size (compressed transfer accounting), so we report MB downloaded - # rather than a misleading percentage. - written = 0 - next_mark = 50 * 1024 * 1024 # log every ~50 MB - with open(dest, "wb") as fh: - for chunk in resp.iter_content(chunk_size=1024 * 1024): - if not chunk: - continue - fh.write(chunk) - written += len(chunk) - if written >= next_mark: - _progress(progress, f"Downloaded {written // (1024 * 1024)} MB...") - next_mark += 50 * 1024 * 1024 + attempt = 0 + while True: + attempt += 1 + resume_from = dest.stat().st_size if dest.exists() else 0 + headers = {"Range": f"bytes={resume_from}-"} if resume_from else {} + try: + # (connect timeout, read timeout): a stalled socket raises after 120s + # of no data, so we can resume rather than hang forever. + with requests.get( + CDFER_SQLITE_URL, stream=True, timeout=(30, 120), headers=headers + ) as resp: + # If we asked to resume but the server ignored Range (200, not + # 206), start the file over to avoid a corrupt append. + if resume_from and resp.status_code == 200: + resume_from = 0 + resp.raise_for_status() + last_modified = last_modified or resp.headers.get("Last-Modified") + mode = "ab" if resume_from else "wb" + written = resume_from + next_mark = ((written // (50 * 1024 * 1024)) + 1) * 50 * 1024 * 1024 + with open(dest, mode) as fh: + for chunk in resp.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + fh.write(chunk) + written += len(chunk) + if written >= next_mark: + _progress(progress, f"Downloaded {written // (1024 * 1024)} MB...") + next_mark += 50 * 1024 * 1024 + break # clean end of stream + except requests.RequestException as exc: + if attempt > max_retries: + raise RuntimeError( + f"CDFER download failed after {max_retries} retries: {exc}" + ) from exc + have_mb = (dest.stat().st_size // (1024 * 1024)) if dest.exists() else 0 + _progress( + progress, + f"Download interrupted ({exc}); resuming from {have_mb} MB " + f"(attempt {attempt}/{max_retries})...", + ) + time.sleep(min(2 * attempt, 10)) if dest.stat().st_size < 1_000_000: raise RuntimeError("CDFER download too small to be valid") @@ -345,8 +417,26 @@ def download_yaqwsx(cache_dir: Path, progress: ProgressFn = None) -> Path: _progress(progress, "Downloading yaqwsx split archive (~421 MB)...") def _curl(url: str, dst: Path) -> bool: + # -C - resumes a partial file; --retry handles transient stalls/drops. return ( - subprocess.run(["curl", "-L", "-f", "-o", str(dst), "--progress-bar", url]).returncode + subprocess.run( + [ + "curl", + "-L", + "-f", + "-C", + "-", + "--retry", + "5", + "--retry-delay", + "2", + "--retry-connrefused", + "-o", + str(dst), + "--progress-bar", + url, + ] + ).returncode == 0 ) diff --git a/src/tools/jlcpcb-api.ts b/src/tools/jlcpcb-api.ts index d8f927c..3d6a485 100644 --- a/src/tools/jlcpcb-api.ts +++ b/src/tools/jlcpcb-api.ts @@ -12,13 +12,16 @@ export function registerJLCPCBApiTools(server: McpServer, callKicadScript: Funct "download_jlcpcb_database", `Download the JLCPCB parts catalog to a local SQLite database for fast offline search. -Uses a prebuilt catalog (no API credentials required by default): - - Primary: CDFER prebuilt SQLite (single file, no extra tools needed) - - Fallback: yaqwsx/jlcparts split archive (requires a 7z CLI) - - Optional: official JLCPCB API if JLCPCB_APP_ID/JLCPCB_API_KEY/JLCPCB_API_SECRET are set +Sources (no API credentials required by default): + - cdfer (default): in-stock subset (~600k parts, ~1.5 GB download). Single file, + no extra tools — the most reliable path, especially on Windows. + - yaqwsx: the FULL catalog (all parts incl. out-of-stock, ~10 GB extracted). + Use source="yaqwsx" if you specifically want everything. Requires a 7z CLI. + - official: official JLCPCB API, used only if JLCPCB_APP_ID/JLCPCB_API_KEY/ + JLCPCB_API_SECRET are set. -This is a one-time setup (downloads ~1.5 GB). Optionally pass source to force one -provider. Re-run with force=true to refresh.`, +One-time setup; downloads resume automatically if interrupted. Re-run with +force=true to refresh.`, { force: z .boolean() @@ -28,7 +31,10 @@ provider. Re-run with force=true to refresh.`, source: z .enum(["cdfer", "yaqwsx", "official"]) .optional() - .describe("Force a single source instead of the default cdfer→yaqwsx→official order"), + .describe( + 'Force one source. "cdfer" (default) = in-stock subset, no 7z needed. ' + + '"yaqwsx" = FULL ~10GB catalog (needs a 7z CLI). "official" = JLCPCB API (needs creds).', + ), }, async (args: { force?: boolean; source?: "cdfer" | "yaqwsx" | "official" }) => { const result = await callKicadScript("download_jlcpcb_database", args); diff --git a/tests/test_jlcpcb_downloader.py b/tests/test_jlcpcb_downloader.py index e306e1a..99fe0ad 100644 --- a/tests/test_jlcpcb_downloader.py +++ b/tests/test_jlcpcb_downloader.py @@ -117,6 +117,65 @@ def test_convert_source_sqlite_produces_manager_schema(tmp_path): con.close() +def _make_yaqwsx_like_source(path: Path) -> None: + """Create a SQLite mirroring yaqwsx's FULL-catalog schema: normalized, NO view. + + category/manufacturer are IDs in sibling tables (no v_components view), which + is what the real ~10GB cache.sqlite3 looks like. + """ + con = sqlite3.connect(str(path)) + con.executescript(""" + CREATE TABLE manufacturers (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE categories (id INTEGER PRIMARY KEY, category TEXT, subcategory TEXT); + CREATE TABLE components ( + lcsc INTEGER PRIMARY KEY, + category_id INTEGER, + mfr TEXT, + package TEXT, + joints INTEGER, + manufacturer_id INTEGER, + basic INTEGER, + preferred INTEGER, + description TEXT, + datasheet TEXT, + stock INTEGER, + price TEXT + ); + """) + con.execute("INSERT INTO manufacturers VALUES (7, 'Texas Instruments')") + con.execute("INSERT INTO categories VALUES (3, 'ICs', 'LDO Regulators')") + con.execute( + "INSERT INTO components VALUES (12345, 3, 'TLV70033', 'SOT-23-5', 5, 7, 0, 0, " + "'3.3V LDO', 'http://ds/12345', 0, ?)", + (json.dumps([{"qFrom": 1, "qTo": None, "price": 0.12}]),), + ) + con.commit() + con.close() + + +def test_convert_yaqwsx_schema_populates_category_and_manufacturer(tmp_path): + """Full-catalog (yaqwsx) source has no v_components view; the join must be built.""" + source = tmp_path / "cache.sqlite3" + target = tmp_path / "jlcpcb_parts.db" + _make_yaqwsx_like_source(source) + + stats = jlcpcb_downloader.convert_source_sqlite(source, target) + assert stats["total"] == 1 + + con = sqlite3.connect(str(target)) + con.row_factory = sqlite3.Row + row = dict(con.execute("SELECT * FROM components WHERE lcsc='C12345'").fetchone()) + con.close() + + # These would be blank without the built v_components join: + assert row["category"] == "ICs" + assert row["subcategory"] == "LDO Regulators" + assert row["manufacturer"] == "Texas Instruments" + assert row["mfr_part"] == "TLV70033" + assert row["library_type"] == "Extended" + assert json.loads(row["price_json"]) == [{"qty": 1, "price": 0.12}] + + def test_normalize_price_json_handles_scalar_and_array_and_empty(): assert json.loads(jlcpcb_downloader.normalize_price_json(None)) == [] assert json.loads(jlcpcb_downloader.normalize_price_json("")) == []