feat(jlcpcb): make the FULL ~10GB catalog reachable + resumable downloads (#199)

People want to be able to pull the whole catalog, and prior downloads stalled or
repeated the same parts. Two fixes:

- Full catalog correctness: yaqwsx's cache.sqlite3 (the full ~10GB set) stores
  category/manufacturer as IDs with no v_components view, so the convert left
  those fields blank. Build an equivalent v_components join for yaqwsx-style
  sources so the full catalog converts with category/subcategory/manufacturer
  populated. Clarify in the tool schema that source="yaqwsx" = full catalog
  (needs 7z) vs cdfer = in-stock subset.
- Resumable downloads: CDFER stream download now uses a read timeout and resumes
  from the partial file via HTTP Range on interruption (up to 5 retries); the
  yaqwsx curl calls use -C - / --retry. Addresses the stall/partial-download
  complaints.

Adds a yaqwsx-schema conversion test.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

python/commands/jlcpcb_downloader.py

@@ -98,19 +98,57 @@ def _library_type(is_basic: Any, is_preferred: Any) -> str:
     return "Extended"
 
 
+def _relation_names(src: sqlite3.Connection) -> List[str]:
+    # Include temp objects so the v_components view built by _ensure_components_view
+    # (a TEMP view) is visible to _pick_source_relation.
+    return [
+        r[0]
+        for r in src.execute(
+            "SELECT name FROM sqlite_master WHERE type IN ('table','view') "
+            "UNION SELECT name FROM sqlite_temp_master WHERE type IN ('table','view')"
+        ).fetchall()
+    ]
+
+
+def _ensure_components_view(src: sqlite3.Connection) -> None:
+    """Create a denormalized ``v_components`` view for yaqwsx-style sources.
+
+    CDFER ships a ``v_components`` view, but yaqwsx's raw ``cache.sqlite3`` (the
+    FULL catalog) stores category/manufacturer as IDs in sibling ``categories``
+    and ``manufacturers`` tables. Without this join the full-catalog convert
+    would leave category/subcategory/manufacturer blank. We build the same view
+    shape CDFER exposes so the rest of the converter is source-agnostic.
+    """
+    names = _relation_names(src)
+    if "v_components" in names:
+        return
+    if not ({"components", "categories", "manufacturers"} <= set(names)):
+        return
+    comp_cols = {r[1] for r in src.execute("PRAGMA table_info([components])").fetchall()}
+    if not ({"category_id", "manufacturer_id"} <= comp_cols):
+        return
+    try:
+        src.execute("""
+            CREATE TEMP VIEW v_components AS
+            SELECT c.*, cat.category AS category, cat.subcategory AS subcategory,
+                   m.name AS manufacturer
+            FROM components c
+            LEFT JOIN categories cat ON cat.id = c.category_id
+            LEFT JOIN manufacturers m ON m.id = c.manufacturer_id
+            """)
+        logger.info("Built v_components join view for yaqwsx-style source")
+    except sqlite3.Error as exc:  # pragma: no cover - defensive
+        logger.debug(f"could not build v_components view: {exc}")
+
+
 def _pick_source_relation(src: sqlite3.Connection) -> str:
     """Choose which table/view to read from the source database.
 
-    Prefers CDFER's denormalized ``v_components`` view (which exposes category,
+    Prefers a denormalized ``v_components`` view (CDFER ships one; for yaqwsx we
-    subcategory and manufacturer names directly). Falls back to the largest
+    build an equivalent in ``_ensure_components_view``). Falls back to the
-    table for yaqwsx-style schemas.
+    largest table.
     """
-    names = [
+    names = _relation_names(src)
-        r[0]
-        for r in src.execute(
-            "SELECT name FROM sqlite_master WHERE type IN ('table','view')"
-        ).fetchall()
-    ]
     if "v_components" in names:
         return "v_components"
     if "components" in names:
@@ -146,6 +184,7 @@ def convert_source_sqlite(
     src = sqlite3.connect(str(source_path))
     src.row_factory = sqlite3.Row
     try:
+        _ensure_components_view(src)
         relation = _pick_source_relation(src)
         _progress(progress, f"Converting from source relation '{relation}'...")
 
@@ -287,8 +326,16 @@ def _head_last_modified(url: str) -> Optional[str]:
     return None
 
 
-def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path, Optional[str]]:
+def download_cdfer(
-    """Stream-download CDFER's single uncompressed SQLite. Returns (path, last_modified)."""
+    cache_dir: Path, progress: ProgressFn = None, max_retries: int = 5
+) -> Tuple[Path, Optional[str]]:
+    """Stream-download CDFER's single uncompressed SQLite, with resume + retry.
+
+    Stalls/drops were a recurring complaint, so this uses a read timeout and, on
+    a network error, retries while *resuming* from the partial file via an HTTP
+    Range request (GitHub Pages supports ranged GETs) instead of restarting.
+    Returns (path, last_modified).
+    """
     import requests
 
     cache_dir.mkdir(parents=True, exist_ok=True)
@@ -301,23 +348,48 @@ def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path,
         + "...",
     )
 
-    with requests.get(CDFER_SQLITE_URL, stream=True, timeout=60) as resp:
+    attempt = 0
-        resp.raise_for_status()
+    while True:
-        last_modified = last_modified or resp.headers.get("Last-Modified")
+        attempt += 1
-        # NOTE: GitHub Pages' Content-Length can understate the real transfer
+        resume_from = dest.stat().st_size if dest.exists() else 0
-        # size (compressed transfer accounting), so we report MB downloaded
+        headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
-        # rather than a misleading percentage.
+        try:
-        written = 0
+            # (connect timeout, read timeout): a stalled socket raises after 120s
-        next_mark = 50 * 1024 * 1024  # log every ~50 MB
+            # of no data, so we can resume rather than hang forever.
-        with open(dest, "wb") as fh:
+            with requests.get(
-            for chunk in resp.iter_content(chunk_size=1024 * 1024):
+                CDFER_SQLITE_URL, stream=True, timeout=(30, 120), headers=headers
-                if not chunk:
+            ) as resp:
-                    continue
+                # If we asked to resume but the server ignored Range (200, not
-                fh.write(chunk)
+                # 206), start the file over to avoid a corrupt append.
-                written += len(chunk)
+                if resume_from and resp.status_code == 200:
-                if written >= next_mark:
+                    resume_from = 0
-                    _progress(progress, f"Downloaded {written // (1024 * 1024)} MB...")
+                resp.raise_for_status()
-                    next_mark += 50 * 1024 * 1024
+                last_modified = last_modified or resp.headers.get("Last-Modified")
+                mode = "ab" if resume_from else "wb"
+                written = resume_from
+                next_mark = ((written // (50 * 1024 * 1024)) + 1) * 50 * 1024 * 1024
+                with open(dest, mode) as fh:
+                    for chunk in resp.iter_content(chunk_size=1024 * 1024):
+                        if not chunk:
+                            continue
+                        fh.write(chunk)
+                        written += len(chunk)
+                        if written >= next_mark:
+                            _progress(progress, f"Downloaded {written // (1024 * 1024)} MB...")
+                            next_mark += 50 * 1024 * 1024
+            break  # clean end of stream
+        except requests.RequestException as exc:
+            if attempt > max_retries:
+                raise RuntimeError(
+                    f"CDFER download failed after {max_retries} retries: {exc}"
+                ) from exc
+            have_mb = (dest.stat().st_size // (1024 * 1024)) if dest.exists() else 0
+            _progress(
+                progress,
+                f"Download interrupted ({exc}); resuming from {have_mb} MB "
+                f"(attempt {attempt}/{max_retries})...",
+            )
+            time.sleep(min(2 * attempt, 10))
 
     if dest.stat().st_size < 1_000_000:
         raise RuntimeError("CDFER download too small to be valid")
@@ -345,8 +417,26 @@ def download_yaqwsx(cache_dir: Path, progress: ProgressFn = None) -> Path:
     _progress(progress, "Downloading yaqwsx split archive (~421 MB)...")
 
     def _curl(url: str, dst: Path) -> bool:
+        # -C - resumes a partial file; --retry handles transient stalls/drops.
         return (
-            subprocess.run(["curl", "-L", "-f", "-o", str(dst), "--progress-bar", url]).returncode
+            subprocess.run(
+                [
+                    "curl",
+                    "-L",
+                    "-f",
+                    "-C",
+                    "-",
+                    "--retry",
+                    "5",
+                    "--retry-delay",
+                    "2",
+                    "--retry-connrefused",
+                    "-o",
+                    str(dst),
+                    "--progress-bar",
+                    url,
+                ]
+            ).returncode
             == 0
         )
 

src/tools/jlcpcb-api.ts

@@ -12,13 +12,16 @@ export function registerJLCPCBApiTools(server: McpServer, callKicadScript: Funct
     "download_jlcpcb_database",
     `Download the JLCPCB parts catalog to a local SQLite database for fast offline search.
 
-Uses a prebuilt catalog (no API credentials required by default):
+Sources (no API credentials required by default):
-  - Primary: CDFER prebuilt SQLite (single file, no extra tools needed)
+  - cdfer (default): in-stock subset (~600k parts, ~1.5 GB download). Single file,
-  - Fallback: yaqwsx/jlcparts split archive (requires a 7z CLI)
+    no extra tools — the most reliable path, especially on Windows.
-  - Optional: official JLCPCB API if JLCPCB_APP_ID/JLCPCB_API_KEY/JLCPCB_API_SECRET are set
+  - yaqwsx: the FULL catalog (all parts incl. out-of-stock, ~10 GB extracted).
+    Use source="yaqwsx" if you specifically want everything. Requires a 7z CLI.
+  - official: official JLCPCB API, used only if JLCPCB_APP_ID/JLCPCB_API_KEY/
+    JLCPCB_API_SECRET are set.
 
-This is a one-time setup (downloads ~1.5 GB). Optionally pass source to force one
+One-time setup; downloads resume automatically if interrupted. Re-run with
-provider. Re-run with force=true to refresh.`,
+force=true to refresh.`,
     {
       force: z
         .boolean()
@@ -28,7 +31,10 @@ provider. Re-run with force=true to refresh.`,
       source: z
         .enum(["cdfer", "yaqwsx", "official"])
         .optional()
-        .describe("Force a single source instead of the default cdfer→yaqwsx→official order"),
+        .describe(
+          'Force one source. "cdfer" (default) = in-stock subset, no 7z needed. ' +
+            '"yaqwsx" = FULL ~10GB catalog (needs a 7z CLI). "official" = JLCPCB API (needs creds).',
+        ),
     },
     async (args: { force?: boolean; source?: "cdfer" | "yaqwsx" | "official" }) => {
       const result = await callKicadScript("download_jlcpcb_database", args);

tests/test_jlcpcb_downloader.py

@@ -117,6 +117,65 @@ def test_convert_source_sqlite_produces_manager_schema(tmp_path):
     con.close()
 
 
+def _make_yaqwsx_like_source(path: Path) -> None:
+    """Create a SQLite mirroring yaqwsx's FULL-catalog schema: normalized, NO view.
+
+    category/manufacturer are IDs in sibling tables (no v_components view), which
+    is what the real ~10GB cache.sqlite3 looks like.
+    """
+    con = sqlite3.connect(str(path))
+    con.executescript("""
+        CREATE TABLE manufacturers (id INTEGER PRIMARY KEY, name TEXT);
+        CREATE TABLE categories (id INTEGER PRIMARY KEY, category TEXT, subcategory TEXT);
+        CREATE TABLE components (
+            lcsc INTEGER PRIMARY KEY,
+            category_id INTEGER,
+            mfr TEXT,
+            package TEXT,
+            joints INTEGER,
+            manufacturer_id INTEGER,
+            basic INTEGER,
+            preferred INTEGER,
+            description TEXT,
+            datasheet TEXT,
+            stock INTEGER,
+            price TEXT
+        );
+        """)
+    con.execute("INSERT INTO manufacturers VALUES (7, 'Texas Instruments')")
+    con.execute("INSERT INTO categories VALUES (3, 'ICs', 'LDO Regulators')")
+    con.execute(
+        "INSERT INTO components VALUES (12345, 3, 'TLV70033', 'SOT-23-5', 5, 7, 0, 0, "
+        "'3.3V LDO', 'http://ds/12345', 0, ?)",
+        (json.dumps([{"qFrom": 1, "qTo": None, "price": 0.12}]),),
+    )
+    con.commit()
+    con.close()
+
+
+def test_convert_yaqwsx_schema_populates_category_and_manufacturer(tmp_path):
+    """Full-catalog (yaqwsx) source has no v_components view; the join must be built."""
+    source = tmp_path / "cache.sqlite3"
+    target = tmp_path / "jlcpcb_parts.db"
+    _make_yaqwsx_like_source(source)
+
+    stats = jlcpcb_downloader.convert_source_sqlite(source, target)
+    assert stats["total"] == 1
+
+    con = sqlite3.connect(str(target))
+    con.row_factory = sqlite3.Row
+    row = dict(con.execute("SELECT * FROM components WHERE lcsc='C12345'").fetchone())
+    con.close()
+
+    # These would be blank without the built v_components join:
+    assert row["category"] == "ICs"
+    assert row["subcategory"] == "LDO Regulators"
+    assert row["manufacturer"] == "Texas Instruments"
+    assert row["mfr_part"] == "TLV70033"
+    assert row["library_type"] == "Extended"
+    assert json.loads(row["price_json"]) == [{"qty": 1, "price": 0.12}]
+
+
 def test_normalize_price_json_handles_scalar_and_array_and_empty():
     assert json.loads(jlcpcb_downloader.normalize_price_json(None)) == []
     assert json.loads(jlcpcb_downloader.normalize_price_json("")) == []