From 7b6d3276885824059307e8af074cad92983f6ab7 Mon Sep 17 00:00:00 2001
From: Matthew Ellis <matthewellis1995@gmail.com>
Date: Thu, 23 Apr 2026 02:41:11 +0100
Subject: [PATCH] fix: improve JLCPCB downloader robustness and disk usage

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 download_jlcpcb.py | 54 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 13 deletions(-)

diff --git a/download_jlcpcb.py b/download_jlcpcb.py
index 63c34a8..8c97b94 100644
--- a/download_jlcpcb.py
+++ b/download_jlcpcb.py
@@ -12,6 +12,7 @@ KiCad MCP server's JLCPCBPartsManager.
 
 import json
 import os
+import shutil
 import sqlite3
 import subprocess
 import sys
@@ -25,27 +26,46 @@ CACHE_DIR = DATA_DIR / "jlcparts_cache"
 CACHE_DIR.mkdir(exist_ok=True)
 
 BASE_URL = "https://yaqwsx.github.io/jlcparts/data"
-PARTS = [f"cache.z{i:02d}" for i in range(1, 20)] + ["cache.zip"]
+MAX_PARTS = 30  # probe up to this many split volumes
 
 TARGET_DB = DATA_DIR / "jlcpcb_parts.db"
 
 
+def curl_download(url: str, dest: Path) -> bool:
+    """Download url to dest with curl. Returns False on 4xx/5xx or network error."""
+    result = subprocess.run(
+        ["curl", "-L", "-f", "-o", str(dest), "--progress-bar", url], capture_output=False
+    )
+    return result.returncode == 0
+
+
 def download_files():
-    """Download all split archive parts."""
+    """Download split archive volumes (z01..zNN) then cache.zip, stopping volumes at 404 or MAX_PARTS."""
     print("Downloading jlcparts database (~421MB)...")
-    for part in PARTS:
+
+    for i in range(1, MAX_PARTS + 1):
+        part = f"cache.z{i:02d}"
         dest = CACHE_DIR / part
         if dest.exists() and dest.stat().st_size > 1000:
             print(f"  {part} already exists, skipping")
             continue
-        url = f"{BASE_URL}/{part}"
         print(f"  Downloading {part}...")
-        result = subprocess.run(
-            ["curl", "-L", "-o", str(dest), "--progress-bar", url], capture_output=False
-        )
-        if result.returncode != 0:
-            print(f"  ERROR downloading {part}")
+        if not curl_download(f"{BASE_URL}/{part}", dest):
+            # -f causes non-zero exit on 4xx/5xx; treat as end of volumes
+            if dest.exists():
+                dest.unlink()
+            print(f"  {part} not found — {i - 1} volumes total")
+            break
+
+    dest = CACHE_DIR / "cache.zip"
+    if dest.exists() and dest.stat().st_size > 1000:
+        print("  cache.zip already exists, skipping")
+    else:
+        print("  Downloading cache.zip...")
+        if not curl_download(f"{BASE_URL}/cache.zip", dest):
+            print("  ERROR downloading cache.zip")
             return False
+
     return True
 
 
@@ -116,7 +136,8 @@ def convert_to_mcp_format():
 
     # Create target DB in MCP format
     dst = sqlite3.connect(str(TARGET_DB))
-    dst.execute("""
+    dst.execute(
+        """
         CREATE TABLE components (
             lcsc TEXT PRIMARY KEY,
             category TEXT,
@@ -132,7 +153,8 @@ def convert_to_mcp_format():
             price_json TEXT,
             last_updated INTEGER
         )
-    """)
+    """
+    )
     dst.execute("CREATE INDEX idx_category ON components(category, subcategory)")
     dst.execute("CREATE INDEX idx_package ON components(package)")
     dst.execute("CREATE INDEX idx_manufacturer ON components(manufacturer)")
@@ -233,12 +255,14 @@ def convert_to_mcp_format():
 
     # Build FTS index
     print(f"  Building full-text search index...")
-    dst.execute("""
+    dst.execute(
+        """
         CREATE VIRTUAL TABLE IF NOT EXISTS components_fts USING fts5(
             lcsc, description, mfr_part, manufacturer,
             content=components
         )
-    """)
+    """
+    )
     dst.execute("INSERT INTO components_fts(components_fts) VALUES('rebuild')")
     dst.commit()
 
@@ -252,6 +276,10 @@ def convert_to_mcp_format():
     dst.close()
     src.close()
 
+    print("Cleaning up source database...")
+    shutil.rmtree(CACHE_DIR)
+    print(f"  Removed {CACHE_DIR}")
+
     db_size = TARGET_DB.stat().st_size / (1024 * 1024)
     print(f"\nDatabase ready: {TARGET_DB}")
     print(f"  Total parts:    {total:,}")