nearxos/kicad-mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(jlcpcb): download prebuilt catalog instead of broken JLCSearch offset loop (#199)

Browse Source 
The download_jlcpcb_database tool paged the community JLCSearch API with an
offset parameter, but that endpoint is a search front-end that ignores offset
and returns the same first 100 parts on every page, so a full catalog download
was impossible.

Add commands/jlcpcb_downloader.py with a layered strategy that reuses prebuilt
catalogs the whole ecosystem already trusts:
  - CDFER single-file SQLite (primary; no 7z/zip, reliable on Windows)
  - yaqwsx/jlcparts split 7z (fallback; only if a 7z CLI is present)
  - official JLCPCB API (optional; cursor pagination, if credentials set)

Conversion reads CDFER's v_components view (or sniffs the largest table for
yaqwsx), C-prefixes integer lcsc, derives library_type from basic/preferred,
maps mfr->mfr_part, and normalizes price JSON to the manager's [{qty,price}]
shape. Rewire _handle_download_jlcpcb_database to use it (closing/reopening the
manager connection so the on-disk db can be rewritten on Windows). Remove the
broken offset loop from jlcsearch.py (client kept for interactive lookups).
Reduce download_jlcpcb.py to a thin CLI wrapper and update the TS tool schema.

Verified end-to-end against live CDFER: 616k parts downloaded + converted in
~40s, FTS search and price-break parsing correct. New unit tests cover the
conversion and source fall-through; no network in tests.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
mixelpixx
2026-05-24 11:53:12 -04:00
parent d765bfec78
commit b21b1410eb
6 changed files with 773 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

331
download_jlcpcb.py
View File

@@ -1,314 +1,69 @@
#!/usr/bin/env python3
"""
Download JLCPCB parts database from yaqwsx/jlcparts pre-built cache.
CLI to download the JLCPCB parts database into the MCP server's local DB.
This downloads the full JLCPCB catalog (~421MB compressed, ~1.5GB SQLite)
from GitHub Pages in ~5 minutes instead of the broken JLCSearch API approach.
This is a thin wrapper around ``commands.jlcpcb_downloader`` (the same code the
``download_jlcpcb_database`` MCP tool uses). It downloads a prebuilt catalog and
converts it into ``jlcpcb_parts.db``:
The cache.sqlite3 file contains all JLCPCB parts with stock, pricing,
and category data. We then convert it into the format expected by the
KiCad MCP server's JLCPCBPartsManager.
CDFER single-file SQLite (primary, no 7z) ->
yaqwsx split-7z (fallback, needs 7z CLI) ->
official JLCPCB API (optional, if JLCPCB_APP_ID/API_KEY/API_SECRET set)
Usage:
python download_jlcpcb.py [--source cdfer|yaqwsx|official] [--force]
"""
import json
import os
import shutil
import sqlite3
import subprocess
import argparse
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent / "python"))
from utils.platform_helper import PlatformHelper # noqa: E402
DATA_DIR = PlatformHelper.get_data_dir()
DATA_DIR.mkdir(parents=True, exist_ok=True)
CACHE_DIR = DATA_DIR / "jlcparts_cache"
CACHE_DIR.mkdir(parents=True, exist_ok=True)
BASE_URL = "https://yaqwsx.github.io/jlcparts/data"
MAX_PARTS = 30 # probe up to this many split volumes
TARGET_DB = DATA_DIR / "jlcpcb_parts.db"
def curl_download(url: str, dest: Path) -> bool:
"""Download url to dest with curl. Returns False on 4xx/5xx or network error."""
result = subprocess.run(
["curl", "-L", "-f", "-o", str(dest), "--progress-bar", url], capture_output=False
)
return result.returncode == 0
def download_files() -> bool:
"""Download split archive volumes (z01..zNN) then cache.zip, stopping volumes at 404 or MAX_PARTS."""
print("Downloading jlcparts database (~421MB)...")
for i in range(1, MAX_PARTS + 1):
part = f"cache.z{i:02d}"
dest = CACHE_DIR / part
if dest.exists() and dest.stat().st_size > 1000:
print(f" {part} already exists, skipping")
continue
print(f" Downloading {part}...")
if not curl_download(f"{BASE_URL}/{part}", dest):
# -f causes non-zero exit on 4xx/5xx; treat as end of volumes
if dest.exists():
dest.unlink()
print(f" {part} not found — {i - 1} volumes total")
break
dest = CACHE_DIR / "cache.zip"
if dest.exists() and dest.stat().st_size > 1000:
print(" cache.zip already exists, skipping")
else:
print(" Downloading cache.zip...")
if not curl_download(f"{BASE_URL}/cache.zip", dest):
print(" ERROR downloading cache.zip")
return False
return True
def extract_database() -> bool:
"""Extract the split 7z archive to get cache.sqlite3."""
cache_sqlite = CACHE_DIR / "cache.sqlite3"
if cache_sqlite.exists() and cache_sqlite.stat().st_size > 100_000_000:
print(f"cache.sqlite3 already extracted ({cache_sqlite.stat().st_size // (1024*1024)}MB)")
return True
print("Extracting archive (requires 7z or p7zip)...")
# Try 7z first, then 7zz (homebrew)
for cmd in ["7z", "7zz", "7za"]:
try:
result = subprocess.run(
[cmd, "x", "-y", "-o" + str(CACHE_DIR), str(CACHE_DIR / "cache.zip")],
capture_output=True,
text=True,
)
if result.returncode == 0:
print(f"Extracted with {cmd}")
return True
else:
print(f" {cmd} failed: {result.stderr[:200]}")
except FileNotFoundError:
continue
print("\nERROR: 7z not found. Install with: brew install p7zip")
return False
def convert_to_mcp_format() -> bool:
"""Convert jlcparts cache.sqlite3 to the MCP server's expected format."""
source = CACHE_DIR / "cache.sqlite3"
if not source.exists():
print("ERROR: cache.sqlite3 not found")
return False
print(f"Reading source database...")
src = sqlite3.connect(str(source))
src.row_factory = sqlite3.Row
# Check schema
tables = [
r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
]
print(f" Source tables: {tables}")
# Find the main components table
comp_table = None
for t in tables:
count = src.execute(f"SELECT COUNT(*) FROM [{t}]").fetchone()[0]
print(f" {t}: {count:,} rows")
if count > 10000 and comp_table is None:
comp_table = t
if not comp_table:
# Try 'components' specifically
comp_table = "components" if "components" in tables else tables[0]
# Get column names
cols = [r[1] for r in src.execute(f"PRAGMA table_info([{comp_table}])").fetchall()]
print(f" Using table '{comp_table}' with columns: {cols[:10]}...")
# Remove old target DB
if TARGET_DB.exists():
TARGET_DB.unlink()
# Create target DB in MCP format
dst = sqlite3.connect(str(TARGET_DB))
dst.execute(
"""
CREATE TABLE components (
lcsc TEXT PRIMARY KEY,
category TEXT,
subcategory TEXT,
mfr_part TEXT,
package TEXT,
solder_joints INTEGER,
manufacturer TEXT,
library_type TEXT,
description TEXT,
datasheet TEXT,
stock INTEGER,
price_json TEXT,
last_updated INTEGER
)
"""
)
dst.execute("CREATE INDEX idx_category ON components(category, subcategory)")
dst.execute("CREATE INDEX idx_package ON components(package)")
dst.execute("CREATE INDEX idx_manufacturer ON components(manufacturer)")
dst.execute("CREATE INDEX idx_library_type ON components(library_type)")
dst.execute("CREATE INDEX idx_mfr_part ON components(mfr_part)")
# Map source columns to our schema
# jlcparts schema varies but commonly has:
# lcsc, mfr, description, joint, manufacturer, basic, preferred, stock, price, url, etc.
print(f"\nConverting parts to MCP format...")
now = int(time.time())
batch = []
count = 0
for row in src.execute(f"SELECT * FROM [{comp_table}]"):
row_dict = dict(row)
# Adapt column names (jlcparts uses various schemas)
lcsc = row_dict.get("lcsc") or row_dict.get("LCSC_Part") or row_dict.get("lcsc_id")
if lcsc is None:
continue
if isinstance(lcsc, int):
lcsc = f"C{lcsc}"
elif not str(lcsc).startswith("C"):
lcsc = f"C{lcsc}"
mfr_part = row_dict.get("mfr") or row_dict.get("MFR_Part") or row_dict.get("mfr_part") or ""
package = row_dict.get("package") or row_dict.get("Package") or ""
manufacturer = row_dict.get("manufacturer") or row_dict.get("Manufacturer") or ""
description = row_dict.get("description") or row_dict.get("Description") or ""
stock = row_dict.get("stock") or row_dict.get("Stock") or 0
category = row_dict.get("category") or row_dict.get("First Category") or ""
subcategory = row_dict.get("subcategory") or row_dict.get("Second Category") or ""
datasheet = row_dict.get("datasheet") or row_dict.get("url") or ""
# Library type
is_basic = row_dict.get("basic") or row_dict.get("is_basic") or row_dict.get("Basic")
is_preferred = (
row_dict.get("preferred") or row_dict.get("is_preferred") or row_dict.get("Preferred")
)
if is_basic:
lib_type = "Basic"
elif is_preferred:
lib_type = "Preferred"
else:
lib_type = "Extended"
# Price
price = row_dict.get("price") or row_dict.get("Price") or 0
price_json = json.dumps([{"qty": 1, "price": price}] if price else [])
batch.append(
(
str(lcsc),
category,
subcategory,
mfr_part,
package,
0,
manufacturer,
lib_type,
description,
datasheet,
int(stock) if stock else 0,
price_json,
now,
)
)
if len(batch) >= 10000:
dst.executemany(
"""
INSERT OR REPLACE INTO components
(lcsc, category, subcategory, mfr_part, package,
solder_joints, manufacturer, library_type, description,
datasheet, stock, price_json, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
batch,
)
count += len(batch)
batch = []
if count % 100000 == 0:
print(f" Converted {count:,} parts...")
if batch:
dst.executemany(
"""
INSERT OR REPLACE INTO components
(lcsc, category, subcategory, mfr_part, package,
solder_joints, manufacturer, library_type, description,
datasheet, stock, price_json, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
batch,
)
count += len(batch)
# Build FTS index
print(f" Building full-text search index...")
dst.execute(
"""
CREATE VIRTUAL TABLE IF NOT EXISTS components_fts USING fts5(
lcsc, description, mfr_part, manufacturer,
content=components
)
"""
)
dst.execute("INSERT INTO components_fts(components_fts) VALUES('rebuild')")
dst.commit()
# Stats
total = dst.execute("SELECT COUNT(*) FROM components").fetchone()[0]
basic = dst.execute("SELECT COUNT(*) FROM components WHERE library_type='Basic'").fetchone()[0]
extended = dst.execute(
"SELECT COUNT(*) FROM components WHERE library_type='Extended'"
).fetchone()[0]
dst.close()
src.close()
print("Cleaning up source database...")
shutil.rmtree(CACHE_DIR)
print(f" Removed {CACHE_DIR}")
db_size = TARGET_DB.stat().st_size / (1024 * 1024)
print(f"\nDatabase ready: {TARGET_DB}")
print(f" Total parts: {total:,}")
print(f" Basic parts: {basic:,}")
print(f" Extended parts: {extended:,}")
print(f" DB size: {db_size:.1f} MB")
return True
from commands import jlcpcb_downloader # noqa: E402
def main() -> None:
parser = argparse.ArgumentParser(description="Download the JLCPCB parts database")
parser.add_argument(
"--source",
choices=["cdfer", "yaqwsx", "official"],
default=None,
help="Force a single source (default: try cdfer, then yaqwsx, then official)",
)
parser.add_argument(
"--force", action="store_true", help="Re-download even if a database already exists"
)
args = parser.parse_args()
print("=" * 60)
print("JLCPCB Parts Database Downloader (jlcparts method)")
print("JLCPCB Parts Database Downloader")
print("=" * 60)
start = time.time()
if not download_files():
sys.exit(1)
result = jlcpcb_downloader.download_database(
force=args.force,
prefer_source=args.source,
progress=lambda msg: print(f" {msg}"),
)
if not extract_database():
sys.exit(1)
if not convert_to_mcp_format():
if not result.get("success"):
print(f"\nERROR: {result.get('message', 'download failed')}")
for err in result.get("errors", []):
print(f" - {err}")
sys.exit(1)
elapsed = time.time() - start
print(f"\nTotal time: {elapsed/60:.1f} minutes")
print(f"\nDatabase ready: {result['db_path']}")
print(f" Source: {result['source']}")
if result.get("catalog_last_modified"):
print(f" Catalog dated: {result['catalog_last_modified']}")
print(f" Total parts: {result['total_parts']:,}")
print(f" Basic parts: {result['basic_parts']:,}")
print(f" Extended parts: {result['extended_parts']:,}")
print(f" DB size: {result['db_size_mb']} MB")
print(f"\nTotal time: {elapsed / 60:.1f} minutes")
print("Done! Restart the MCP server (/mcp) to use the new database.")