fix(jlcpcb): download prebuilt catalog instead of broken JLCSearch offset loop (#199)
The download_jlcpcb_database tool paged the community JLCSearch API with an
offset parameter, but that endpoint is a search front-end that ignores offset
and returns the same first 100 parts on every page, so a full catalog download
was impossible.
Add commands/jlcpcb_downloader.py with a layered strategy that reuses prebuilt
catalogs the whole ecosystem already trusts:
- CDFER single-file SQLite (primary; no 7z/zip, reliable on Windows)
- yaqwsx/jlcparts split 7z (fallback; only if a 7z CLI is present)
- official JLCPCB API (optional; cursor pagination, if credentials set)
Conversion reads CDFER's v_components view (or sniffs the largest table for
yaqwsx), C-prefixes integer lcsc, derives library_type from basic/preferred,
maps mfr->mfr_part, and normalizes price JSON to the manager's [{qty,price}]
shape. Rewire _handle_download_jlcpcb_database to use it (closing/reopening the
manager connection so the on-disk db can be rewritten on Windows). Remove the
broken offset loop from jlcsearch.py (client kept for interactive lookups).
Reduce download_jlcpcb.py to a thin CLI wrapper and update the TS tool schema.
Verified end-to-end against live CDFER: 616k parts downloaded + converted in
~40s, FTS search and price-break parsing correct. New unit tests cover the
conversion and source fall-through; no network in tests.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
mixelpixx
@@ -1,314 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Download JLCPCB parts database from yaqwsx/jlcparts pre-built cache.
|
||||
CLI to download the JLCPCB parts database into the MCP server's local DB.
|
||||
|
||||
This downloads the full JLCPCB catalog (~421MB compressed, ~1.5GB SQLite)
|
||||
from GitHub Pages in ~5 minutes instead of the broken JLCSearch API approach.
|
||||
This is a thin wrapper around ``commands.jlcpcb_downloader`` (the same code the
|
||||
``download_jlcpcb_database`` MCP tool uses). It downloads a prebuilt catalog and
|
||||
converts it into ``jlcpcb_parts.db``:
|
||||
|
||||
The cache.sqlite3 file contains all JLCPCB parts with stock, pricing,
|
||||
and category data. We then convert it into the format expected by the
|
||||
KiCad MCP server's JLCPCBPartsManager.
|
||||
CDFER single-file SQLite (primary, no 7z) ->
|
||||
yaqwsx split-7z (fallback, needs 7z CLI) ->
|
||||
official JLCPCB API (optional, if JLCPCB_APP_ID/API_KEY/API_SECRET set)
|
||||
|
||||
Usage:
|
||||
python download_jlcpcb.py [--source cdfer|yaqwsx|official] [--force]
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent / "python"))
|
||||
from utils.platform_helper import PlatformHelper # noqa: E402
|
||||
|
||||
DATA_DIR = PlatformHelper.get_data_dir()
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CACHE_DIR = DATA_DIR / "jlcparts_cache"
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
BASE_URL = "https://yaqwsx.github.io/jlcparts/data"
|
||||
MAX_PARTS = 30 # probe up to this many split volumes
|
||||
|
||||
TARGET_DB = DATA_DIR / "jlcpcb_parts.db"
|
||||
|
||||
|
||||
def curl_download(url: str, dest: Path) -> bool:
|
||||
"""Download url to dest with curl. Returns False on 4xx/5xx or network error."""
|
||||
result = subprocess.run(
|
||||
["curl", "-L", "-f", "-o", str(dest), "--progress-bar", url], capture_output=False
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def download_files() -> bool:
|
||||
"""Download split archive volumes (z01..zNN) then cache.zip, stopping volumes at 404 or MAX_PARTS."""
|
||||
print("Downloading jlcparts database (~421MB)...")
|
||||
|
||||
for i in range(1, MAX_PARTS + 1):
|
||||
part = f"cache.z{i:02d}"
|
||||
dest = CACHE_DIR / part
|
||||
if dest.exists() and dest.stat().st_size > 1000:
|
||||
print(f" {part} already exists, skipping")
|
||||
continue
|
||||
print(f" Downloading {part}...")
|
||||
if not curl_download(f"{BASE_URL}/{part}", dest):
|
||||
# -f causes non-zero exit on 4xx/5xx; treat as end of volumes
|
||||
if dest.exists():
|
||||
dest.unlink()
|
||||
print(f" {part} not found — {i - 1} volumes total")
|
||||
break
|
||||
|
||||
dest = CACHE_DIR / "cache.zip"
|
||||
if dest.exists() and dest.stat().st_size > 1000:
|
||||
print(" cache.zip already exists, skipping")
|
||||
else:
|
||||
print(" Downloading cache.zip...")
|
||||
if not curl_download(f"{BASE_URL}/cache.zip", dest):
|
||||
print(" ERROR downloading cache.zip")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def extract_database() -> bool:
|
||||
"""Extract the split 7z archive to get cache.sqlite3."""
|
||||
cache_sqlite = CACHE_DIR / "cache.sqlite3"
|
||||
if cache_sqlite.exists() and cache_sqlite.stat().st_size > 100_000_000:
|
||||
print(f"cache.sqlite3 already extracted ({cache_sqlite.stat().st_size // (1024*1024)}MB)")
|
||||
return True
|
||||
|
||||
print("Extracting archive (requires 7z or p7zip)...")
|
||||
# Try 7z first, then 7zz (homebrew)
|
||||
for cmd in ["7z", "7zz", "7za"]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[cmd, "x", "-y", "-o" + str(CACHE_DIR), str(CACHE_DIR / "cache.zip")],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f"Extracted with {cmd}")
|
||||
return True
|
||||
else:
|
||||
print(f" {cmd} failed: {result.stderr[:200]}")
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
|
||||
print("\nERROR: 7z not found. Install with: brew install p7zip")
|
||||
return False
|
||||
|
||||
|
||||
def convert_to_mcp_format() -> bool:
|
||||
"""Convert jlcparts cache.sqlite3 to the MCP server's expected format."""
|
||||
source = CACHE_DIR / "cache.sqlite3"
|
||||
if not source.exists():
|
||||
print("ERROR: cache.sqlite3 not found")
|
||||
return False
|
||||
|
||||
print(f"Reading source database...")
|
||||
src = sqlite3.connect(str(source))
|
||||
src.row_factory = sqlite3.Row
|
||||
|
||||
# Check schema
|
||||
tables = [
|
||||
r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
|
||||
]
|
||||
print(f" Source tables: {tables}")
|
||||
|
||||
# Find the main components table
|
||||
comp_table = None
|
||||
for t in tables:
|
||||
count = src.execute(f"SELECT COUNT(*) FROM [{t}]").fetchone()[0]
|
||||
print(f" {t}: {count:,} rows")
|
||||
if count > 10000 and comp_table is None:
|
||||
comp_table = t
|
||||
|
||||
if not comp_table:
|
||||
# Try 'components' specifically
|
||||
comp_table = "components" if "components" in tables else tables[0]
|
||||
|
||||
# Get column names
|
||||
cols = [r[1] for r in src.execute(f"PRAGMA table_info([{comp_table}])").fetchall()]
|
||||
print(f" Using table '{comp_table}' with columns: {cols[:10]}...")
|
||||
|
||||
# Remove old target DB
|
||||
if TARGET_DB.exists():
|
||||
TARGET_DB.unlink()
|
||||
|
||||
# Create target DB in MCP format
|
||||
dst = sqlite3.connect(str(TARGET_DB))
|
||||
dst.execute(
|
||||
"""
|
||||
CREATE TABLE components (
|
||||
lcsc TEXT PRIMARY KEY,
|
||||
category TEXT,
|
||||
subcategory TEXT,
|
||||
mfr_part TEXT,
|
||||
package TEXT,
|
||||
solder_joints INTEGER,
|
||||
manufacturer TEXT,
|
||||
library_type TEXT,
|
||||
description TEXT,
|
||||
datasheet TEXT,
|
||||
stock INTEGER,
|
||||
price_json TEXT,
|
||||
last_updated INTEGER
|
||||
)
|
||||
"""
|
||||
)
|
||||
dst.execute("CREATE INDEX idx_category ON components(category, subcategory)")
|
||||
dst.execute("CREATE INDEX idx_package ON components(package)")
|
||||
dst.execute("CREATE INDEX idx_manufacturer ON components(manufacturer)")
|
||||
dst.execute("CREATE INDEX idx_library_type ON components(library_type)")
|
||||
dst.execute("CREATE INDEX idx_mfr_part ON components(mfr_part)")
|
||||
|
||||
# Map source columns to our schema
|
||||
# jlcparts schema varies but commonly has:
|
||||
# lcsc, mfr, description, joint, manufacturer, basic, preferred, stock, price, url, etc.
|
||||
print(f"\nConverting parts to MCP format...")
|
||||
now = int(time.time())
|
||||
batch = []
|
||||
count = 0
|
||||
|
||||
for row in src.execute(f"SELECT * FROM [{comp_table}]"):
|
||||
row_dict = dict(row)
|
||||
|
||||
# Adapt column names (jlcparts uses various schemas)
|
||||
lcsc = row_dict.get("lcsc") or row_dict.get("LCSC_Part") or row_dict.get("lcsc_id")
|
||||
if lcsc is None:
|
||||
continue
|
||||
if isinstance(lcsc, int):
|
||||
lcsc = f"C{lcsc}"
|
||||
elif not str(lcsc).startswith("C"):
|
||||
lcsc = f"C{lcsc}"
|
||||
|
||||
mfr_part = row_dict.get("mfr") or row_dict.get("MFR_Part") or row_dict.get("mfr_part") or ""
|
||||
package = row_dict.get("package") or row_dict.get("Package") or ""
|
||||
manufacturer = row_dict.get("manufacturer") or row_dict.get("Manufacturer") or ""
|
||||
description = row_dict.get("description") or row_dict.get("Description") or ""
|
||||
stock = row_dict.get("stock") or row_dict.get("Stock") or 0
|
||||
category = row_dict.get("category") or row_dict.get("First Category") or ""
|
||||
subcategory = row_dict.get("subcategory") or row_dict.get("Second Category") or ""
|
||||
datasheet = row_dict.get("datasheet") or row_dict.get("url") or ""
|
||||
|
||||
# Library type
|
||||
is_basic = row_dict.get("basic") or row_dict.get("is_basic") or row_dict.get("Basic")
|
||||
is_preferred = (
|
||||
row_dict.get("preferred") or row_dict.get("is_preferred") or row_dict.get("Preferred")
|
||||
)
|
||||
if is_basic:
|
||||
lib_type = "Basic"
|
||||
elif is_preferred:
|
||||
lib_type = "Preferred"
|
||||
else:
|
||||
lib_type = "Extended"
|
||||
|
||||
# Price
|
||||
price = row_dict.get("price") or row_dict.get("Price") or 0
|
||||
price_json = json.dumps([{"qty": 1, "price": price}] if price else [])
|
||||
|
||||
batch.append(
|
||||
(
|
||||
str(lcsc),
|
||||
category,
|
||||
subcategory,
|
||||
mfr_part,
|
||||
package,
|
||||
0,
|
||||
manufacturer,
|
||||
lib_type,
|
||||
description,
|
||||
datasheet,
|
||||
int(stock) if stock else 0,
|
||||
price_json,
|
||||
now,
|
||||
)
|
||||
)
|
||||
|
||||
if len(batch) >= 10000:
|
||||
dst.executemany(
|
||||
"""
|
||||
INSERT OR REPLACE INTO components
|
||||
(lcsc, category, subcategory, mfr_part, package,
|
||||
solder_joints, manufacturer, library_type, description,
|
||||
datasheet, stock, price_json, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
batch,
|
||||
)
|
||||
count += len(batch)
|
||||
batch = []
|
||||
if count % 100000 == 0:
|
||||
print(f" Converted {count:,} parts...")
|
||||
|
||||
if batch:
|
||||
dst.executemany(
|
||||
"""
|
||||
INSERT OR REPLACE INTO components
|
||||
(lcsc, category, subcategory, mfr_part, package,
|
||||
solder_joints, manufacturer, library_type, description,
|
||||
datasheet, stock, price_json, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
batch,
|
||||
)
|
||||
count += len(batch)
|
||||
|
||||
# Build FTS index
|
||||
print(f" Building full-text search index...")
|
||||
dst.execute(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS components_fts USING fts5(
|
||||
lcsc, description, mfr_part, manufacturer,
|
||||
content=components
|
||||
)
|
||||
"""
|
||||
)
|
||||
dst.execute("INSERT INTO components_fts(components_fts) VALUES('rebuild')")
|
||||
dst.commit()
|
||||
|
||||
# Stats
|
||||
total = dst.execute("SELECT COUNT(*) FROM components").fetchone()[0]
|
||||
basic = dst.execute("SELECT COUNT(*) FROM components WHERE library_type='Basic'").fetchone()[0]
|
||||
extended = dst.execute(
|
||||
"SELECT COUNT(*) FROM components WHERE library_type='Extended'"
|
||||
).fetchone()[0]
|
||||
|
||||
dst.close()
|
||||
src.close()
|
||||
|
||||
print("Cleaning up source database...")
|
||||
shutil.rmtree(CACHE_DIR)
|
||||
print(f" Removed {CACHE_DIR}")
|
||||
|
||||
db_size = TARGET_DB.stat().st_size / (1024 * 1024)
|
||||
print(f"\nDatabase ready: {TARGET_DB}")
|
||||
print(f" Total parts: {total:,}")
|
||||
print(f" Basic parts: {basic:,}")
|
||||
print(f" Extended parts: {extended:,}")
|
||||
print(f" DB size: {db_size:.1f} MB")
|
||||
return True
|
||||
from commands import jlcpcb_downloader # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Download the JLCPCB parts database")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
choices=["cdfer", "yaqwsx", "official"],
|
||||
default=None,
|
||||
help="Force a single source (default: try cdfer, then yaqwsx, then official)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force", action="store_true", help="Re-download even if a database already exists"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("JLCPCB Parts Database Downloader (jlcparts method)")
|
||||
print("JLCPCB Parts Database Downloader")
|
||||
print("=" * 60)
|
||||
start = time.time()
|
||||
|
||||
if not download_files():
|
||||
sys.exit(1)
|
||||
result = jlcpcb_downloader.download_database(
|
||||
force=args.force,
|
||||
prefer_source=args.source,
|
||||
progress=lambda msg: print(f" {msg}"),
|
||||
)
|
||||
|
||||
if not extract_database():
|
||||
sys.exit(1)
|
||||
|
||||
if not convert_to_mcp_format():
|
||||
if not result.get("success"):
|
||||
print(f"\nERROR: {result.get('message', 'download failed')}")
|
||||
for err in result.get("errors", []):
|
||||
print(f" - {err}")
|
||||
sys.exit(1)
|
||||
|
||||
elapsed = time.time() - start
|
||||
print(f"\nTotal time: {elapsed/60:.1f} minutes")
|
||||
print(f"\nDatabase ready: {result['db_path']}")
|
||||
print(f" Source: {result['source']}")
|
||||
if result.get("catalog_last_modified"):
|
||||
print(f" Catalog dated: {result['catalog_last_modified']}")
|
||||
print(f" Total parts: {result['total_parts']:,}")
|
||||
print(f" Basic parts: {result['basic_parts']:,}")
|
||||
print(f" Extended parts: {result['extended_parts']:,}")
|
||||
print(f" DB size: {result['db_size_mb']} MB")
|
||||
print(f"\nTotal time: {elapsed / 60:.1f} minutes")
|
||||
print("Done! Restart the MCP server (/mcp) to use the new database.")
|
||||
|
||||
|
||||
|
||||
501
python/commands/jlcpcb_downloader.py
Normal file
501
python/commands/jlcpcb_downloader.py
Normal file
@@ -0,0 +1,501 @@
|
||||
"""
|
||||
JLCPCB parts catalog downloader (issue #199).
|
||||
|
||||
The old approach paged the community JLCSearch API with an ``offset`` parameter,
|
||||
but that endpoint is a search front-end that ignores ``offset`` and returns the
|
||||
same first 100 parts on every page, so a full catalog download was impossible.
|
||||
|
||||
This module replaces that with a layered download of a *prebuilt* catalog and
|
||||
converts it into the schema expected by ``JLCPCBPartsManager``
|
||||
(``get_data_dir()/jlcpcb_parts.db``):
|
||||
|
||||
1. CDFER (primary) -- https://github.com/CDFER/jlcpcb-parts-database
|
||||
A single, raw, uncompressed SQLite file on GitHub Pages. No 7z/zip
|
||||
needed, so this is the reliable cross-platform (incl. Windows) path.
|
||||
2. yaqwsx/jlcparts (fallback) -- https://github.com/yaqwsx/jlcparts
|
||||
The canonical upstream, published as a split 7z archive. Used only if
|
||||
CDFER fails AND a 7z CLI is available.
|
||||
3. Official JLCPCB API (optional) -- requires JLCPCB_APP_ID / JLCPCB_API_KEY /
|
||||
JLCPCB_API_SECRET. Uses cursor pagination (works correctly).
|
||||
|
||||
Data is MIT-licensed via CDFER and yaqwsx/jlcparts; the underlying catalog facts
|
||||
originate from JLCPCB / LCSC and are subject to their terms.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from utils.platform_helper import PlatformHelper
|
||||
|
||||
logger = logging.getLogger("kicad_interface")
|
||||
|
||||
CDFER_SQLITE_URL = "https://cdfer.github.io/jlcpcb-parts-database/jlcpcb-components.sqlite3"
|
||||
YAQWSX_BASE_URL = "https://yaqwsx.github.io/jlcparts/data"
|
||||
YAQWSX_MAX_VOLUMES = 30
|
||||
|
||||
ProgressFn = Optional[Callable[[str], None]]
|
||||
|
||||
|
||||
def _progress(progress: ProgressFn, message: str) -> None:
|
||||
if progress:
|
||||
try:
|
||||
progress(message)
|
||||
except Exception: # pragma: no cover - progress must never break a download
|
||||
pass
|
||||
logger.info(message)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conversion (shared by the CDFER and yaqwsx paths)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def normalize_price_json(raw: Any) -> str:
|
||||
"""Normalize a source price value into the manager's ``[{"qty","price"}]`` shape.
|
||||
|
||||
Handles both CDFER/jlcparts JSON arrays (``[{"qFrom","qTo","price"}, ...]``)
|
||||
and a bare scalar price. Returns a JSON string (``"[]"`` when no price).
|
||||
"""
|
||||
if raw is None or raw == "":
|
||||
return json.dumps([])
|
||||
|
||||
# JSON array string (CDFER / jlcparts store price as TEXT JSON)
|
||||
if isinstance(raw, str) and raw.strip().startswith("["):
|
||||
try:
|
||||
arr = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return json.dumps([])
|
||||
out: List[Dict[str, Any]] = []
|
||||
for item in arr if isinstance(arr, list) else []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
price = item.get("price")
|
||||
if price is None:
|
||||
continue
|
||||
qty = item.get("qFrom", item.get("qty", 1)) or 1
|
||||
out.append({"qty": qty, "price": price})
|
||||
return json.dumps(out)
|
||||
|
||||
# Scalar numeric price
|
||||
try:
|
||||
price_val = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
return json.dumps([])
|
||||
return json.dumps([{"qty": 1, "price": price_val}]) if price_val else json.dumps([])
|
||||
|
||||
|
||||
def _library_type(is_basic: Any, is_preferred: Any) -> str:
|
||||
if is_basic:
|
||||
return "Basic"
|
||||
if is_preferred:
|
||||
return "Preferred"
|
||||
return "Extended"
|
||||
|
||||
|
||||
def _pick_source_relation(src: sqlite3.Connection) -> str:
|
||||
"""Choose which table/view to read from the source database.
|
||||
|
||||
Prefers CDFER's denormalized ``v_components`` view (which exposes category,
|
||||
subcategory and manufacturer names directly). Falls back to the largest
|
||||
table for yaqwsx-style schemas.
|
||||
"""
|
||||
names = [
|
||||
r[0]
|
||||
for r in src.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type IN ('table','view')"
|
||||
).fetchall()
|
||||
]
|
||||
if "v_components" in names:
|
||||
return "v_components"
|
||||
if "components" in names:
|
||||
return "components"
|
||||
# Fallback: largest table
|
||||
best, best_count = None, -1
|
||||
for name in names:
|
||||
try:
|
||||
count = src.execute(f"SELECT COUNT(*) FROM [{name}]").fetchone()[0]
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
if count > best_count:
|
||||
best, best_count = name, count
|
||||
if not best:
|
||||
raise RuntimeError("No usable table/view found in source database")
|
||||
return best
|
||||
|
||||
|
||||
def convert_source_sqlite(
|
||||
source_path: Path, target_path: Path, progress: ProgressFn = None
|
||||
) -> Dict[str, int]:
|
||||
"""Convert a prebuilt source SQLite (CDFER or yaqwsx) into the MCP schema.
|
||||
|
||||
Writes a fresh ``target_path`` (deleting any existing file) with the
|
||||
``components`` table + FTS index that ``JLCPCBPartsManager`` expects.
|
||||
Returns ``{"total","basic","extended"}`` counts.
|
||||
"""
|
||||
source_path = Path(source_path)
|
||||
target_path = Path(target_path)
|
||||
if not source_path.exists():
|
||||
raise FileNotFoundError(f"source database not found: {source_path}")
|
||||
|
||||
src = sqlite3.connect(str(source_path))
|
||||
src.row_factory = sqlite3.Row
|
||||
try:
|
||||
relation = _pick_source_relation(src)
|
||||
_progress(progress, f"Converting from source relation '{relation}'...")
|
||||
|
||||
if target_path.exists():
|
||||
target_path.unlink()
|
||||
|
||||
dst = sqlite3.connect(str(target_path))
|
||||
try:
|
||||
dst.execute("""
|
||||
CREATE TABLE components (
|
||||
lcsc TEXT PRIMARY KEY,
|
||||
category TEXT,
|
||||
subcategory TEXT,
|
||||
mfr_part TEXT,
|
||||
package TEXT,
|
||||
solder_joints INTEGER,
|
||||
manufacturer TEXT,
|
||||
library_type TEXT,
|
||||
description TEXT,
|
||||
datasheet TEXT,
|
||||
stock INTEGER,
|
||||
price_json TEXT,
|
||||
last_updated INTEGER
|
||||
)
|
||||
""")
|
||||
dst.execute("CREATE INDEX idx_category ON components(category, subcategory)")
|
||||
dst.execute("CREATE INDEX idx_package ON components(package)")
|
||||
dst.execute("CREATE INDEX idx_manufacturer ON components(manufacturer)")
|
||||
dst.execute("CREATE INDEX idx_library_type ON components(library_type)")
|
||||
dst.execute("CREATE INDEX idx_mfr_part ON components(mfr_part)")
|
||||
|
||||
now = int(time.time())
|
||||
batch: List[Tuple] = []
|
||||
count = 0
|
||||
|
||||
insert_sql = """
|
||||
INSERT OR REPLACE INTO components
|
||||
(lcsc, category, subcategory, mfr_part, package, solder_joints,
|
||||
manufacturer, library_type, description, datasheet, stock,
|
||||
price_json, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
|
||||
for row in src.execute(f"SELECT * FROM [{relation}]"):
|
||||
rd = dict(row)
|
||||
|
||||
lcsc = rd.get("lcsc") or rd.get("LCSC_Part") or rd.get("lcsc_id")
|
||||
if lcsc is None:
|
||||
continue
|
||||
if isinstance(lcsc, int):
|
||||
lcsc = f"C{lcsc}"
|
||||
elif not str(lcsc).startswith("C"):
|
||||
lcsc = f"C{lcsc}"
|
||||
|
||||
mfr_part = rd.get("mfr") or rd.get("MFR_Part") or rd.get("mfr_part") or ""
|
||||
package = rd.get("package") or rd.get("Package") or ""
|
||||
manufacturer = rd.get("manufacturer") or rd.get("Manufacturer") or ""
|
||||
description = rd.get("description") or rd.get("Description") or ""
|
||||
stock = rd.get("stock") or rd.get("Stock") or 0
|
||||
category = rd.get("category") or rd.get("First Category") or ""
|
||||
subcategory = rd.get("subcategory") or rd.get("Second Category") or ""
|
||||
datasheet = rd.get("datasheet") or rd.get("url") or ""
|
||||
joints = rd.get("joints") or rd.get("solder_joints") or 0
|
||||
|
||||
lib_type = _library_type(
|
||||
rd.get("basic") or rd.get("is_basic") or rd.get("Basic"),
|
||||
rd.get("preferred") or rd.get("is_preferred") or rd.get("Preferred"),
|
||||
)
|
||||
price_json = normalize_price_json(rd.get("price") or rd.get("Price"))
|
||||
|
||||
batch.append(
|
||||
(
|
||||
str(lcsc),
|
||||
category,
|
||||
subcategory,
|
||||
mfr_part,
|
||||
package,
|
||||
int(joints) if joints else 0,
|
||||
manufacturer,
|
||||
lib_type,
|
||||
description,
|
||||
datasheet,
|
||||
int(stock) if stock else 0,
|
||||
price_json,
|
||||
now,
|
||||
)
|
||||
)
|
||||
|
||||
if len(batch) >= 10000:
|
||||
dst.executemany(insert_sql, batch)
|
||||
count += len(batch)
|
||||
batch = []
|
||||
if count % 100000 == 0:
|
||||
_progress(progress, f"Converted {count:,} parts...")
|
||||
|
||||
if batch:
|
||||
dst.executemany(insert_sql, batch)
|
||||
count += len(batch)
|
||||
|
||||
_progress(progress, "Building full-text search index...")
|
||||
dst.execute("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS components_fts USING fts5(
|
||||
lcsc, description, mfr_part, manufacturer,
|
||||
content=components
|
||||
)
|
||||
""")
|
||||
dst.execute("INSERT INTO components_fts(components_fts) VALUES('rebuild')")
|
||||
dst.commit()
|
||||
|
||||
total = dst.execute("SELECT COUNT(*) FROM components").fetchone()[0]
|
||||
basic = dst.execute(
|
||||
"SELECT COUNT(*) FROM components WHERE library_type='Basic'"
|
||||
).fetchone()[0]
|
||||
extended = dst.execute(
|
||||
"SELECT COUNT(*) FROM components WHERE library_type='Extended'"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
dst.close()
|
||||
finally:
|
||||
src.close()
|
||||
|
||||
return {"total": total, "basic": basic, "extended": extended}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source downloads
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _head_last_modified(url: str) -> Optional[str]:
|
||||
try:
|
||||
import requests
|
||||
|
||||
resp = requests.head(url, allow_redirects=True, timeout=30)
|
||||
if resp.ok:
|
||||
return resp.headers.get("Last-Modified")
|
||||
except Exception as exc: # network/dns/etc.
|
||||
logger.debug(f"HEAD {url} failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def download_cdfer(cache_dir: Path, progress: ProgressFn = None) -> Tuple[Path, Optional[str]]:
|
||||
"""Stream-download CDFER's single uncompressed SQLite. Returns (path, last_modified)."""
|
||||
import requests
|
||||
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = cache_dir / "cdfer.sqlite3"
|
||||
last_modified = _head_last_modified(CDFER_SQLITE_URL)
|
||||
_progress(
|
||||
progress,
|
||||
"Downloading CDFER prebuilt SQLite (~1.5 GB)"
|
||||
+ (f" [catalog dated {last_modified}]" if last_modified else "")
|
||||
+ "...",
|
||||
)
|
||||
|
||||
with requests.get(CDFER_SQLITE_URL, stream=True, timeout=60) as resp:
|
||||
resp.raise_for_status()
|
||||
last_modified = last_modified or resp.headers.get("Last-Modified")
|
||||
# NOTE: GitHub Pages' Content-Length can understate the real transfer
|
||||
# size (compressed transfer accounting), so we report MB downloaded
|
||||
# rather than a misleading percentage.
|
||||
written = 0
|
||||
next_mark = 50 * 1024 * 1024 # log every ~50 MB
|
||||
with open(dest, "wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
fh.write(chunk)
|
||||
written += len(chunk)
|
||||
if written >= next_mark:
|
||||
_progress(progress, f"Downloaded {written // (1024 * 1024)} MB...")
|
||||
next_mark += 50 * 1024 * 1024
|
||||
|
||||
if dest.stat().st_size < 1_000_000:
|
||||
raise RuntimeError("CDFER download too small to be valid")
|
||||
return dest, last_modified
|
||||
|
||||
|
||||
def _find_7z() -> Optional[str]:
|
||||
for cmd in ("7z", "7zz", "7za"):
|
||||
if shutil.which(cmd):
|
||||
return cmd
|
||||
return None
|
||||
|
||||
|
||||
def download_yaqwsx(cache_dir: Path, progress: ProgressFn = None) -> Path:
|
||||
"""Download + extract the yaqwsx split 7z archive. Requires a 7z CLI."""
|
||||
import subprocess
|
||||
|
||||
seven_zip = _find_7z()
|
||||
if not seven_zip:
|
||||
raise RuntimeError("yaqwsx fallback requires a 7z CLI (7z/7zz/7za) which was not found")
|
||||
if not shutil.which("curl"):
|
||||
raise RuntimeError("yaqwsx fallback requires curl which was not found")
|
||||
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
_progress(progress, "Downloading yaqwsx split archive (~421 MB)...")
|
||||
|
||||
def _curl(url: str, dst: Path) -> bool:
|
||||
return (
|
||||
subprocess.run(["curl", "-L", "-f", "-o", str(dst), "--progress-bar", url]).returncode
|
||||
== 0
|
||||
)
|
||||
|
||||
for i in range(1, YAQWSX_MAX_VOLUMES + 1):
|
||||
part = f"cache.z{i:02d}"
|
||||
dst = cache_dir / part
|
||||
if dst.exists() and dst.stat().st_size > 1000:
|
||||
continue
|
||||
if not _curl(f"{YAQWSX_BASE_URL}/{part}", dst):
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
break
|
||||
|
||||
zip_dst = cache_dir / "cache.zip"
|
||||
if not (zip_dst.exists() and zip_dst.stat().st_size > 1000):
|
||||
if not _curl(f"{YAQWSX_BASE_URL}/cache.zip", zip_dst):
|
||||
raise RuntimeError("failed to download yaqwsx cache.zip")
|
||||
|
||||
_progress(progress, f"Extracting archive with {seven_zip}...")
|
||||
result = subprocess.run(
|
||||
[seven_zip, "x", "-y", "-o" + str(cache_dir), str(zip_dst)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"7z extraction failed: {result.stderr[:300]}")
|
||||
|
||||
sqlite_path = cache_dir / "cache.sqlite3"
|
||||
if not sqlite_path.exists():
|
||||
raise RuntimeError("yaqwsx archive did not yield cache.sqlite3")
|
||||
return sqlite_path
|
||||
|
||||
|
||||
def _download_official(target: Path, force: bool, progress: ProgressFn) -> Dict[str, Any]:
|
||||
"""Optional fallback: official JLCPCB API (cursor pagination). Requires creds."""
|
||||
from commands.jlcpcb import JLCPCBClient
|
||||
from commands.jlcpcb_parts import JLCPCBPartsManager
|
||||
|
||||
client = JLCPCBClient()
|
||||
if not (client.app_id and client.access_key and client.secret_key):
|
||||
raise RuntimeError("official API credentials not set")
|
||||
|
||||
_progress(progress, "Downloading via official JLCPCB API...")
|
||||
parts = client.download_full_database(
|
||||
callback=lambda page, total, msg: _progress(progress, msg)
|
||||
)
|
||||
if force and target.exists():
|
||||
target.unlink()
|
||||
|
||||
mgr = JLCPCBPartsManager(db_path=str(target))
|
||||
try:
|
||||
mgr.import_parts(parts, progress_callback=lambda c, t, m: _progress(progress, m))
|
||||
stats = mgr.get_database_stats()
|
||||
finally:
|
||||
mgr.close()
|
||||
return _result(
|
||||
"official-api",
|
||||
target,
|
||||
stats["total_parts"],
|
||||
stats["basic_parts"],
|
||||
stats["extended_parts"],
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _result(
|
||||
source: str,
|
||||
target: Path,
|
||||
total: int,
|
||||
basic: int,
|
||||
extended: int,
|
||||
last_modified: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
db_size_mb = round(target.stat().st_size / (1024 * 1024), 2) if target.exists() else 0
|
||||
return {
|
||||
"success": True,
|
||||
"source": source,
|
||||
"total_parts": total,
|
||||
"basic_parts": basic,
|
||||
"extended_parts": extended,
|
||||
"db_size_mb": db_size_mb,
|
||||
"db_path": str(target),
|
||||
"catalog_last_modified": last_modified,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def download_database(
|
||||
force: bool = False,
|
||||
prefer_source: Optional[str] = None,
|
||||
progress: ProgressFn = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Download the JLCPCB catalog into ``jlcpcb_parts.db`` using a layered strategy.
|
||||
|
||||
Order: CDFER (primary) -> yaqwsx (if 7z available) -> official API (if creds).
|
||||
``prefer_source`` (``"cdfer"``/``"yaqwsx"``/``"official"``) forces a single source.
|
||||
Returns a result dict (see ``_result``) or ``{"success": False, "message": ...}``.
|
||||
|
||||
NOTE: callers that hold an open ``JLCPCBPartsManager`` on the target file must
|
||||
close it before calling (the prebuilt paths recreate the file) and reopen after.
|
||||
"""
|
||||
data_dir = PlatformHelper.get_data_dir()
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = data_dir / "jlcpcb_parts.db"
|
||||
cache_dir = data_dir / "jlcparts_cache"
|
||||
|
||||
order = [prefer_source] if prefer_source else ["cdfer", "yaqwsx", "official"]
|
||||
errors: List[str] = []
|
||||
|
||||
for src in order:
|
||||
try:
|
||||
if src == "cdfer":
|
||||
src_path, last_mod = download_cdfer(cache_dir, progress)
|
||||
stats = convert_source_sqlite(src_path, target, progress)
|
||||
_safe_rmtree(cache_dir)
|
||||
return _result(
|
||||
"cdfer", target, stats["total"], stats["basic"], stats["extended"], last_mod
|
||||
)
|
||||
if src == "yaqwsx":
|
||||
src_path = download_yaqwsx(cache_dir, progress)
|
||||
stats = convert_source_sqlite(src_path, target, progress)
|
||||
_safe_rmtree(cache_dir)
|
||||
return _result(
|
||||
"yaqwsx", target, stats["total"], stats["basic"], stats["extended"], None
|
||||
)
|
||||
if src == "official":
|
||||
return _download_official(target, force, progress)
|
||||
errors.append(f"{src}: unknown source")
|
||||
except Exception as exc:
|
||||
logger.warning(f"JLCPCB source '{src}' failed: {exc}")
|
||||
errors.append(f"{src}: {exc}")
|
||||
continue
|
||||
|
||||
return {
|
||||
"success": False,
|
||||
"message": "All JLCPCB download sources failed. "
|
||||
"Install a 7z CLI for the yaqwsx fallback, or set JLCPCB_APP_ID/"
|
||||
"JLCPCB_API_KEY/JLCPCB_API_SECRET for the official API.",
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def _safe_rmtree(path: Path) -> None:
|
||||
try:
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
except OSError as exc: # pragma: no cover - cleanup is best-effort
|
||||
logger.debug(f"cache cleanup failed for {path}: {exc}")
|
||||
@@ -137,59 +137,12 @@ class JLCSearchClient:
|
||||
logger.error(f"Failed to get part C{lcsc_number}: {e}")
|
||||
return None
|
||||
|
||||
def download_all_components(
|
||||
self, callback: Optional[Callable[[int, str], None]] = None, batch_size: int = 100
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Download all components from jlcsearch database
|
||||
|
||||
Note: tscircuit API has a hard-coded 100 result limit per request.
|
||||
Full catalog download requires ~25,000 paginated requests (~40-60 minutes).
|
||||
|
||||
Args:
|
||||
callback: Optional progress callback function(parts_count, status_msg)
|
||||
batch_size: Number of parts per batch (max 100 due to API limit)
|
||||
|
||||
Returns:
|
||||
List of all parts
|
||||
"""
|
||||
all_parts = []
|
||||
offset = 0
|
||||
|
||||
logger.info("Starting full jlcsearch parts database download...")
|
||||
|
||||
while True:
|
||||
try:
|
||||
batch = self.search_components("components", limit=batch_size, offset=offset)
|
||||
|
||||
# Stop if no results returned (end of catalog)
|
||||
if not batch or len(batch) == 0:
|
||||
break
|
||||
|
||||
all_parts.extend(batch)
|
||||
offset += len(batch)
|
||||
|
||||
if callback:
|
||||
callback(len(all_parts), f"Downloaded {len(all_parts)} parts...")
|
||||
else:
|
||||
logger.info(f"Downloaded {len(all_parts)} parts so far...")
|
||||
|
||||
# Continue pagination - API returns exactly 100 results per page until exhausted
|
||||
# Only stop when we get 0 results (handled above)
|
||||
|
||||
# Rate limiting - be nice to the API
|
||||
time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading parts at offset {offset}: {e}")
|
||||
if len(all_parts) > 0:
|
||||
logger.warning(f"Partial download available: {len(all_parts)} parts")
|
||||
return all_parts
|
||||
else:
|
||||
raise
|
||||
|
||||
logger.info(f"Download complete: {len(all_parts)} parts retrieved")
|
||||
return all_parts
|
||||
# NOTE: bulk catalog download was removed (issue #199). The JLCSearch
|
||||
# endpoint is a *search front-end* that ignores the ``offset`` parameter,
|
||||
# so offset-paged "download everything" loops returned the same first 100
|
||||
# parts forever. Full-catalog download now uses a prebuilt source via
|
||||
# ``commands.jlcpcb_downloader.download_database()``. This client remains
|
||||
# for interactive/parametric lookups only (search_components, etc.).
|
||||
|
||||
|
||||
def test_jlcsearch_connection() -> bool:
|
||||
|
||||
@@ -6262,12 +6262,18 @@ print("ok")
|
||||
# JLCPCB API handlers
|
||||
|
||||
def _handle_download_jlcpcb_database(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Download JLCPCB parts database from JLCSearch API"""
|
||||
"""Download the JLCPCB parts catalog from a prebuilt source (issue #199).
|
||||
|
||||
Layered strategy (see commands.jlcpcb_downloader): CDFER single-file
|
||||
SQLite (primary, no 7z) -> yaqwsx split-7z (fallback) -> official JLCPCB
|
||||
API (optional, if credentials set). Replaces the broken JLCSearch
|
||||
offset-pagination download.
|
||||
"""
|
||||
from commands import jlcpcb_downloader
|
||||
|
||||
try:
|
||||
force = params.get("force", False)
|
||||
|
||||
# Check if database exists
|
||||
import os
|
||||
prefer_source = params.get("source") # optional: cdfer|yaqwsx|official
|
||||
|
||||
stats = self.jlcpcb_parts.get_database_stats()
|
||||
if stats["total_parts"] > 0 and not force:
|
||||
@@ -6277,36 +6283,38 @@ print("ok")
|
||||
"stats": stats,
|
||||
}
|
||||
|
||||
logger.info("Downloading JLCPCB parts database from JLCSearch...")
|
||||
# The prebuilt paths recreate jlcpcb_parts.db on disk, so the open
|
||||
# manager connection must be released first (Windows file locking),
|
||||
# then reopened on the freshly written database.
|
||||
self.jlcpcb_parts.close()
|
||||
|
||||
# Download parts from JLCSearch public API (no auth required)
|
||||
parts = self.jlcsearch_client.download_all_components(
|
||||
callback=lambda total, msg: logger.info(f"{msg}")
|
||||
result = jlcpcb_downloader.download_database(
|
||||
force=force,
|
||||
prefer_source=prefer_source,
|
||||
progress=lambda msg: logger.info(msg),
|
||||
)
|
||||
|
||||
# Import into database
|
||||
logger.info(f"Importing {len(parts)} parts into database...")
|
||||
self.jlcpcb_parts.import_jlcsearch_parts(
|
||||
parts, progress_callback=lambda curr, total, msg: logger.info(msg)
|
||||
)
|
||||
# Reopen the manager on the new database regardless of outcome.
|
||||
self.jlcpcb_parts = JLCPCBPartsManager()
|
||||
|
||||
# Get final stats
|
||||
if not result.get("success"):
|
||||
return result
|
||||
|
||||
# Refresh counts from the reopened manager (authoritative).
|
||||
stats = self.jlcpcb_parts.get_database_stats()
|
||||
|
||||
# Calculate database size
|
||||
db_size_mb = os.path.getsize(self.jlcpcb_parts.db_path) / (1024 * 1024)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"total_parts": stats["total_parts"],
|
||||
"basic_parts": stats["basic_parts"],
|
||||
"extended_parts": stats["extended_parts"],
|
||||
"db_size_mb": round(db_size_mb, 2),
|
||||
"db_path": stats["db_path"],
|
||||
}
|
||||
result["total_parts"] = stats["total_parts"]
|
||||
result["basic_parts"] = stats["basic_parts"]
|
||||
result["extended_parts"] = stats["extended_parts"]
|
||||
result["db_path"] = stats["db_path"]
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading JLCPCB database: {e}", exc_info=True)
|
||||
# Best-effort: ensure the manager is usable after a failure.
|
||||
try:
|
||||
self.jlcpcb_parts = JLCPCBPartsManager()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"Failed to download database: {str(e)}",
|
||||
|
||||
@@ -10,21 +10,27 @@ export function registerJLCPCBApiTools(server: McpServer, callKicadScript: Funct
|
||||
// Download JLCPCB parts database
|
||||
server.tool(
|
||||
"download_jlcpcb_database",
|
||||
`Download the complete JLCPCB parts catalog to local database.
|
||||
`Download the JLCPCB parts catalog to a local SQLite database for fast offline search.
|
||||
|
||||
This is a one-time setup that downloads ~2.5M+ parts from JLCSearch API.
|
||||
No API credentials required - uses public JLCSearch API.
|
||||
Uses a prebuilt catalog (no API credentials required by default):
|
||||
- Primary: CDFER prebuilt SQLite (single file, no extra tools needed)
|
||||
- Fallback: yaqwsx/jlcparts split archive (requires a 7z CLI)
|
||||
- Optional: official JLCPCB API if JLCPCB_APP_ID/JLCPCB_API_KEY/JLCPCB_API_SECRET are set
|
||||
|
||||
The download takes 5-10 minutes and creates a local SQLite database
|
||||
for fast offline searching.`,
|
||||
This is a one-time setup (downloads ~1.5 GB). Optionally pass source to force one
|
||||
provider. Re-run with force=true to refresh.`,
|
||||
{
|
||||
force: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(false)
|
||||
.describe("Force re-download even if database exists"),
|
||||
source: z
|
||||
.enum(["cdfer", "yaqwsx", "official"])
|
||||
.optional()
|
||||
.describe("Force a single source instead of the default cdfer→yaqwsx→official order"),
|
||||
},
|
||||
async (args: { force?: boolean }) => {
|
||||
async (args: { force?: boolean; source?: "cdfer" | "yaqwsx" | "official" }) => {
|
||||
const result = await callKicadScript("download_jlcpcb_database", args);
|
||||
if (result.success) {
|
||||
return {
|
||||
@@ -33,6 +39,10 @@ for fast offline searching.`,
|
||||
type: "text",
|
||||
text:
|
||||
`✓ Successfully downloaded JLCPCB parts database\n\n` +
|
||||
`Source: ${result.source}\n` +
|
||||
(result.catalog_last_modified
|
||||
? `Catalog dated: ${result.catalog_last_modified}\n`
|
||||
: "") +
|
||||
`Total parts: ${result.total_parts}\n` +
|
||||
`Basic parts: ${result.basic_parts}\n` +
|
||||
`Extended parts: ${result.extended_parts}\n` +
|
||||
@@ -46,9 +56,7 @@ for fast offline searching.`,
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`✗ Failed to download JLCPCB database: ${result.message || "Unknown error"}\n\n` +
|
||||
`Make sure JLCPCB_API_KEY and JLCPCB_API_SECRET environment variables are set.`,
|
||||
text: `✗ Failed to download JLCPCB database: ${result.message || "Unknown error"}`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
172
tests/test_jlcpcb_downloader.py
Normal file
172
tests/test_jlcpcb_downloader.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
Regression tests for the JLCPCB prebuilt-catalog downloader (issue #199).
|
||||
|
||||
These tests do NOT hit the network. They build a small SQLite source that
|
||||
mimics CDFER's schema (incl. the v_components view and bare-integer lcsc) and
|
||||
verify that convert_source_sqlite() produces a database in the schema that
|
||||
JLCPCBPartsManager consumes, plus that download_database() falls through
|
||||
sources gracefully when none are usable.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Match the import root used elsewhere in the test suite (python/ on sys.path).
|
||||
PYTHON_DIR = Path(__file__).resolve().parent.parent / "python"
|
||||
if str(PYTHON_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(PYTHON_DIR))
|
||||
|
||||
from commands import jlcpcb_downloader # noqa: E402
|
||||
|
||||
|
||||
def _make_cdfer_like_source(path: Path) -> None:
|
||||
"""Create a tiny SQLite mirroring CDFER: components + categories + manufacturers + view."""
|
||||
con = sqlite3.connect(str(path))
|
||||
con.executescript("""
|
||||
CREATE TABLE manufacturers (id INTEGER PRIMARY KEY, name TEXT);
|
||||
CREATE TABLE categories (id INTEGER PRIMARY KEY, category TEXT, subcategory TEXT);
|
||||
CREATE TABLE components (
|
||||
lcsc INTEGER PRIMARY KEY,
|
||||
category_id INTEGER,
|
||||
mfr TEXT,
|
||||
package TEXT,
|
||||
joints INTEGER,
|
||||
manufacturer_id INTEGER,
|
||||
basic INTEGER,
|
||||
preferred INTEGER,
|
||||
description TEXT,
|
||||
datasheet TEXT,
|
||||
stock INTEGER,
|
||||
price TEXT
|
||||
);
|
||||
CREATE VIEW v_components AS
|
||||
SELECT c.lcsc, c.category_id, cat.category, cat.subcategory, c.mfr,
|
||||
c.package, c.joints, m.name AS manufacturer, c.basic, c.preferred,
|
||||
c.description, c.datasheet, c.stock, c.price
|
||||
FROM components c
|
||||
JOIN categories cat ON cat.id = c.category_id
|
||||
JOIN manufacturers m ON m.id = c.manufacturer_id;
|
||||
""")
|
||||
con.execute("INSERT INTO manufacturers VALUES (1, 'Yageo')")
|
||||
con.execute("INSERT INTO categories VALUES (1, 'Resistors', 'Chip Resistor')")
|
||||
# basic part, with CDFER-style price JSON (qFrom/qTo/price)
|
||||
con.execute(
|
||||
"INSERT INTO components VALUES (25804, 1, 'RC0603FR-071KL', '0603', 2, 1, 1, 0, "
|
||||
"'1k 1% 0603 resistor', 'http://ds/25804', 5000, ?)",
|
||||
(
|
||||
json.dumps(
|
||||
[
|
||||
{"qFrom": 1, "qTo": 99, "price": 0.0042},
|
||||
{"qFrom": 100, "qTo": None, "price": 0.0021},
|
||||
]
|
||||
),
|
||||
),
|
||||
)
|
||||
# extended part, empty price
|
||||
con.execute(
|
||||
"INSERT INTO components VALUES (11111, 1, 'EXT-PART', '0402', 2, 1, 0, 0, "
|
||||
"'extended resistor', '', 12, '[]')"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def test_convert_source_sqlite_produces_manager_schema(tmp_path):
|
||||
source = tmp_path / "cdfer.sqlite3"
|
||||
target = tmp_path / "jlcpcb_parts.db"
|
||||
_make_cdfer_like_source(source)
|
||||
|
||||
stats = jlcpcb_downloader.convert_source_sqlite(source, target)
|
||||
|
||||
assert stats["total"] == 2
|
||||
assert stats["basic"] == 1
|
||||
assert stats["extended"] == 1
|
||||
|
||||
con = sqlite3.connect(str(target))
|
||||
con.row_factory = sqlite3.Row
|
||||
rows = {r["lcsc"]: dict(r) for r in con.execute("SELECT * FROM components")}
|
||||
|
||||
# bare-int lcsc must become C-prefixed
|
||||
assert "C25804" in rows and "C11111" in rows
|
||||
|
||||
basic = rows["C25804"]
|
||||
assert basic["library_type"] == "Basic"
|
||||
assert basic["mfr_part"] == "RC0603FR-071KL" # mfr -> mfr_part
|
||||
assert basic["manufacturer"] == "Yageo" # via v_components view
|
||||
assert basic["category"] == "Resistors"
|
||||
assert basic["subcategory"] == "Chip Resistor"
|
||||
assert basic["stock"] == 5000
|
||||
|
||||
# CDFER price JSON normalized to [{"qty","price"}]
|
||||
breaks = json.loads(basic["price_json"])
|
||||
assert breaks[0] == {"qty": 1, "price": 0.0042}
|
||||
assert breaks[1] == {"qty": 100, "price": 0.0021}
|
||||
|
||||
assert rows["C11111"]["library_type"] == "Extended"
|
||||
assert json.loads(rows["C11111"]["price_json"]) == []
|
||||
|
||||
# FTS index exists and is queryable
|
||||
fts_hit = con.execute(
|
||||
"SELECT lcsc FROM components_fts WHERE components_fts MATCH '1k*'"
|
||||
).fetchall()
|
||||
assert any(r[0] == "C25804" for r in fts_hit)
|
||||
con.close()
|
||||
|
||||
|
||||
def test_normalize_price_json_handles_scalar_and_array_and_empty():
|
||||
assert json.loads(jlcpcb_downloader.normalize_price_json(None)) == []
|
||||
assert json.loads(jlcpcb_downloader.normalize_price_json("")) == []
|
||||
assert json.loads(jlcpcb_downloader.normalize_price_json("[]")) == []
|
||||
assert json.loads(jlcpcb_downloader.normalize_price_json(0.05)) == [{"qty": 1, "price": 0.05}]
|
||||
arr = json.loads(jlcpcb_downloader.normalize_price_json('[{"qFrom":10,"qTo":50,"price":1.5}]'))
|
||||
assert arr == [{"qty": 10, "price": 1.5}]
|
||||
|
||||
|
||||
def test_download_database_falls_through_when_no_source_available(tmp_path, monkeypatch):
|
||||
"""With CDFER download failing, no 7z present, and no API creds -> clean failure dict."""
|
||||
monkeypatch.setattr(
|
||||
jlcpcb_downloader.PlatformHelper, "get_data_dir", staticmethod(lambda: tmp_path)
|
||||
)
|
||||
|
||||
def _boom(*a, **k):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
monkeypatch.setattr(jlcpcb_downloader, "download_cdfer", _boom)
|
||||
monkeypatch.setattr(jlcpcb_downloader, "_find_7z", lambda: None)
|
||||
# Ensure no official creds leak in from the environment.
|
||||
for var in ("JLCPCB_APP_ID", "JLCPCB_API_KEY", "JLCPCB_API_SECRET"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
result = jlcpcb_downloader.download_database()
|
||||
|
||||
assert result["success"] is False
|
||||
assert "errors" in result and len(result["errors"]) >= 1
|
||||
assert any("cdfer" in e for e in result["errors"])
|
||||
|
||||
|
||||
def test_download_database_prefer_cdfer_converts(tmp_path, monkeypatch):
|
||||
"""prefer_source='cdfer' downloads then converts via the real conversion path."""
|
||||
monkeypatch.setattr(
|
||||
jlcpcb_downloader.PlatformHelper, "get_data_dir", staticmethod(lambda: tmp_path)
|
||||
)
|
||||
|
||||
def _fake_cdfer(cache_dir, progress=None):
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
src = cache_dir / "cdfer.sqlite3"
|
||||
_make_cdfer_like_source(src)
|
||||
return src, "Thu, 02 Apr 2026 06:22:46 GMT"
|
||||
|
||||
monkeypatch.setattr(jlcpcb_downloader, "download_cdfer", _fake_cdfer)
|
||||
|
||||
result = jlcpcb_downloader.download_database(prefer_source="cdfer")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "cdfer"
|
||||
assert result["total_parts"] == 2
|
||||
assert result["basic_parts"] == 1
|
||||
assert result["catalog_last_modified"].startswith("Thu, 02 Apr 2026")
|
||||
assert (tmp_path / "jlcpcb_parts.db").exists()
|
||||
Reference in New Issue
Block a user