feat(jlcpcb): make the FULL ~10GB catalog reachable + resumable downloads (#199)

People want to be able to pull the whole catalog, and prior downloads stalled or
repeated the same parts. Two fixes:

- Full catalog correctness: yaqwsx's cache.sqlite3 (the full ~10GB set) stores
  category/manufacturer as IDs with no v_components view, so the convert left
  those fields blank. Build an equivalent v_components join for yaqwsx-style
  sources so the full catalog converts with category/subcategory/manufacturer
  populated. Clarify in the tool schema that source="yaqwsx" = full catalog
  (needs 7z) vs cdfer = in-stock subset.
- Resumable downloads: CDFER stream download now uses a read timeout and resumes
  from the partial file via HTTP Range on interruption (up to 5 retries); the
  yaqwsx curl calls use -C - / --retry. Addresses the stall/partial-download
  complaints.

Adds a yaqwsx-schema conversion test.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
mixelpixx
2026-05-24 13:57:07 -04:00
parent cb5f744754
commit 030d008843
3 changed files with 191 additions and 36 deletions

View File

@@ -117,6 +117,65 @@ def test_convert_source_sqlite_produces_manager_schema(tmp_path):
con.close()
def _make_yaqwsx_like_source(path: Path) -> None:
"""Create a SQLite mirroring yaqwsx's FULL-catalog schema: normalized, NO view.
category/manufacturer are IDs in sibling tables (no v_components view), which
is what the real ~10GB cache.sqlite3 looks like.
"""
con = sqlite3.connect(str(path))
con.executescript("""
CREATE TABLE manufacturers (id INTEGER PRIMARY KEY, name TEXT);
CREATE TABLE categories (id INTEGER PRIMARY KEY, category TEXT, subcategory TEXT);
CREATE TABLE components (
lcsc INTEGER PRIMARY KEY,
category_id INTEGER,
mfr TEXT,
package TEXT,
joints INTEGER,
manufacturer_id INTEGER,
basic INTEGER,
preferred INTEGER,
description TEXT,
datasheet TEXT,
stock INTEGER,
price TEXT
);
""")
con.execute("INSERT INTO manufacturers VALUES (7, 'Texas Instruments')")
con.execute("INSERT INTO categories VALUES (3, 'ICs', 'LDO Regulators')")
con.execute(
"INSERT INTO components VALUES (12345, 3, 'TLV70033', 'SOT-23-5', 5, 7, 0, 0, "
"'3.3V LDO', 'http://ds/12345', 0, ?)",
(json.dumps([{"qFrom": 1, "qTo": None, "price": 0.12}]),),
)
con.commit()
con.close()
def test_convert_yaqwsx_schema_populates_category_and_manufacturer(tmp_path):
"""Full-catalog (yaqwsx) source has no v_components view; the join must be built."""
source = tmp_path / "cache.sqlite3"
target = tmp_path / "jlcpcb_parts.db"
_make_yaqwsx_like_source(source)
stats = jlcpcb_downloader.convert_source_sqlite(source, target)
assert stats["total"] == 1
con = sqlite3.connect(str(target))
con.row_factory = sqlite3.Row
row = dict(con.execute("SELECT * FROM components WHERE lcsc='C12345'").fetchone())
con.close()
# These would be blank without the built v_components join:
assert row["category"] == "ICs"
assert row["subcategory"] == "LDO Regulators"
assert row["manufacturer"] == "Texas Instruments"
assert row["mfr_part"] == "TLV70033"
assert row["library_type"] == "Extended"
assert json.loads(row["price_json"]) == [{"qty": 1, "price": 0.12}]
def test_normalize_price_json_handles_scalar_and_array_and_empty():
assert json.loads(jlcpcb_downloader.normalize_price_json(None)) == []
assert json.loads(jlcpcb_downloader.normalize_price_json("")) == []