Store reusable scraper capabilities in scrape DB

This commit is contained in:
2026-08-06 01:01:55 +02:00
parent b11ada9c91
commit 9e3b4de4e2
7 changed files with 411 additions and 12 deletions
+25
View File
@@ -28,3 +28,28 @@ test("local scrape schema can learn picture target rules from manual corrections
assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
});
test("local scrape schema stores reusable source capabilities for pictures texts and features", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_source_capabilities`/);
assert.match(schema, /`capability` ENUM\('search', 'pictures', 'texts', 'features'\) NOT NULL/);
assert.match(schema, /`parser_key` VARCHAR\(128\) NOT NULL/);
assert.match(schema, /`selector_config` JSON NULL/);
assert.match(schema, /`extraction_config` JSON NULL/);
assert.match(schema, /`quality_config` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_source_capability_parser`/);
});
test("local scrape schema links manufacturer mappings to selected source capabilities", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities`/);
assert.match(schema, /`target_strategy_override` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\) NULL/);
assert.match(schema, /`config_override` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_manufacturer_source_capability`/);
});
test("local scrape schema seeds Hudy and Idealo extraction capabilities", () => {
assert.match(schema, /'hudy-gallery'/);
assert.match(schema, /'hudy-product-texts'/);
assert.match(schema, /'hudy-product-params'/);
assert.match(schema, /'idealo-splide-gallery'/);
assert.match(schema, /'productGalleryOnly'/);
});
+3
View File
@@ -59,6 +59,9 @@ test("mapping source queries are read-only", () => {
assert.doesNotThrow(() =>
assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })),
);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_manufacturer_sources/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_source_capabilities/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /capabilities/);
});
test("manufacturer refresh updates are narrowly allowlisted", () => {