Store reusable scraper capabilities in scrape DB
This commit is contained in:
@@ -28,3 +28,28 @@ test("local scrape schema can learn picture target rules from manual corrections
|
||||
assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
|
||||
assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
|
||||
});
|
||||
|
||||
test("local scrape schema stores reusable source capabilities for pictures texts and features", () => {
|
||||
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_source_capabilities`/);
|
||||
assert.match(schema, /`capability` ENUM\('search', 'pictures', 'texts', 'features'\) NOT NULL/);
|
||||
assert.match(schema, /`parser_key` VARCHAR\(128\) NOT NULL/);
|
||||
assert.match(schema, /`selector_config` JSON NULL/);
|
||||
assert.match(schema, /`extraction_config` JSON NULL/);
|
||||
assert.match(schema, /`quality_config` JSON NULL/);
|
||||
assert.match(schema, /UNIQUE KEY `uq_scrape_source_capability_parser`/);
|
||||
});
|
||||
|
||||
test("local scrape schema links manufacturer mappings to selected source capabilities", () => {
|
||||
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities`/);
|
||||
assert.match(schema, /`target_strategy_override` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\) NULL/);
|
||||
assert.match(schema, /`config_override` JSON NULL/);
|
||||
assert.match(schema, /UNIQUE KEY `uq_scrape_manufacturer_source_capability`/);
|
||||
});
|
||||
|
||||
test("local scrape schema seeds Hudy and Idealo extraction capabilities", () => {
|
||||
assert.match(schema, /'hudy-gallery'/);
|
||||
assert.match(schema, /'hudy-product-texts'/);
|
||||
assert.match(schema, /'hudy-product-params'/);
|
||||
assert.match(schema, /'idealo-splide-gallery'/);
|
||||
assert.match(schema, /'productGalleryOnly'/);
|
||||
});
|
||||
|
||||
@@ -59,6 +59,9 @@ test("mapping source queries are read-only", () => {
|
||||
assert.doesNotThrow(() =>
|
||||
assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })),
|
||||
);
|
||||
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_manufacturer_sources/);
|
||||
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_source_capabilities/);
|
||||
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /capabilities/);
|
||||
});
|
||||
|
||||
test("manufacturer refresh updates are narrowly allowlisted", () => {
|
||||
|
||||
Reference in New Issue
Block a user