Store reusable scraper capabilities in scrape DB
This commit is contained in:
@@ -83,6 +83,51 @@ CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
|
||||
CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_source_capabilities` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`source_id` BIGINT UNSIGNED NOT NULL,
|
||||
`capability` ENUM('search', 'pictures', 'texts', 'features') NOT NULL,
|
||||
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
||||
`priority` INT NOT NULL DEFAULT 100,
|
||||
`parser_key` VARCHAR(128) NOT NULL,
|
||||
`parser_module` VARCHAR(255) NULL,
|
||||
`browser_engine` VARCHAR(32) NULL,
|
||||
`browser_headless` TINYINT(1) NULL,
|
||||
`wait_after_load_ms` INT UNSIGNED NULL,
|
||||
`min_width` INT UNSIGNED NULL,
|
||||
`min_height` INT UNSIGNED NULL,
|
||||
`min_longest_side` INT UNSIGNED NULL,
|
||||
`default_target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
|
||||
`selector_config` JSON NULL,
|
||||
`extraction_config` JSON NULL,
|
||||
`quality_config` JSON NULL,
|
||||
`note` TEXT NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uq_scrape_source_capability_parser` (`source_id`, `capability`, `parser_key`),
|
||||
KEY `idx_scrape_source_capabilities_lookup` (`source_id`, `capability`, `enabled`, `priority`),
|
||||
CONSTRAINT `fk_scrape_source_capabilities_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`manufacturer_source_id` BIGINT UNSIGNED NOT NULL,
|
||||
`source_capability_id` BIGINT UNSIGNED NOT NULL,
|
||||
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
||||
`priority` INT NOT NULL DEFAULT 100,
|
||||
`target_strategy_override` ENUM('unknown', 'color', 'size', 'combination', 'manual') NULL,
|
||||
`config_override` JSON NULL,
|
||||
`note` TEXT NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uq_scrape_manufacturer_source_capability` (`manufacturer_source_id`, `source_capability_id`),
|
||||
KEY `idx_scrape_manufacturer_source_capabilities_capability` (`source_capability_id`),
|
||||
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_mapping` FOREIGN KEY (`manufacturer_source_id`) REFERENCES `scrape_manufacturer_sources` (`id`),
|
||||
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_capability` FOREIGN KEY (`source_capability_id`) REFERENCES `scrape_source_capabilities` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_products` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`manufacturer_id` INT UNSIGNED NULL,
|
||||
@@ -218,3 +263,213 @@ CREATE TABLE IF NOT EXISTS `scrape_runs` (
|
||||
KEY `idx_scrape_runs_source` (`source_id`),
|
||||
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
INSERT INTO `scrape_source_capabilities` (
|
||||
`source_id`,
|
||||
`capability`,
|
||||
`enabled`,
|
||||
`priority`,
|
||||
`parser_key`,
|
||||
`parser_module`,
|
||||
`browser_engine`,
|
||||
`browser_headless`,
|
||||
`wait_after_load_ms`,
|
||||
`min_longest_side`,
|
||||
`default_target_strategy`,
|
||||
`selector_config`,
|
||||
`extraction_config`,
|
||||
`quality_config`,
|
||||
`note`
|
||||
)
|
||||
SELECT
|
||||
`id`,
|
||||
'pictures',
|
||||
1,
|
||||
10,
|
||||
'hudy-gallery',
|
||||
'hudy.picture-gallery',
|
||||
NULL,
|
||||
NULL,
|
||||
NULL,
|
||||
800,
|
||||
'color',
|
||||
JSON_OBJECT(
|
||||
'colorSwitcherSelector', 'a[id^="product-change-color-"]',
|
||||
'eanTableSelector', 'table.product-params__params-table',
|
||||
'topImageSelector', 'p#snippet-productInfo-image img.product-top__img',
|
||||
'gallerySelector', 'p#snippet-productInfo-image, .pdbox__window'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'verifyVariantBy', JSON_ARRAY('ean', 'color'),
|
||||
'pictureSource', 'largeGallery',
|
||||
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
|
||||
),
|
||||
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
|
||||
'Hudy product gallery pictures. Verify the color variant by EAN before extracting large gallery images.'
|
||||
FROM `scrape_sources`
|
||||
WHERE `source_key` = 'hudy'
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`enabled` = VALUES(`enabled`),
|
||||
`priority` = VALUES(`priority`),
|
||||
`parser_module` = VALUES(`parser_module`),
|
||||
`min_longest_side` = VALUES(`min_longest_side`),
|
||||
`default_target_strategy` = VALUES(`default_target_strategy`),
|
||||
`selector_config` = VALUES(`selector_config`),
|
||||
`extraction_config` = VALUES(`extraction_config`),
|
||||
`quality_config` = VALUES(`quality_config`),
|
||||
`note` = VALUES(`note`);
|
||||
|
||||
INSERT INTO `scrape_source_capabilities` (
|
||||
`source_id`,
|
||||
`capability`,
|
||||
`enabled`,
|
||||
`priority`,
|
||||
`parser_key`,
|
||||
`parser_module`,
|
||||
`browser_engine`,
|
||||
`browser_headless`,
|
||||
`wait_after_load_ms`,
|
||||
`default_target_strategy`,
|
||||
`selector_config`,
|
||||
`extraction_config`,
|
||||
`note`
|
||||
)
|
||||
SELECT
|
||||
`id`,
|
||||
'texts',
|
||||
1,
|
||||
30,
|
||||
'hudy-product-texts',
|
||||
'hudy.product-texts',
|
||||
NULL,
|
||||
NULL,
|
||||
NULL,
|
||||
'manual',
|
||||
JSON_OBJECT('descriptionSelector', '.product-detail, .product-top, [itemprop="description"]'),
|
||||
JSON_OBJECT('fields', JSON_ARRAY('shortDescription', 'description', 'marketingText')),
|
||||
'Hudy text extraction placeholder. User will teach exact text fields before automation is trusted.'
|
||||
FROM `scrape_sources`
|
||||
WHERE `source_key` = 'hudy'
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`enabled` = VALUES(`enabled`),
|
||||
`priority` = VALUES(`priority`),
|
||||
`parser_module` = VALUES(`parser_module`),
|
||||
`selector_config` = VALUES(`selector_config`),
|
||||
`extraction_config` = VALUES(`extraction_config`),
|
||||
`note` = VALUES(`note`);
|
||||
|
||||
INSERT INTO `scrape_source_capabilities` (
|
||||
`source_id`,
|
||||
`capability`,
|
||||
`enabled`,
|
||||
`priority`,
|
||||
`parser_key`,
|
||||
`parser_module`,
|
||||
`browser_engine`,
|
||||
`browser_headless`,
|
||||
`wait_after_load_ms`,
|
||||
`default_target_strategy`,
|
||||
`selector_config`,
|
||||
`extraction_config`,
|
||||
`note`
|
||||
)
|
||||
SELECT
|
||||
`id`,
|
||||
'features',
|
||||
1,
|
||||
20,
|
||||
'hudy-product-params',
|
||||
'hudy.product-params',
|
||||
NULL,
|
||||
NULL,
|
||||
NULL,
|
||||
'manual',
|
||||
JSON_OBJECT('paramsTableSelector', 'table.product-params__params-table'),
|
||||
JSON_OBJECT('fields', JSON_ARRAY('label', 'value'), 'mergeStrategy', 'review-first'),
|
||||
'Hudy feature extraction placeholder. Product parameters are kept reviewable before catalog writes.'
|
||||
FROM `scrape_sources`
|
||||
WHERE `source_key` = 'hudy'
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`enabled` = VALUES(`enabled`),
|
||||
`priority` = VALUES(`priority`),
|
||||
`parser_module` = VALUES(`parser_module`),
|
||||
`selector_config` = VALUES(`selector_config`),
|
||||
`extraction_config` = VALUES(`extraction_config`),
|
||||
`note` = VALUES(`note`);
|
||||
|
||||
INSERT INTO `scrape_source_capabilities` (
|
||||
`source_id`,
|
||||
`capability`,
|
||||
`enabled`,
|
||||
`priority`,
|
||||
`parser_key`,
|
||||
`parser_module`,
|
||||
`browser_engine`,
|
||||
`browser_headless`,
|
||||
`wait_after_load_ms`,
|
||||
`min_longest_side`,
|
||||
`default_target_strategy`,
|
||||
`selector_config`,
|
||||
`extraction_config`,
|
||||
`quality_config`,
|
||||
`note`
|
||||
)
|
||||
SELECT
|
||||
`id`,
|
||||
'pictures',
|
||||
1,
|
||||
20,
|
||||
'idealo-splide-gallery',
|
||||
'idealo.splide-gallery',
|
||||
'firefox',
|
||||
1,
|
||||
3000,
|
||||
800,
|
||||
'color',
|
||||
JSON_OBJECT(
|
||||
'galleryTrackSelector', 'div.splide__track.container',
|
||||
'slideImageSelector', 'ul.splide__list li.splide__slide img'
|
||||
),
|
||||
JSON_OBJECT(
|
||||
'urlAttributes', JSON_ARRAY('srcset', 'data-srcset', 'data-large', 'data-original', 'data-src', 'src'),
|
||||
'pictureSource', 'productGalleryOnly',
|
||||
'rejectSearchThumbnails', true,
|
||||
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
|
||||
),
|
||||
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
|
||||
'Idealo is primarily an image backup source. Use product gallery images only, not search-result thumbnails.'
|
||||
FROM `scrape_sources`
|
||||
WHERE `source_key` = 'idealo'
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`enabled` = VALUES(`enabled`),
|
||||
`priority` = VALUES(`priority`),
|
||||
`parser_module` = VALUES(`parser_module`),
|
||||
`browser_engine` = VALUES(`browser_engine`),
|
||||
`browser_headless` = VALUES(`browser_headless`),
|
||||
`wait_after_load_ms` = VALUES(`wait_after_load_ms`),
|
||||
`min_longest_side` = VALUES(`min_longest_side`),
|
||||
`default_target_strategy` = VALUES(`default_target_strategy`),
|
||||
`selector_config` = VALUES(`selector_config`),
|
||||
`extraction_config` = VALUES(`extraction_config`),
|
||||
`quality_config` = VALUES(`quality_config`),
|
||||
`note` = VALUES(`note`);
|
||||
|
||||
INSERT INTO `scrape_manufacturer_source_capabilities` (
|
||||
`manufacturer_source_id`,
|
||||
`source_capability_id`,
|
||||
`enabled`,
|
||||
`priority`,
|
||||
`target_strategy_override`
|
||||
)
|
||||
SELECT
|
||||
`sms`.`id`,
|
||||
`sc`.`id`,
|
||||
`sc`.`enabled`,
|
||||
`sc`.`priority`,
|
||||
NULL
|
||||
FROM `scrape_manufacturer_sources` `sms`
|
||||
JOIN `scrape_source_capabilities` `sc`
|
||||
ON `sc`.`source_id` = `sms`.`source_id`
|
||||
ON DUPLICATE KEY UPDATE
|
||||
`enabled` = VALUES(`enabled`),
|
||||
`priority` = VALUES(`priority`);
|
||||
|
||||
Reference in New Issue
Block a user