CREATE DATABASE IF NOT EXISTS `catalog_scrape` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; USE `catalog_scrape`; CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` ( `id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT, `source_key` VARCHAR(64) NOT NULL, `source_name` VARCHAR(128) NOT NULL, `source_type` VARCHAR(32) NOT NULL DEFAULT '', `base_url` VARCHAR(512) NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean', `search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '', `fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none', `search_url_template` VARCHAR(1000) NOT NULL, `fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '', `search_context_keys` VARCHAR(255) NOT NULL DEFAULT '', `access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser', `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default', `browser_headless` TINYINT(1) NOT NULL DEFAULT 0, `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `note` TEXT NULL, PRIMARY KEY (`id_product_catalog_source`), UNIQUE KEY `uq_product_catalog_source_key` (`source_key`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` ( `id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT, `id_manufacturer` INT UNSIGNED NOT NULL, `id_product_catalog_source` INT UNSIGNED NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `note` TEXT NULL, PRIMARY KEY (`id_product_catalog_manufacturer_source`), UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`), KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_sources` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `legacy_catalog_source_id` BIGINT UNSIGNED NULL, `source_key` VARCHAR(64) NOT NULL, `source_name` VARCHAR(128) NOT NULL, `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier', `base_url` VARCHAR(1024) NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `search_primary_key` VARCHAR(64) NULL, `search_fallback_keys` VARCHAR(255) NULL, `fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none', `search_url_template` VARCHAR(2048) NULL, `fallback_url_template` VARCHAR(2048) NULL, `search_context_keys` VARCHAR(255) NULL, `access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser', `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default', `browser_headless` TINYINT(1) NOT NULL DEFAULT 0, `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `scraper_config` JSON NULL, `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `last_health_status` SMALLINT UNSIGNED NULL, `last_health_checked_at` DATETIME NULL, `note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`), UNIQUE KEY `uq_scrape_sources_key` (`source_key`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `manufacturer_id` INT UNSIGNED NOT NULL, `source_id` BIGINT UNSIGNED NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`), KEY `idx_scrape_manufacturer_source_source` (`source_id`), CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_source_capabilities` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `source_id` BIGINT UNSIGNED NOT NULL, `capability` ENUM('search', 'pictures', 'texts', 'features') NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `parser_key` VARCHAR(128) NOT NULL, `parser_module` VARCHAR(255) NULL, `browser_engine` VARCHAR(32) NULL, `browser_headless` TINYINT(1) NULL, `wait_after_load_ms` INT UNSIGNED NULL, `min_width` INT UNSIGNED NULL, `min_height` INT UNSIGNED NULL, `min_longest_side` INT UNSIGNED NULL, `default_target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown', `selector_config` JSON NULL, `extraction_config` JSON NULL, `quality_config` JSON NULL, `note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_source_capability_parser` (`source_id`, `capability`, `parser_key`), KEY `idx_scrape_source_capabilities_lookup` (`source_id`, `capability`, `enabled`, `priority`), CONSTRAINT `fk_scrape_source_capabilities_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `manufacturer_source_id` BIGINT UNSIGNED NOT NULL, `source_capability_id` BIGINT UNSIGNED NOT NULL, `enabled` TINYINT(1) NOT NULL DEFAULT 1, `priority` INT NOT NULL DEFAULT 100, `target_strategy_override` ENUM('unknown', 'color', 'size', 'combination', 'manual') NULL, `config_override` JSON NULL, `note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_manufacturer_source_capability` (`manufacturer_source_id`, `source_capability_id`), KEY `idx_scrape_manufacturer_source_capabilities_capability` (`source_capability_id`), CONSTRAINT `fk_scrape_manufacturer_source_capabilities_mapping` FOREIGN KEY (`manufacturer_source_id`) REFERENCES `scrape_manufacturer_sources` (`id`), CONSTRAINT `fk_scrape_manufacturer_source_capabilities_capability` FOREIGN KEY (`source_capability_id`) REFERENCES `scrape_source_capabilities` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_products` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `manufacturer_id` INT UNSIGNED NULL, `id_product` BIGINT UNSIGNED NULL, `supplier_reference` VARCHAR(255) NULL, `ean13` VARCHAR(32) NULL, `color` VARCHAR(255) NULL, `source_id` BIGINT UNSIGNED NOT NULL, `source_url` VARCHAR(2048) NOT NULL, `match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean', `source_product_key` VARCHAR(255) NULL, `product_name` TEXT NULL, `raw_data` JSON NULL, `parsed_data` JSON NULL, `status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found', `error_message` TEXT NULL, `fetched_at` DATETIME NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `idx_scrape_products_ean` (`ean13`), KEY `idx_scrape_products_reference` (`supplier_reference`), KEY `idx_scrape_products_source` (`source_id`), CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_assets` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `scrape_product_id` BIGINT UNSIGNED NOT NULL, `asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image', `asset_url` VARCHAR(2048) NOT NULL, `local_path` VARCHAR(1024) NULL, `content_hash` CHAR(64) NULL, `position` INT UNSIGNED NOT NULL DEFAULT 0, `alt_text` VARCHAR(512) NULL, `status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered', `error_message` TEXT NULL, `fetched_at` DATETIME NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `idx_scrape_assets_product` (`scrape_product_id`), CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_image_blobs` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `sha256` CHAR(64) NOT NULL, `perceptual_hash` CHAR(16) NULL, `mime_type` VARCHAR(64) NOT NULL, `width` INT UNSIGNED NULL, `height` INT UNSIGNED NULL, `size_bytes` INT UNSIGNED NOT NULL, `image_data` LONGBLOB NOT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_image_blobs_sha256` (`sha256`), KEY `idx_scrape_image_blobs_phash` (`perceptual_hash`), KEY `idx_scrape_image_blobs_size` (`width`, `height`, `size_bytes`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_picture_candidates` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `image_blob_id` BIGINT UNSIGNED NOT NULL, `source_id` BIGINT UNSIGNED NOT NULL, `scrape_product_id` BIGINT UNSIGNED NULL, `manufacturer_id` INT UNSIGNED NULL, `id_product_catalog` BIGINT UNSIGNED NULL, `id_product` BIGINT UNSIGNED NULL, `supplier_reference` VARCHAR(255) NULL, `ean13` VARCHAR(32) NULL, `color` VARCHAR(255) NULL, `size_value` VARCHAR(128) NULL, `combi` VARCHAR(255) NULL, `source_page_url` VARCHAR(2048) NULL, `source_image_url` VARCHAR(2048) NOT NULL, `source_position` INT UNSIGNED NOT NULL DEFAULT 0, `target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown', `target_color` VARCHAR(255) NULL, `target_size` VARCHAR(128) NULL, `target_combi` VARCHAR(255) NULL, `target_ean13` VARCHAR(32) NULL, `target_id_product_catalog` BIGINT UNSIGNED NULL, `match_confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00, `duplicate_status` ENUM('unique', 'exact_duplicate', 'visual_duplicate', 'review') NOT NULL DEFAULT 'review', `duplicate_of_candidate_id` BIGINT UNSIGNED NULL, `status` ENUM('draft', 'approved', 'rejected') NOT NULL DEFAULT 'draft', `review_note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uq_scrape_picture_candidate_source_image` (`source_id`, `source_image_url`(512)), KEY `idx_scrape_picture_candidates_blob` (`image_blob_id`), KEY `idx_scrape_picture_candidates_source` (`source_id`), KEY `idx_scrape_picture_candidates_product` (`manufacturer_id`, `supplier_reference`, `ean13`), KEY `idx_scrape_picture_candidates_target` (`target_strategy`, `target_color`, `target_size`, `target_combi`), KEY `idx_scrape_picture_candidates_duplicate` (`duplicate_status`, `duplicate_of_candidate_id`), CONSTRAINT `fk_scrape_picture_candidates_blob` FOREIGN KEY (`image_blob_id`) REFERENCES `scrape_image_blobs` (`id`), CONSTRAINT `fk_scrape_picture_candidates_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`), CONSTRAINT `fk_scrape_picture_candidates_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `manufacturer_id` INT UNSIGNED NULL, `source_id` BIGINT UNSIGNED NULL, `id_category_default` INT UNSIGNED NULL, `catalog_category` VARCHAR(255) NULL, `product_name_pattern` VARCHAR(255) NULL, `target_strategy` ENUM('color', 'size', 'combination', 'manual') NOT NULL, `confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00, `examples_count` INT UNSIGNED NOT NULL DEFAULT 0, `note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `idx_scrape_picture_target_rules_lookup` (`manufacturer_id`, `source_id`, `id_category_default`), CONSTRAINT `fk_scrape_picture_target_rules_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS `scrape_runs` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `source_id` BIGINT UNSIGNED NULL, `manufacturer_id` INT UNSIGNED NULL, `started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `finished_at` DATETIME NULL, `status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running', `request_count` INT UNSIGNED NOT NULL DEFAULT 0, `success_count` INT UNSIGNED NOT NULL DEFAULT 0, `error_count` INT UNSIGNED NOT NULL DEFAULT 0, `notes` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `idx_scrape_runs_source` (`source_id`), CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) ) ENGINE=InnoDB; INSERT INTO `scrape_source_capabilities` ( `source_id`, `capability`, `enabled`, `priority`, `parser_key`, `parser_module`, `browser_engine`, `browser_headless`, `wait_after_load_ms`, `min_longest_side`, `default_target_strategy`, `selector_config`, `extraction_config`, `quality_config`, `note` ) SELECT `id`, 'pictures', 1, 10, 'hudy-gallery', 'hudy.picture-gallery', NULL, NULL, NULL, 800, 'color', JSON_OBJECT( 'colorSwitcherSelector', 'a[id^="product-change-color-"]', 'eanTableSelector', 'table.product-params__params-table', 'topImageSelector', 'p#snippet-productInfo-image img.product-top__img', 'gallerySelector', 'p#snippet-productInfo-image, .pdbox__window' ), JSON_OBJECT( 'verifyVariantBy', JSON_ARRAY('ean', 'color'), 'pictureSource', 'largeGallery', 'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash') ), JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true), 'Hudy product gallery pictures. Verify the color variant by EAN before extracting large gallery images.' FROM `scrape_sources` WHERE `source_key` = 'hudy' ON DUPLICATE KEY UPDATE `enabled` = VALUES(`enabled`), `priority` = VALUES(`priority`), `parser_module` = VALUES(`parser_module`), `min_longest_side` = VALUES(`min_longest_side`), `default_target_strategy` = VALUES(`default_target_strategy`), `selector_config` = VALUES(`selector_config`), `extraction_config` = VALUES(`extraction_config`), `quality_config` = VALUES(`quality_config`), `note` = VALUES(`note`); INSERT INTO `scrape_source_capabilities` ( `source_id`, `capability`, `enabled`, `priority`, `parser_key`, `parser_module`, `browser_engine`, `browser_headless`, `wait_after_load_ms`, `default_target_strategy`, `selector_config`, `extraction_config`, `note` ) SELECT `id`, 'texts', 1, 30, 'hudy-product-texts', 'hudy.product-texts', NULL, NULL, NULL, 'manual', JSON_OBJECT('descriptionSelector', '.product-detail, .product-top, [itemprop="description"]'), JSON_OBJECT('fields', JSON_ARRAY('shortDescription', 'description', 'marketingText')), 'Hudy text extraction placeholder. User will teach exact text fields before automation is trusted.' FROM `scrape_sources` WHERE `source_key` = 'hudy' ON DUPLICATE KEY UPDATE `enabled` = VALUES(`enabled`), `priority` = VALUES(`priority`), `parser_module` = VALUES(`parser_module`), `selector_config` = VALUES(`selector_config`), `extraction_config` = VALUES(`extraction_config`), `note` = VALUES(`note`); INSERT INTO `scrape_source_capabilities` ( `source_id`, `capability`, `enabled`, `priority`, `parser_key`, `parser_module`, `browser_engine`, `browser_headless`, `wait_after_load_ms`, `default_target_strategy`, `selector_config`, `extraction_config`, `note` ) SELECT `id`, 'features', 1, 20, 'hudy-product-params', 'hudy.product-params', NULL, NULL, NULL, 'manual', JSON_OBJECT('paramsTableSelector', 'table.product-params__params-table'), JSON_OBJECT('fields', JSON_ARRAY('label', 'value'), 'mergeStrategy', 'review-first'), 'Hudy feature extraction placeholder. Product parameters are kept reviewable before catalog writes.' FROM `scrape_sources` WHERE `source_key` = 'hudy' ON DUPLICATE KEY UPDATE `enabled` = VALUES(`enabled`), `priority` = VALUES(`priority`), `parser_module` = VALUES(`parser_module`), `selector_config` = VALUES(`selector_config`), `extraction_config` = VALUES(`extraction_config`), `note` = VALUES(`note`); INSERT INTO `scrape_source_capabilities` ( `source_id`, `capability`, `enabled`, `priority`, `parser_key`, `parser_module`, `browser_engine`, `browser_headless`, `wait_after_load_ms`, `min_longest_side`, `default_target_strategy`, `selector_config`, `extraction_config`, `quality_config`, `note` ) SELECT `id`, 'pictures', 1, 20, 'idealo-splide-gallery', 'idealo.splide-gallery', 'firefox', 1, 3000, 800, 'color', JSON_OBJECT( 'galleryTrackSelector', 'div.splide__track.container', 'slideImageSelector', 'ul.splide__list li.splide__slide img' ), JSON_OBJECT( 'urlAttributes', JSON_ARRAY('srcset', 'data-srcset', 'data-large', 'data-original', 'data-src', 'src'), 'pictureSource', 'productGalleryOnly', 'rejectSearchThumbnails', true, 'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash') ), JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true), 'Idealo is primarily an image backup source. Use product gallery images only, not search-result thumbnails.' FROM `scrape_sources` WHERE `source_key` = 'idealo' ON DUPLICATE KEY UPDATE `enabled` = VALUES(`enabled`), `priority` = VALUES(`priority`), `parser_module` = VALUES(`parser_module`), `browser_engine` = VALUES(`browser_engine`), `browser_headless` = VALUES(`browser_headless`), `wait_after_load_ms` = VALUES(`wait_after_load_ms`), `min_longest_side` = VALUES(`min_longest_side`), `default_target_strategy` = VALUES(`default_target_strategy`), `selector_config` = VALUES(`selector_config`), `extraction_config` = VALUES(`extraction_config`), `quality_config` = VALUES(`quality_config`), `note` = VALUES(`note`); INSERT INTO `scrape_manufacturer_source_capabilities` ( `manufacturer_source_id`, `source_capability_id`, `enabled`, `priority`, `target_strategy_override` ) SELECT `sms`.`id`, `sc`.`id`, `sc`.`enabled`, `sc`.`priority`, NULL FROM `scrape_manufacturer_sources` `sms` JOIN `scrape_source_capabilities` `sc` ON `sc`.`source_id` = `sms`.`source_id` ON DUPLICATE KEY UPDATE `enabled` = VALUES(`enabled`), `priority` = VALUES(`priority`);