221 lines
10 KiB
SQL
221 lines
10 KiB
SQL
CREATE DATABASE IF NOT EXISTS `catalog_scrape`
|
|
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
|
|
|
USE `catalog_scrape`;
|
|
|
|
CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` (
|
|
`id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`source_key` VARCHAR(64) NOT NULL,
|
|
`source_name` VARCHAR(128) NOT NULL,
|
|
`source_type` VARCHAR(32) NOT NULL DEFAULT '',
|
|
`base_url` VARCHAR(512) NOT NULL,
|
|
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
|
`priority` INT NOT NULL DEFAULT 100,
|
|
`search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean',
|
|
`search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '',
|
|
`fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none',
|
|
`search_url_template` VARCHAR(1000) NOT NULL,
|
|
`fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '',
|
|
`search_context_keys` VARCHAR(255) NOT NULL DEFAULT '',
|
|
`access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser',
|
|
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
|
|
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
|
|
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
|
|
`note` TEXT NULL,
|
|
PRIMARY KEY (`id_product_catalog_source`),
|
|
UNIQUE KEY `uq_product_catalog_source_key` (`source_key`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` (
|
|
`id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`id_manufacturer` INT UNSIGNED NOT NULL,
|
|
`id_product_catalog_source` INT UNSIGNED NOT NULL,
|
|
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
|
`priority` INT NOT NULL DEFAULT 100,
|
|
`note` TEXT NULL,
|
|
PRIMARY KEY (`id_product_catalog_manufacturer_source`),
|
|
UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`),
|
|
KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_sources` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`legacy_catalog_source_id` BIGINT UNSIGNED NULL,
|
|
`source_key` VARCHAR(64) NOT NULL,
|
|
`source_name` VARCHAR(128) NOT NULL,
|
|
`source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
|
|
`base_url` VARCHAR(1024) NOT NULL,
|
|
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
|
`priority` INT NOT NULL DEFAULT 100,
|
|
`search_primary_key` VARCHAR(64) NULL,
|
|
`search_fallback_keys` VARCHAR(255) NULL,
|
|
`fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none',
|
|
`search_url_template` VARCHAR(2048) NULL,
|
|
`fallback_url_template` VARCHAR(2048) NULL,
|
|
`search_context_keys` VARCHAR(255) NULL,
|
|
`access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser',
|
|
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
|
|
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
|
|
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
|
|
`scraper_config` JSON NULL,
|
|
`request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
|
|
`last_health_status` SMALLINT UNSIGNED NULL,
|
|
`last_health_checked_at` DATETIME NULL,
|
|
`note` TEXT NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`),
|
|
UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`manufacturer_id` INT UNSIGNED NOT NULL,
|
|
`source_id` BIGINT UNSIGNED NOT NULL,
|
|
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
|
`priority` INT NOT NULL DEFAULT 100,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`),
|
|
KEY `idx_scrape_manufacturer_source_source` (`source_id`),
|
|
CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_products` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`manufacturer_id` INT UNSIGNED NULL,
|
|
`id_product` BIGINT UNSIGNED NULL,
|
|
`supplier_reference` VARCHAR(255) NULL,
|
|
`ean13` VARCHAR(32) NULL,
|
|
`color` VARCHAR(255) NULL,
|
|
`source_id` BIGINT UNSIGNED NOT NULL,
|
|
`source_url` VARCHAR(2048) NOT NULL,
|
|
`match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean',
|
|
`source_product_key` VARCHAR(255) NULL,
|
|
`product_name` TEXT NULL,
|
|
`raw_data` JSON NULL,
|
|
`parsed_data` JSON NULL,
|
|
`status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found',
|
|
`error_message` TEXT NULL,
|
|
`fetched_at` DATETIME NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
KEY `idx_scrape_products_ean` (`ean13`),
|
|
KEY `idx_scrape_products_reference` (`supplier_reference`),
|
|
KEY `idx_scrape_products_source` (`source_id`),
|
|
CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_assets` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`scrape_product_id` BIGINT UNSIGNED NOT NULL,
|
|
`asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image',
|
|
`asset_url` VARCHAR(2048) NOT NULL,
|
|
`local_path` VARCHAR(1024) NULL,
|
|
`content_hash` CHAR(64) NULL,
|
|
`position` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`alt_text` VARCHAR(512) NULL,
|
|
`status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered',
|
|
`error_message` TEXT NULL,
|
|
`fetched_at` DATETIME NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
KEY `idx_scrape_assets_product` (`scrape_product_id`),
|
|
CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_image_blobs` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`sha256` CHAR(64) NOT NULL,
|
|
`perceptual_hash` CHAR(16) NULL,
|
|
`mime_type` VARCHAR(64) NOT NULL,
|
|
`width` INT UNSIGNED NULL,
|
|
`height` INT UNSIGNED NULL,
|
|
`size_bytes` INT UNSIGNED NOT NULL,
|
|
`image_data` LONGBLOB NOT NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uq_scrape_image_blobs_sha256` (`sha256`),
|
|
KEY `idx_scrape_image_blobs_phash` (`perceptual_hash`),
|
|
KEY `idx_scrape_image_blobs_size` (`width`, `height`, `size_bytes`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_picture_candidates` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`image_blob_id` BIGINT UNSIGNED NOT NULL,
|
|
`source_id` BIGINT UNSIGNED NOT NULL,
|
|
`scrape_product_id` BIGINT UNSIGNED NULL,
|
|
`manufacturer_id` INT UNSIGNED NULL,
|
|
`id_product_catalog` BIGINT UNSIGNED NULL,
|
|
`id_product` BIGINT UNSIGNED NULL,
|
|
`supplier_reference` VARCHAR(255) NULL,
|
|
`ean13` VARCHAR(32) NULL,
|
|
`color` VARCHAR(255) NULL,
|
|
`size_value` VARCHAR(128) NULL,
|
|
`combi` VARCHAR(255) NULL,
|
|
`source_page_url` VARCHAR(2048) NULL,
|
|
`source_image_url` VARCHAR(2048) NOT NULL,
|
|
`source_position` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
|
|
`target_color` VARCHAR(255) NULL,
|
|
`target_size` VARCHAR(128) NULL,
|
|
`target_combi` VARCHAR(255) NULL,
|
|
`target_ean13` VARCHAR(32) NULL,
|
|
`target_id_product_catalog` BIGINT UNSIGNED NULL,
|
|
`match_confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
|
|
`duplicate_status` ENUM('unique', 'exact_duplicate', 'visual_duplicate', 'review') NOT NULL DEFAULT 'review',
|
|
`duplicate_of_candidate_id` BIGINT UNSIGNED NULL,
|
|
`status` ENUM('draft', 'approved', 'rejected') NOT NULL DEFAULT 'draft',
|
|
`review_note` TEXT NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uq_scrape_picture_candidate_source_image` (`source_id`, `source_image_url`(512)),
|
|
KEY `idx_scrape_picture_candidates_blob` (`image_blob_id`),
|
|
KEY `idx_scrape_picture_candidates_source` (`source_id`),
|
|
KEY `idx_scrape_picture_candidates_product` (`manufacturer_id`, `supplier_reference`, `ean13`),
|
|
KEY `idx_scrape_picture_candidates_target` (`target_strategy`, `target_color`, `target_size`, `target_combi`),
|
|
KEY `idx_scrape_picture_candidates_duplicate` (`duplicate_status`, `duplicate_of_candidate_id`),
|
|
CONSTRAINT `fk_scrape_picture_candidates_blob` FOREIGN KEY (`image_blob_id`) REFERENCES `scrape_image_blobs` (`id`),
|
|
CONSTRAINT `fk_scrape_picture_candidates_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`),
|
|
CONSTRAINT `fk_scrape_picture_candidates_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`manufacturer_id` INT UNSIGNED NULL,
|
|
`source_id` BIGINT UNSIGNED NULL,
|
|
`id_category_default` INT UNSIGNED NULL,
|
|
`catalog_category` VARCHAR(255) NULL,
|
|
`product_name_pattern` VARCHAR(255) NULL,
|
|
`target_strategy` ENUM('color', 'size', 'combination', 'manual') NOT NULL,
|
|
`confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
|
|
`examples_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`note` TEXT NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
KEY `idx_scrape_picture_target_rules_lookup` (`manufacturer_id`, `source_id`, `id_category_default`),
|
|
CONSTRAINT `fk_scrape_picture_target_rules_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
|
) ENGINE=InnoDB;
|
|
|
|
CREATE TABLE IF NOT EXISTS `scrape_runs` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`source_id` BIGINT UNSIGNED NULL,
|
|
`manufacturer_id` INT UNSIGNED NULL,
|
|
`started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
`finished_at` DATETIME NULL,
|
|
`status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running',
|
|
`request_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`success_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`error_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`notes` TEXT NULL,
|
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`id`),
|
|
KEY `idx_scrape_runs_source` (`source_id`),
|
|
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
|
) ENGINE=InnoDB;
|