Move source mapping to scrape database

This commit is contained in:
2026-08-05 21:40:58 +02:00
parent 17082f6d9e
commit a72bdab9ab
6 changed files with 221 additions and 14 deletions
+22
View File
@@ -5,6 +5,7 @@ USE `catalog_scrape`;
CREATE TABLE IF NOT EXISTS `scrape_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`legacy_catalog_source_id` BIGINT UNSIGNED NULL,
`source_key` VARCHAR(64) NOT NULL,
`source_name` VARCHAR(128) NOT NULL,
`source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
@@ -13,19 +14,40 @@ CREATE TABLE IF NOT EXISTS `scrape_sources` (
`priority` INT NOT NULL DEFAULT 100,
`search_primary_key` VARCHAR(64) NULL,
`search_fallback_keys` VARCHAR(255) NULL,
`fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none',
`search_url_template` VARCHAR(2048) NULL,
`fallback_url_template` VARCHAR(2048) NULL,
`search_context_keys` VARCHAR(255) NULL,
`access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser',
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`scraper_config` JSON NULL,
`request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`last_health_status` SMALLINT UNSIGNED NULL,
`last_health_checked_at` DATETIME NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`),
UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NOT NULL,
`source_id` BIGINT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`),
KEY `idx_scrape_manufacturer_source_source` (`source_id`),
CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_products` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NULL,