Add local scrape database mode and schema

This commit is contained in:
2026-08-05 16:18:50 +02:00
parent 1076c3c040
commit 38dfb9fd54
12 changed files with 193 additions and 25 deletions
+88
View File
@@ -0,0 +1,88 @@
CREATE DATABASE IF NOT EXISTS `catalog_scrape`
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
USE `catalog_scrape`;
CREATE TABLE IF NOT EXISTS `scrape_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_key` VARCHAR(64) NOT NULL,
`source_name` VARCHAR(128) NOT NULL,
`source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
`base_url` VARCHAR(1024) NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`search_primary_key` VARCHAR(64) NULL,
`search_fallback_keys` VARCHAR(255) NULL,
`search_url_template` VARCHAR(2048) NULL,
`fallback_url_template` VARCHAR(2048) NULL,
`search_context_keys` VARCHAR(255) NULL,
`scraper_config` JSON NULL,
`request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`last_health_status` SMALLINT UNSIGNED NULL,
`last_health_checked_at` DATETIME NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_products` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NULL,
`id_product` BIGINT UNSIGNED NULL,
`supplier_reference` VARCHAR(255) NULL,
`ean13` VARCHAR(32) NULL,
`color` VARCHAR(255) NULL,
`source_id` BIGINT UNSIGNED NOT NULL,
`source_url` VARCHAR(2048) NOT NULL,
`match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean',
`source_product_key` VARCHAR(255) NULL,
`product_name` TEXT NULL,
`raw_data` JSON NULL,
`parsed_data` JSON NULL,
`status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found',
`error_message` TEXT NULL,
`fetched_at` DATETIME NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_scrape_products_ean` (`ean13`),
KEY `idx_scrape_products_reference` (`supplier_reference`),
KEY `idx_scrape_products_source` (`source_id`),
CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_assets` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`scrape_product_id` BIGINT UNSIGNED NOT NULL,
`asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image',
`asset_url` VARCHAR(2048) NOT NULL,
`local_path` VARCHAR(1024) NULL,
`content_hash` CHAR(64) NULL,
`position` INT UNSIGNED NOT NULL DEFAULT 0,
`alt_text` VARCHAR(512) NULL,
`status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered',
`error_message` TEXT NULL,
`fetched_at` DATETIME NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_scrape_assets_product` (`scrape_product_id`),
CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_runs` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_id` BIGINT UNSIGNED NULL,
`manufacturer_id` INT UNSIGNED NULL,
`started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`finished_at` DATETIME NULL,
`status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running',
`request_count` INT UNSIGNED NOT NULL DEFAULT 0,
`success_count` INT UNSIGNED NOT NULL DEFAULT 0,
`error_count` INT UNSIGNED NOT NULL DEFAULT 0,
`notes` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_scrape_runs_source` (`source_id`),
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;