Prepare scrape image storage schema

agent.md

@@ -164,6 +164,9 @@ There are three selectable data sources:
 - The source-candidates popup must provide a `Reload` action beside `Close`; `Reload` bypasses only the current product's cached source result and re-runs the source lookup using the current local mapping configuration. It must not switch database modes or write to Live DB.
 - Source candidates show how they were found, including EAN/reference/name and color context.
 - Each source row has its own `Get pictures` action.
+- Picture candidates must show source, image dimensions and the proposed target assignment. The target assignment is not always color: supported strategies are `color`, `size`, `combination` and `manual`. Start by using manual corrections from the user, then learn reusable rules in Local scrape DB.
+- Scraped pictures and their metadata belong in Local scrape DB, not browser localStorage. Store the binary image in `scrape_image_blobs` with `sha256`, `perceptual_hash`, mime type, dimensions and size. Store each source/target proposal in `scrape_picture_candidates`.
+- Use `sha256` to remove exact duplicate image files and a perceptual hash to flag visually similar images from different sources or sizes. Exact duplicates may be collapsed automatically; visual duplicates should remain reviewable until the user approves the rule.
 - Product source cache is cleared when a new product loads.
 - Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images.
 - Idealo uses the browser adapter and is intended mainly as a picture source.

sql/local-scrape-db.sql

@@ -127,6 +127,81 @@ CREATE TABLE IF NOT EXISTS `scrape_assets` (
   CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
 ) ENGINE=InnoDB;
 
+CREATE TABLE IF NOT EXISTS `scrape_image_blobs` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `sha256` CHAR(64) NOT NULL,
+  `perceptual_hash` CHAR(16) NULL,
+  `mime_type` VARCHAR(64) NOT NULL,
+  `width` INT UNSIGNED NULL,
+  `height` INT UNSIGNED NULL,
+  `size_bytes` INT UNSIGNED NOT NULL,
+  `image_data` LONGBLOB NOT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_image_blobs_sha256` (`sha256`),
+  KEY `idx_scrape_image_blobs_phash` (`perceptual_hash`),
+  KEY `idx_scrape_image_blobs_size` (`width`, `height`, `size_bytes`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_picture_candidates` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `image_blob_id` BIGINT UNSIGNED NOT NULL,
+  `source_id` BIGINT UNSIGNED NOT NULL,
+  `scrape_product_id` BIGINT UNSIGNED NULL,
+  `manufacturer_id` INT UNSIGNED NULL,
+  `id_product_catalog` BIGINT UNSIGNED NULL,
+  `id_product` BIGINT UNSIGNED NULL,
+  `supplier_reference` VARCHAR(255) NULL,
+  `ean13` VARCHAR(32) NULL,
+  `color` VARCHAR(255) NULL,
+  `size_value` VARCHAR(128) NULL,
+  `combi` VARCHAR(255) NULL,
+  `source_page_url` VARCHAR(2048) NULL,
+  `source_image_url` VARCHAR(2048) NOT NULL,
+  `source_position` INT UNSIGNED NOT NULL DEFAULT 0,
+  `target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
+  `target_color` VARCHAR(255) NULL,
+  `target_size` VARCHAR(128) NULL,
+  `target_combi` VARCHAR(255) NULL,
+  `target_ean13` VARCHAR(32) NULL,
+  `target_id_product_catalog` BIGINT UNSIGNED NULL,
+  `match_confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
+  `duplicate_status` ENUM('unique', 'exact_duplicate', 'visual_duplicate', 'review') NOT NULL DEFAULT 'review',
+  `duplicate_of_candidate_id` BIGINT UNSIGNED NULL,
+  `status` ENUM('draft', 'approved', 'rejected') NOT NULL DEFAULT 'draft',
+  `review_note` TEXT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_picture_candidate_source_image` (`source_id`, `source_image_url`(512)),
+  KEY `idx_scrape_picture_candidates_blob` (`image_blob_id`),
+  KEY `idx_scrape_picture_candidates_source` (`source_id`),
+  KEY `idx_scrape_picture_candidates_product` (`manufacturer_id`, `supplier_reference`, `ean13`),
+  KEY `idx_scrape_picture_candidates_target` (`target_strategy`, `target_color`, `target_size`, `target_combi`),
+  KEY `idx_scrape_picture_candidates_duplicate` (`duplicate_status`, `duplicate_of_candidate_id`),
+  CONSTRAINT `fk_scrape_picture_candidates_blob` FOREIGN KEY (`image_blob_id`) REFERENCES `scrape_image_blobs` (`id`),
+  CONSTRAINT `fk_scrape_picture_candidates_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`),
+  CONSTRAINT `fk_scrape_picture_candidates_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `manufacturer_id` INT UNSIGNED NULL,
+  `source_id` BIGINT UNSIGNED NULL,
+  `id_category_default` INT UNSIGNED NULL,
+  `catalog_category` VARCHAR(255) NULL,
+  `product_name_pattern` VARCHAR(255) NULL,
+  `target_strategy` ENUM('color', 'size', 'combination', 'manual') NOT NULL,
+  `confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
+  `examples_count` INT UNSIGNED NOT NULL DEFAULT 0,
+  `note` TEXT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `idx_scrape_picture_target_rules_lookup` (`manufacturer_id`, `source_id`, `id_category_default`),
+  CONSTRAINT `fk_scrape_picture_target_rules_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
+) ENGINE=InnoDB;
+
 CREATE TABLE IF NOT EXISTS `scrape_runs` (
   `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
   `source_id` BIGINT UNSIGNED NULL,

test/scrape-image-schema.test.ts

@@ -0,0 +1,30 @@
+// @ts-nocheck
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+
+const schema = fs.readFileSync("sql/local-scrape-db.sql", "utf8");
+
+test("local scrape schema stores image binaries with exact and visual hashes", () => {
+  assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_image_blobs`/);
+  assert.match(schema, /`sha256` CHAR\(64\) NOT NULL/);
+  assert.match(schema, /`perceptual_hash` CHAR\(16\) NULL/);
+  assert.match(schema, /`image_data` LONGBLOB NOT NULL/);
+  assert.match(schema, /UNIQUE KEY `uq_scrape_image_blobs_sha256` \(`sha256`\)/);
+});
+
+test("local scrape schema tracks picture candidates and their target combination logic", () => {
+  assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_candidates`/);
+  assert.match(schema, /`target_strategy` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\)/);
+  assert.match(schema, /`target_color` VARCHAR\(255\) NULL/);
+  assert.match(schema, /`target_size` VARCHAR\(128\) NULL/);
+  assert.match(schema, /`target_combi` VARCHAR\(255\) NULL/);
+  assert.match(schema, /`duplicate_status` ENUM\('unique', 'exact_duplicate', 'visual_duplicate', 'review'\)/);
+});
+
+test("local scrape schema can learn picture target rules from manual corrections", () => {
+  assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules`/);
+  assert.match(schema, /`product_name_pattern` VARCHAR\(255\) NULL/);
+  assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
+  assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
+});