diff --git a/agent.md b/agent.md index fa01091..980e373 100644 --- a/agent.md +++ b/agent.md @@ -164,6 +164,9 @@ There are three selectable data sources: - The source-candidates popup must provide a `Reload` action beside `Close`; `Reload` bypasses only the current product's cached source result and re-runs the source lookup using the current local mapping configuration. It must not switch database modes or write to Live DB. - Source candidates show how they were found, including EAN/reference/name and color context. - Each source row has its own `Get pictures` action. +- Picture candidates must show source, image dimensions and the proposed target assignment. The target assignment is not always color: supported strategies are `color`, `size`, `combination` and `manual`. Start by using manual corrections from the user, then learn reusable rules in Local scrape DB. +- Scraped pictures and their metadata belong in Local scrape DB, not browser localStorage. Store the binary image in `scrape_image_blobs` with `sha256`, `perceptual_hash`, mime type, dimensions and size. Store each source/target proposal in `scrape_picture_candidates`. +- Use `sha256` to remove exact duplicate image files and a perceptual hash to flag visually similar images from different sources or sizes. Exact duplicates may be collapsed automatically; visual duplicates should remain reviewable until the user approves the rule. - Product source cache is cleared when a new product loads. - Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images. - Idealo uses the browser adapter and is intended mainly as a picture source. diff --git a/sql/local-scrape-db.sql b/sql/local-scrape-db.sql index 7087883..4dc86c6 100644 --- a/sql/local-scrape-db.sql +++ b/sql/local-scrape-db.sql @@ -127,6 +127,81 @@ CREATE TABLE IF NOT EXISTS `scrape_assets` ( CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) ) ENGINE=InnoDB; +CREATE TABLE IF NOT EXISTS `scrape_image_blobs` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `sha256` CHAR(64) NOT NULL, + `perceptual_hash` CHAR(16) NULL, + `mime_type` VARCHAR(64) NOT NULL, + `width` INT UNSIGNED NULL, + `height` INT UNSIGNED NULL, + `size_bytes` INT UNSIGNED NOT NULL, + `image_data` LONGBLOB NOT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_scrape_image_blobs_sha256` (`sha256`), + KEY `idx_scrape_image_blobs_phash` (`perceptual_hash`), + KEY `idx_scrape_image_blobs_size` (`width`, `height`, `size_bytes`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `scrape_picture_candidates` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `image_blob_id` BIGINT UNSIGNED NOT NULL, + `source_id` BIGINT UNSIGNED NOT NULL, + `scrape_product_id` BIGINT UNSIGNED NULL, + `manufacturer_id` INT UNSIGNED NULL, + `id_product_catalog` BIGINT UNSIGNED NULL, + `id_product` BIGINT UNSIGNED NULL, + `supplier_reference` VARCHAR(255) NULL, + `ean13` VARCHAR(32) NULL, + `color` VARCHAR(255) NULL, + `size_value` VARCHAR(128) NULL, + `combi` VARCHAR(255) NULL, + `source_page_url` VARCHAR(2048) NULL, + `source_image_url` VARCHAR(2048) NOT NULL, + `source_position` INT UNSIGNED NOT NULL DEFAULT 0, + `target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown', + `target_color` VARCHAR(255) NULL, + `target_size` VARCHAR(128) NULL, + `target_combi` VARCHAR(255) NULL, + `target_ean13` VARCHAR(32) NULL, + `target_id_product_catalog` BIGINT UNSIGNED NULL, + `match_confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00, + `duplicate_status` ENUM('unique', 'exact_duplicate', 'visual_duplicate', 'review') NOT NULL DEFAULT 'review', + `duplicate_of_candidate_id` BIGINT UNSIGNED NULL, + `status` ENUM('draft', 'approved', 'rejected') NOT NULL DEFAULT 'draft', + `review_note` TEXT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_scrape_picture_candidate_source_image` (`source_id`, `source_image_url`(512)), + KEY `idx_scrape_picture_candidates_blob` (`image_blob_id`), + KEY `idx_scrape_picture_candidates_source` (`source_id`), + KEY `idx_scrape_picture_candidates_product` (`manufacturer_id`, `supplier_reference`, `ean13`), + KEY `idx_scrape_picture_candidates_target` (`target_strategy`, `target_color`, `target_size`, `target_combi`), + KEY `idx_scrape_picture_candidates_duplicate` (`duplicate_status`, `duplicate_of_candidate_id`), + CONSTRAINT `fk_scrape_picture_candidates_blob` FOREIGN KEY (`image_blob_id`) REFERENCES `scrape_image_blobs` (`id`), + CONSTRAINT `fk_scrape_picture_candidates_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`), + CONSTRAINT `fk_scrape_picture_candidates_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `manufacturer_id` INT UNSIGNED NULL, + `source_id` BIGINT UNSIGNED NULL, + `id_category_default` INT UNSIGNED NULL, + `catalog_category` VARCHAR(255) NULL, + `product_name_pattern` VARCHAR(255) NULL, + `target_strategy` ENUM('color', 'size', 'combination', 'manual') NOT NULL, + `confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00, + `examples_count` INT UNSIGNED NOT NULL DEFAULT 0, + `note` TEXT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_scrape_picture_target_rules_lookup` (`manufacturer_id`, `source_id`, `id_category_default`), + CONSTRAINT `fk_scrape_picture_target_rules_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) +) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS `scrape_runs` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `source_id` BIGINT UNSIGNED NULL, diff --git a/test/scrape-image-schema.test.ts b/test/scrape-image-schema.test.ts new file mode 100644 index 0000000..5470786 --- /dev/null +++ b/test/scrape-image-schema.test.ts @@ -0,0 +1,30 @@ +// @ts-nocheck +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; + +const schema = fs.readFileSync("sql/local-scrape-db.sql", "utf8"); + +test("local scrape schema stores image binaries with exact and visual hashes", () => { + assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_image_blobs`/); + assert.match(schema, /`sha256` CHAR\(64\) NOT NULL/); + assert.match(schema, /`perceptual_hash` CHAR\(16\) NULL/); + assert.match(schema, /`image_data` LONGBLOB NOT NULL/); + assert.match(schema, /UNIQUE KEY `uq_scrape_image_blobs_sha256` \(`sha256`\)/); +}); + +test("local scrape schema tracks picture candidates and their target combination logic", () => { + assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_candidates`/); + assert.match(schema, /`target_strategy` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\)/); + assert.match(schema, /`target_color` VARCHAR\(255\) NULL/); + assert.match(schema, /`target_size` VARCHAR\(128\) NULL/); + assert.match(schema, /`target_combi` VARCHAR\(255\) NULL/); + assert.match(schema, /`duplicate_status` ENUM\('unique', 'exact_duplicate', 'visual_duplicate', 'review'\)/); +}); + +test("local scrape schema can learn picture target rules from manual corrections", () => { + assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules`/); + assert.match(schema, /`product_name_pattern` VARCHAR\(255\) NULL/); + assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/); + assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/); +});