Store reusable scraper capabilities in scrape DB

agent.md

@@ -120,6 +120,8 @@ There are three selectable data sources:
 - Uses the separate local MariaDB database named `catalog_scrape`.
 - Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
 - Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
+- Source extraction rules are stored as reusable capabilities in Local scrape DB: `search`, `pictures`, `texts` and `features`. The code is only the generic engine; source-specific selectors, parser keys, minimum picture quality, browser engine choices and target strategies must live in DB so the same Hudy/Idealo rules can be reused for other manufacturers.
+- Manufacturer-to-source mapping may enable or override individual capabilities, for example Hudy `pictures/texts/features` and Idealo `pictures`. Do not duplicate the same source configuration per manufacturer when a reusable source capability already exists.
 - On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source.
 - The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
 - The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`.

scripts/migrate-mapping-to-scrape-db.ts

@@ -57,7 +57,11 @@ const connection = await mariadb.createConnection({
 try {
   await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
   const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
-  for (const statement of schemaSql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) {
+  const schemaStatements = schemaSql
+    .split(/;\s*(?:\r?\n|$)/)
+    .map((item) => item.trim())
+    .filter(Boolean);
+  for (const statement of schemaStatements) {
     await connection.query(statement);
   }
 
@@ -250,9 +254,17 @@ try {
       priority = VALUES(priority)
   `);
 
+  for (const statement of schemaStatements.filter(isCapabilitySeedStatement)) {
+    await connection.query(statement);
+  }
+
   const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`);
   const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`);
   console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`);
 } finally {
   await connection.end();
 }
+
+function isCapabilitySeedStatement(statement) {
+  return /^INSERT INTO `scrape_(source_capabilities|manufacturer_source_capabilities)`/i.test(statement);
+}

sql/local-scrape-db.sql

@@ -83,6 +83,51 @@ CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
   CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
 ) ENGINE=InnoDB;
 
+CREATE TABLE IF NOT EXISTS `scrape_source_capabilities` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `source_id` BIGINT UNSIGNED NOT NULL,
+  `capability` ENUM('search', 'pictures', 'texts', 'features') NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `parser_key` VARCHAR(128) NOT NULL,
+  `parser_module` VARCHAR(255) NULL,
+  `browser_engine` VARCHAR(32) NULL,
+  `browser_headless` TINYINT(1) NULL,
+  `wait_after_load_ms` INT UNSIGNED NULL,
+  `min_width` INT UNSIGNED NULL,
+  `min_height` INT UNSIGNED NULL,
+  `min_longest_side` INT UNSIGNED NULL,
+  `default_target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
+  `selector_config` JSON NULL,
+  `extraction_config` JSON NULL,
+  `quality_config` JSON NULL,
+  `note` TEXT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_source_capability_parser` (`source_id`, `capability`, `parser_key`),
+  KEY `idx_scrape_source_capabilities_lookup` (`source_id`, `capability`, `enabled`, `priority`),
+  CONSTRAINT `fk_scrape_source_capabilities_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `manufacturer_source_id` BIGINT UNSIGNED NOT NULL,
+  `source_capability_id` BIGINT UNSIGNED NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `target_strategy_override` ENUM('unknown', 'color', 'size', 'combination', 'manual') NULL,
+  `config_override` JSON NULL,
+  `note` TEXT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_manufacturer_source_capability` (`manufacturer_source_id`, `source_capability_id`),
+  KEY `idx_scrape_manufacturer_source_capabilities_capability` (`source_capability_id`),
+  CONSTRAINT `fk_scrape_manufacturer_source_capabilities_mapping` FOREIGN KEY (`manufacturer_source_id`) REFERENCES `scrape_manufacturer_sources` (`id`),
+  CONSTRAINT `fk_scrape_manufacturer_source_capabilities_capability` FOREIGN KEY (`source_capability_id`) REFERENCES `scrape_source_capabilities` (`id`)
+) ENGINE=InnoDB;
+
 CREATE TABLE IF NOT EXISTS `scrape_products` (
   `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
   `manufacturer_id` INT UNSIGNED NULL,
@@ -218,3 +263,213 @@ CREATE TABLE IF NOT EXISTS `scrape_runs` (
   KEY `idx_scrape_runs_source` (`source_id`),
   CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
 ) ENGINE=InnoDB;
+
+INSERT INTO `scrape_source_capabilities` (
+  `source_id`,
+  `capability`,
+  `enabled`,
+  `priority`,
+  `parser_key`,
+  `parser_module`,
+  `browser_engine`,
+  `browser_headless`,
+  `wait_after_load_ms`,
+  `min_longest_side`,
+  `default_target_strategy`,
+  `selector_config`,
+  `extraction_config`,
+  `quality_config`,
+  `note`
+)
+SELECT
+  `id`,
+  'pictures',
+  1,
+  10,
+  'hudy-gallery',
+  'hudy.picture-gallery',
+  NULL,
+  NULL,
+  NULL,
+  800,
+  'color',
+  JSON_OBJECT(
+    'colorSwitcherSelector', 'a[id^="product-change-color-"]',
+    'eanTableSelector', 'table.product-params__params-table',
+    'topImageSelector', 'p#snippet-productInfo-image img.product-top__img',
+    'gallerySelector', 'p#snippet-productInfo-image, .pdbox__window'
+  ),
+  JSON_OBJECT(
+    'verifyVariantBy', JSON_ARRAY('ean', 'color'),
+    'pictureSource', 'largeGallery',
+    'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
+  ),
+  JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
+  'Hudy product gallery pictures. Verify the color variant by EAN before extracting large gallery images.'
+FROM `scrape_sources`
+WHERE `source_key` = 'hudy'
+ON DUPLICATE KEY UPDATE
+  `enabled` = VALUES(`enabled`),
+  `priority` = VALUES(`priority`),
+  `parser_module` = VALUES(`parser_module`),
+  `min_longest_side` = VALUES(`min_longest_side`),
+  `default_target_strategy` = VALUES(`default_target_strategy`),
+  `selector_config` = VALUES(`selector_config`),
+  `extraction_config` = VALUES(`extraction_config`),
+  `quality_config` = VALUES(`quality_config`),
+  `note` = VALUES(`note`);
+
+INSERT INTO `scrape_source_capabilities` (
+  `source_id`,
+  `capability`,
+  `enabled`,
+  `priority`,
+  `parser_key`,
+  `parser_module`,
+  `browser_engine`,
+  `browser_headless`,
+  `wait_after_load_ms`,
+  `default_target_strategy`,
+  `selector_config`,
+  `extraction_config`,
+  `note`
+)
+SELECT
+  `id`,
+  'texts',
+  1,
+  30,
+  'hudy-product-texts',
+  'hudy.product-texts',
+  NULL,
+  NULL,
+  NULL,
+  'manual',
+  JSON_OBJECT('descriptionSelector', '.product-detail, .product-top, [itemprop="description"]'),
+  JSON_OBJECT('fields', JSON_ARRAY('shortDescription', 'description', 'marketingText')),
+  'Hudy text extraction placeholder. User will teach exact text fields before automation is trusted.'
+FROM `scrape_sources`
+WHERE `source_key` = 'hudy'
+ON DUPLICATE KEY UPDATE
+  `enabled` = VALUES(`enabled`),
+  `priority` = VALUES(`priority`),
+  `parser_module` = VALUES(`parser_module`),
+  `selector_config` = VALUES(`selector_config`),
+  `extraction_config` = VALUES(`extraction_config`),
+  `note` = VALUES(`note`);
+
+INSERT INTO `scrape_source_capabilities` (
+  `source_id`,
+  `capability`,
+  `enabled`,
+  `priority`,
+  `parser_key`,
+  `parser_module`,
+  `browser_engine`,
+  `browser_headless`,
+  `wait_after_load_ms`,
+  `default_target_strategy`,
+  `selector_config`,
+  `extraction_config`,
+  `note`
+)
+SELECT
+  `id`,
+  'features',
+  1,
+  20,
+  'hudy-product-params',
+  'hudy.product-params',
+  NULL,
+  NULL,
+  NULL,
+  'manual',
+  JSON_OBJECT('paramsTableSelector', 'table.product-params__params-table'),
+  JSON_OBJECT('fields', JSON_ARRAY('label', 'value'), 'mergeStrategy', 'review-first'),
+  'Hudy feature extraction placeholder. Product parameters are kept reviewable before catalog writes.'
+FROM `scrape_sources`
+WHERE `source_key` = 'hudy'
+ON DUPLICATE KEY UPDATE
+  `enabled` = VALUES(`enabled`),
+  `priority` = VALUES(`priority`),
+  `parser_module` = VALUES(`parser_module`),
+  `selector_config` = VALUES(`selector_config`),
+  `extraction_config` = VALUES(`extraction_config`),
+  `note` = VALUES(`note`);
+
+INSERT INTO `scrape_source_capabilities` (
+  `source_id`,
+  `capability`,
+  `enabled`,
+  `priority`,
+  `parser_key`,
+  `parser_module`,
+  `browser_engine`,
+  `browser_headless`,
+  `wait_after_load_ms`,
+  `min_longest_side`,
+  `default_target_strategy`,
+  `selector_config`,
+  `extraction_config`,
+  `quality_config`,
+  `note`
+)
+SELECT
+  `id`,
+  'pictures',
+  1,
+  20,
+  'idealo-splide-gallery',
+  'idealo.splide-gallery',
+  'firefox',
+  1,
+  3000,
+  800,
+  'color',
+  JSON_OBJECT(
+    'galleryTrackSelector', 'div.splide__track.container',
+    'slideImageSelector', 'ul.splide__list li.splide__slide img'
+  ),
+  JSON_OBJECT(
+    'urlAttributes', JSON_ARRAY('srcset', 'data-srcset', 'data-large', 'data-original', 'data-src', 'src'),
+    'pictureSource', 'productGalleryOnly',
+    'rejectSearchThumbnails', true,
+    'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
+  ),
+  JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
+  'Idealo is primarily an image backup source. Use product gallery images only, not search-result thumbnails.'
+FROM `scrape_sources`
+WHERE `source_key` = 'idealo'
+ON DUPLICATE KEY UPDATE
+  `enabled` = VALUES(`enabled`),
+  `priority` = VALUES(`priority`),
+  `parser_module` = VALUES(`parser_module`),
+  `browser_engine` = VALUES(`browser_engine`),
+  `browser_headless` = VALUES(`browser_headless`),
+  `wait_after_load_ms` = VALUES(`wait_after_load_ms`),
+  `min_longest_side` = VALUES(`min_longest_side`),
+  `default_target_strategy` = VALUES(`default_target_strategy`),
+  `selector_config` = VALUES(`selector_config`),
+  `extraction_config` = VALUES(`extraction_config`),
+  `quality_config` = VALUES(`quality_config`),
+  `note` = VALUES(`note`);
+
+INSERT INTO `scrape_manufacturer_source_capabilities` (
+  `manufacturer_source_id`,
+  `source_capability_id`,
+  `enabled`,
+  `priority`,
+  `target_strategy_override`
+)
+SELECT
+  `sms`.`id`,
+  `sc`.`id`,
+  `sc`.`enabled`,
+  `sc`.`priority`,
+  NULL
+FROM `scrape_manufacturer_sources` `sms`
+JOIN `scrape_source_capabilities` `sc`
+  ON `sc`.`source_id` = `sms`.`source_id`
+ON DUPLICATE KEY UPDATE
+  `enabled` = VALUES(`enabled`),
+  `priority` = VALUES(`priority`);

src/services/catalog-products.ts

@@ -1077,6 +1077,8 @@ function normalizeMappingSource(row, tableName) {
     browserEngine: String(row.browser_engine ?? "default").trim().toLowerCase(),
     browserHeadless: normalizeEnabled(row.browser_headless ?? 0),
     waitAfterLoadMs: Number(row.wait_after_load_ms) || 3000,
+    scraperConfig: parseJsonValue(row.scraper_config, {}),
+    capabilities: parseCapabilities(row.capabilities),
     enabled: normalizeEnabled(row.mapping_enabled ?? row.enabled ?? row.active ?? 1)
       && normalizeEnabled(row.enabled ?? 1),
     note: row.note ?? row.description ?? "",
@@ -1084,6 +1086,46 @@ function normalizeMappingSource(row, tableName) {
   };
 }
 
+function parseCapabilities(value) {
+  const parsed = parseJsonValue(value, []);
+  if (!Array.isArray(parsed)) return [];
+
+  return parsed
+    .filter(Boolean)
+    .filter((item) => item.capability)
+    .map((item) => ({
+      capability: item.capability,
+      enabled: normalizeEnabled(item.enabled ?? 1),
+      priority: Number(item.priority) || 100,
+      parserKey: item.parserKey ?? "",
+      parserModule: item.parserModule ?? "",
+      browserEngine: String(item.browserEngine ?? "").trim().toLowerCase(),
+      browserHeadless: item.browserHeadless === null ? null : normalizeEnabled(item.browserHeadless ?? 0),
+      waitAfterLoadMs: item.waitAfterLoadMs === null ? null : Number(item.waitAfterLoadMs) || null,
+      minWidth: item.minWidth === null ? null : Number(item.minWidth) || null,
+      minHeight: item.minHeight === null ? null : Number(item.minHeight) || null,
+      minLongestSide: item.minLongestSide === null ? null : Number(item.minLongestSide) || null,
+      defaultTargetStrategy: item.defaultTargetStrategy ?? "unknown",
+      targetStrategyOverride: item.targetStrategyOverride || "",
+      selectorConfig: parseJsonValue(item.selectorConfig, {}),
+      extractionConfig: parseJsonValue(item.extractionConfig, {}),
+      qualityConfig: parseJsonValue(item.qualityConfig, {}),
+      configOverride: parseJsonValue(item.configOverride, {}),
+      note: item.note ?? "",
+    }));
+}
+
+function parseJsonValue(value, fallback) {
+  if (value === null || value === undefined || value === "") return fallback;
+  if (typeof value === "object" && !Buffer.isBuffer(value)) return value;
+
+  try {
+    return JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : String(value));
+  } catch {
+    return fallback;
+  }
+}
+
 function parseList(value) {
   return String(value ?? "")
     .split(",")

src/sql/catalog-maker.ts

@@ -362,13 +362,17 @@ export function mappingSourceTablesSql() {
     FROM information_schema.tables
     WHERE table_schema = DATABASE()
       AND table_name IN (
-        'ps_product_catalog_source',
+        'scrape_sources',
-        'ps_product_catalog_manufacturer_source'
+        'scrape_manufacturer_sources',
+        'scrape_source_capabilities',
+        'scrape_manufacturer_source_capabilities'
       )
     ORDER BY FIELD(
       table_name,
-      'ps_product_catalog_manufacturer_source',
+      'scrape_manufacturer_sources',
-      'ps_product_catalog_source'
+      'scrape_sources',
+      'scrape_manufacturer_source_capabilities',
+      'scrape_source_capabilities'
     )
   `;
 }
@@ -379,9 +383,9 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
 
   return `
     SELECT
-      ms.id_product_catalog_manufacturer_source AS mapping_id,
+      ms.id AS mapping_id,
-      ms.id_manufacturer,
+      ms.manufacturer_id AS id_manufacturer,
-      s.id_product_catalog_source,
+      s.id AS id_product_catalog_source,
       ms.enabled AS mapping_enabled,
       ms.priority AS mapping_priority,
       s.source_key,
@@ -400,11 +404,67 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
       s.browser_engine,
       s.browser_headless,
       s.wait_after_load_ms,
+      s.scraper_config,
+      s.note,
+      JSON_ARRAYAGG(
+        CASE
+          WHEN sc.id IS NULL THEN NULL
+          ELSE JSON_OBJECT(
+            'capability', sc.capability,
+            'enabled', COALESCE(msc.enabled, sc.enabled),
+            'priority', COALESCE(msc.priority, sc.priority),
+            'parserKey', sc.parser_key,
+            'parserModule', sc.parser_module,
+            'browserEngine', COALESCE(sc.browser_engine, ''),
+            'browserHeadless', sc.browser_headless,
+            'waitAfterLoadMs', sc.wait_after_load_ms,
+            'minWidth', sc.min_width,
+            'minHeight', sc.min_height,
+            'minLongestSide', sc.min_longest_side,
+            'defaultTargetStrategy', sc.default_target_strategy,
+            'targetStrategyOverride', COALESCE(msc.target_strategy_override, ''),
+            'selectorConfig', sc.selector_config,
+            'extractionConfig', sc.extraction_config,
+            'qualityConfig', sc.quality_config,
+            'configOverride', msc.config_override,
+            'note', sc.note
+          )
+        END
+      ) AS capabilities
+    FROM ${scrapeSchema}.scrape_manufacturer_sources ms
+    JOIN ${scrapeSchema}.scrape_sources s
+      ON s.id = ms.source_id
+    LEFT JOIN ${scrapeSchema}.scrape_source_capabilities sc
+      ON sc.source_id = s.id
+     AND sc.enabled = 1
+    LEFT JOIN ${scrapeSchema}.scrape_manufacturer_source_capabilities msc
+      ON msc.source_capability_id = sc.id
+     AND msc.manufacturer_source_id = ms.id
+    WHERE ms.manufacturer_id = ${safeManufacturerId}
+    GROUP BY
+      ms.id,
+      ms.manufacturer_id,
+      s.id,
+      ms.enabled,
+      ms.priority,
+      s.source_key,
+      s.source_name,
+      s.source_type,
+      s.base_url,
+      s.enabled,
+      s.priority,
+      s.search_primary_key,
+      s.search_fallback_keys,
+      s.fallback_source_type,
+      s.search_url_template,
+      s.fallback_url_template,
+      s.search_context_keys,
+      s.access_mode,
+      s.browser_engine,
+      s.browser_headless,
+      s.wait_after_load_ms,
+      s.scraper_config,
       s.note
-    FROM ${scrapeSchema}.ps_product_catalog_manufacturer_source ms
-    JOIN ${scrapeSchema}.ps_product_catalog_source s
-      ON s.id_product_catalog_source = ms.id_product_catalog_source
-    WHERE ms.id_manufacturer = ${safeManufacturerId}
     ORDER BY
       ms.priority ASC,
       s.priority ASC,

test/scrape-image-schema.test.ts

@@ -28,3 +28,28 @@ test("local scrape schema can learn picture target rules from manual corrections
   assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
   assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
 });
+
+test("local scrape schema stores reusable source capabilities for pictures texts and features", () => {
+  assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_source_capabilities`/);
+  assert.match(schema, /`capability` ENUM\('search', 'pictures', 'texts', 'features'\) NOT NULL/);
+  assert.match(schema, /`parser_key` VARCHAR\(128\) NOT NULL/);
+  assert.match(schema, /`selector_config` JSON NULL/);
+  assert.match(schema, /`extraction_config` JSON NULL/);
+  assert.match(schema, /`quality_config` JSON NULL/);
+  assert.match(schema, /UNIQUE KEY `uq_scrape_source_capability_parser`/);
+});
+
+test("local scrape schema links manufacturer mappings to selected source capabilities", () => {
+  assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities`/);
+  assert.match(schema, /`target_strategy_override` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\) NULL/);
+  assert.match(schema, /`config_override` JSON NULL/);
+  assert.match(schema, /UNIQUE KEY `uq_scrape_manufacturer_source_capability`/);
+});
+
+test("local scrape schema seeds Hudy and Idealo extraction capabilities", () => {
+  assert.match(schema, /'hudy-gallery'/);
+  assert.match(schema, /'hudy-product-texts'/);
+  assert.match(schema, /'hudy-product-params'/);
+  assert.match(schema, /'idealo-splide-gallery'/);
+  assert.match(schema, /'productGalleryOnly'/);
+});

test/sql-catalog-maker.test.ts

@@ -59,6 +59,9 @@ test("mapping source queries are read-only", () => {
   assert.doesNotThrow(() =>
     assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })),
   );
+  assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_manufacturer_sources/);
+  assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_source_capabilities/);
+  assert.match(mappingSourcesSql({ manufacturerId: 13 }), /capabilities/);
 });
 
 test("manufacturer refresh updates are narrowly allowlisted", () => {