rajch_ales/product-import-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store reusable scraper capabilities in scrape DB

Browse Source
This commit is contained in:
rajch_ales
2026-08-06 01:01:55 +02:00
parent b11ada9c91
commit 9e3b4de4e2
7 changed files with 411 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent.md
+2
View File
@@ -120,6 +120,8 @@ There are three selectable data sources:
- Uses the separate local MariaDB database named `catalog_scrape`.
- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
- Source extraction rules are stored as reusable capabilities in Local scrape DB: `search`, `pictures`, `texts` and `features`. The code is only the generic engine; source-specific selectors, parser keys, minimum picture quality, browser engine choices and target strategies must live in DB so the same Hudy/Idealo rules can be reused for other manufacturers.
- Manufacturer-to-source mapping may enable or override individual capabilities, for example Hudy `pictures/texts/features` and Idealo `pictures`. Do not duplicate the same source configuration per manufacturer when a reusable source capability already exists.
- On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source.
- The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
- The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`.
scripts/migrate-mapping-to-scrape-db.ts
+13 -1
View File
@@ -57,7 +57,11 @@ const connection = await mariadb.createConnection({
try {
await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
for (const statement of schemaSql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) {
const schemaStatements = schemaSql
.split(/;\s*(?:\r?\n|$)/)
.map((item) => item.trim())
.filter(Boolean);
for (const statement of schemaStatements) {
await connection.query(statement);
}
@@ -250,9 +254,17 @@ try {
priority = VALUES(priority)
`);
for (const statement of schemaStatements.filter(isCapabilitySeedStatement)) {
await connection.query(statement);
}
const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`);
const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`);
console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`);
} finally {
await connection.end();
}
function isCapabilitySeedStatement(statement) {
return /^INSERT INTO `scrape_(source_capabilities|manufacturer_source_capabilities)`/i.test(statement);
}
sql/local-scrape-db.sql
+255
View File
@@ -83,6 +83,51 @@ CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_source_capabilities` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_id` BIGINT UNSIGNED NOT NULL,
`capability` ENUM('search', 'pictures', 'texts', 'features') NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`parser_key` VARCHAR(128) NOT NULL,
`parser_module` VARCHAR(255) NULL,
`browser_engine` VARCHAR(32) NULL,
`browser_headless` TINYINT(1) NULL,
`wait_after_load_ms` INT UNSIGNED NULL,
`min_width` INT UNSIGNED NULL,
`min_height` INT UNSIGNED NULL,
`min_longest_side` INT UNSIGNED NULL,
`default_target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
`selector_config` JSON NULL,
`extraction_config` JSON NULL,
`quality_config` JSON NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_source_capability_parser` (`source_id`, `capability`, `parser_key`),
KEY `idx_scrape_source_capabilities_lookup` (`source_id`, `capability`, `enabled`, `priority`),
CONSTRAINT `fk_scrape_source_capabilities_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_source_id` BIGINT UNSIGNED NOT NULL,
`source_capability_id` BIGINT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`target_strategy_override` ENUM('unknown', 'color', 'size', 'combination', 'manual') NULL,
`config_override` JSON NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_manufacturer_source_capability` (`manufacturer_source_id`, `source_capability_id`),
KEY `idx_scrape_manufacturer_source_capabilities_capability` (`source_capability_id`),
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_mapping` FOREIGN KEY (`manufacturer_source_id`) REFERENCES `scrape_manufacturer_sources` (`id`),
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_capability` FOREIGN KEY (`source_capability_id`) REFERENCES `scrape_source_capabilities` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_products` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NULL,
@@ -218,3 +263,213 @@ CREATE TABLE IF NOT EXISTS `scrape_runs` (
KEY `idx_scrape_runs_source` (`source_id`),
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`min_longest_side`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`quality_config`,
`note`
)
SELECT
`id`,
'pictures',
1,
10,
'hudy-gallery',
'hudy.picture-gallery',
NULL,
NULL,
NULL,
800,
'color',
JSON_OBJECT(
'colorSwitcherSelector', 'a[id^="product-change-color-"]',
'eanTableSelector', 'table.product-params__params-table',
'topImageSelector', 'p#snippet-productInfo-image img.product-top__img',
'gallerySelector', 'p#snippet-productInfo-image, .pdbox__window'
),
JSON_OBJECT(
'verifyVariantBy', JSON_ARRAY('ean', 'color'),
'pictureSource', 'largeGallery',
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
),
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
'Hudy product gallery pictures. Verify the color variant by EAN before extracting large gallery images.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`min_longest_side` = VALUES(`min_longest_side`),
`default_target_strategy` = VALUES(`default_target_strategy`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`quality_config` = VALUES(`quality_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`note`
)
SELECT
`id`,
'texts',
1,
30,
'hudy-product-texts',
'hudy.product-texts',
NULL,
NULL,
NULL,
'manual',
JSON_OBJECT('descriptionSelector', '.product-detail, .product-top, [itemprop="description"]'),
JSON_OBJECT('fields', JSON_ARRAY('shortDescription', 'description', 'marketingText')),
'Hudy text extraction placeholder. User will teach exact text fields before automation is trusted.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`note`
)
SELECT
`id`,
'features',
1,
20,
'hudy-product-params',
'hudy.product-params',
NULL,
NULL,
NULL,
'manual',
JSON_OBJECT('paramsTableSelector', 'table.product-params__params-table'),
JSON_OBJECT('fields', JSON_ARRAY('label', 'value'), 'mergeStrategy', 'review-first'),
'Hudy feature extraction placeholder. Product parameters are kept reviewable before catalog writes.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`min_longest_side`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`quality_config`,
`note`
)
SELECT
`id`,
'pictures',
1,
20,
'idealo-splide-gallery',
'idealo.splide-gallery',
'firefox',
1,
3000,
800,
'color',
JSON_OBJECT(
'galleryTrackSelector', 'div.splide__track.container',
'slideImageSelector', 'ul.splide__list li.splide__slide img'
),
JSON_OBJECT(
'urlAttributes', JSON_ARRAY('srcset', 'data-srcset', 'data-large', 'data-original', 'data-src', 'src'),
'pictureSource', 'productGalleryOnly',
'rejectSearchThumbnails', true,
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
),
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
'Idealo is primarily an image backup source. Use product gallery images only, not search-result thumbnails.'
FROM `scrape_sources`
WHERE `source_key` = 'idealo'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`browser_engine` = VALUES(`browser_engine`),
`browser_headless` = VALUES(`browser_headless`),
`wait_after_load_ms` = VALUES(`wait_after_load_ms`),
`min_longest_side` = VALUES(`min_longest_side`),
`default_target_strategy` = VALUES(`default_target_strategy`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`quality_config` = VALUES(`quality_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_manufacturer_source_capabilities` (
`manufacturer_source_id`,
`source_capability_id`,
`enabled`,
`priority`,
`target_strategy_override`
)
SELECT
`sms`.`id`,
`sc`.`id`,
`sc`.`enabled`,
`sc`.`priority`,
NULL
FROM `scrape_manufacturer_sources` `sms`
JOIN `scrape_source_capabilities` `sc`
ON `sc`.`source_id` = `sms`.`source_id`
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`);
src/services/catalog-products.ts
+42
View File
@@ -1077,6 +1077,8 @@ function normalizeMappingSource(row, tableName) {
browserEngine: String(row.browser_engine ?? "default").trim().toLowerCase(),
browserHeadless: normalizeEnabled(row.browser_headless ?? 0),
waitAfterLoadMs: Number(row.wait_after_load_ms) || 3000,
scraperConfig: parseJsonValue(row.scraper_config, {}),
capabilities: parseCapabilities(row.capabilities),
enabled: normalizeEnabled(row.mapping_enabled ?? row.enabled ?? row.active ?? 1)
&& normalizeEnabled(row.enabled ?? 1),
note: row.note ?? row.description ?? "",
@@ -1084,6 +1086,46 @@ function normalizeMappingSource(row, tableName) {
};
}
function parseCapabilities(value) {
const parsed = parseJsonValue(value, []);
if (!Array.isArray(parsed)) return [];
return parsed
.filter(Boolean)
.filter((item) => item.capability)
.map((item) => ({
capability: item.capability,
enabled: normalizeEnabled(item.enabled ?? 1),
priority: Number(item.priority) || 100,
parserKey: item.parserKey ?? "",
parserModule: item.parserModule ?? "",
browserEngine: String(item.browserEngine ?? "").trim().toLowerCase(),
browserHeadless: item.browserHeadless === null ? null : normalizeEnabled(item.browserHeadless ?? 0),
waitAfterLoadMs: item.waitAfterLoadMs === null ? null : Number(item.waitAfterLoadMs) || null,
minWidth: item.minWidth === null ? null : Number(item.minWidth) || null,
minHeight: item.minHeight === null ? null : Number(item.minHeight) || null,
minLongestSide: item.minLongestSide === null ? null : Number(item.minLongestSide) || null,
defaultTargetStrategy: item.defaultTargetStrategy ?? "unknown",
targetStrategyOverride: item.targetStrategyOverride || "",
selectorConfig: parseJsonValue(item.selectorConfig, {}),
extractionConfig: parseJsonValue(item.extractionConfig, {}),
qualityConfig: parseJsonValue(item.qualityConfig, {}),
configOverride: parseJsonValue(item.configOverride, {}),
note: item.note ?? "",
}));
}
function parseJsonValue(value, fallback) {
if (value === null || value === undefined || value === "") return fallback;
if (typeof value === "object" && !Buffer.isBuffer(value)) return value;
try {
return JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : String(value));
} catch {
return fallback;
}
}
function parseList(value) {
return String(value ?? "")
.split(",")
src/sql/catalog-maker.ts
+71 -11
View File
@@ -362,13 +362,17 @@ export function mappingSourceTablesSql() {
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name IN (
'ps_product_catalog_source',
'ps_product_catalog_manufacturer_source'
'scrape_sources',
'scrape_manufacturer_sources',
'scrape_source_capabilities',
'scrape_manufacturer_source_capabilities'
)
ORDER BY FIELD(
table_name,
'ps_product_catalog_manufacturer_source',
'ps_product_catalog_source'
'scrape_manufacturer_sources',
'scrape_sources',
'scrape_manufacturer_source_capabilities',
'scrape_source_capabilities'
)
`;
}
@@ -379,9 +383,9 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
return `
SELECT
ms.id_product_catalog_manufacturer_source AS mapping_id,
ms.id_manufacturer,
s.id_product_catalog_source,
ms.id AS mapping_id,
ms.manufacturer_id AS id_manufacturer,
s.id AS id_product_catalog_source,
ms.enabled AS mapping_enabled,
ms.priority AS mapping_priority,
s.source_key,
@@ -400,11 +404,67 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
s.browser_engine,
s.browser_headless,
s.wait_after_load_ms,
s.scraper_config,
s.note,
JSON_ARRAYAGG(
CASE
WHEN sc.id IS NULL THEN NULL
ELSE JSON_OBJECT(
'capability', sc.capability,
'enabled', COALESCE(msc.enabled, sc.enabled),
'priority', COALESCE(msc.priority, sc.priority),
'parserKey', sc.parser_key,
'parserModule', sc.parser_module,
'browserEngine', COALESCE(sc.browser_engine, ''),
'browserHeadless', sc.browser_headless,
'waitAfterLoadMs', sc.wait_after_load_ms,
'minWidth', sc.min_width,
'minHeight', sc.min_height,
'minLongestSide', sc.min_longest_side,
'defaultTargetStrategy', sc.default_target_strategy,
'targetStrategyOverride', COALESCE(msc.target_strategy_override, ''),
'selectorConfig', sc.selector_config,
'extractionConfig', sc.extraction_config,
'qualityConfig', sc.quality_config,
'configOverride', msc.config_override,
'note', sc.note
)
END
) AS capabilities
FROM ${scrapeSchema}.scrape_manufacturer_sources ms
JOIN ${scrapeSchema}.scrape_sources s
ON s.id = ms.source_id
LEFT JOIN ${scrapeSchema}.scrape_source_capabilities sc
ON sc.source_id = s.id
AND sc.enabled = 1
LEFT JOIN ${scrapeSchema}.scrape_manufacturer_source_capabilities msc
ON msc.source_capability_id = sc.id
AND msc.manufacturer_source_id = ms.id
WHERE ms.manufacturer_id = ${safeManufacturerId}
GROUP BY
ms.id,
ms.manufacturer_id,
s.id,
ms.enabled,
ms.priority,
s.source_key,
s.source_name,
s.source_type,
s.base_url,
s.enabled,
s.priority,
s.search_primary_key,
s.search_fallback_keys,
s.fallback_source_type,
s.search_url_template,
s.fallback_url_template,
s.search_context_keys,
s.access_mode,
s.browser_engine,
s.browser_headless,
s.wait_after_load_ms,
s.scraper_config,
s.note
FROM ${scrapeSchema}.ps_product_catalog_manufacturer_source ms
JOIN ${scrapeSchema}.ps_product_catalog_source s
ON s.id_product_catalog_source = ms.id_product_catalog_source
WHERE ms.id_manufacturer = ${safeManufacturerId}
ORDER BY
ms.priority ASC,
s.priority ASC,
test/scrape-image-schema.test.ts
+25
View File
@@ -28,3 +28,28 @@ test("local scrape schema can learn picture target rules from manual corrections
assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
});
test("local scrape schema stores reusable source capabilities for pictures texts and features", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_source_capabilities`/);
assert.match(schema, /`capability` ENUM\('search', 'pictures', 'texts', 'features'\) NOT NULL/);
assert.match(schema, /`parser_key` VARCHAR\(128\) NOT NULL/);
assert.match(schema, /`selector_config` JSON NULL/);
assert.match(schema, /`extraction_config` JSON NULL/);
assert.match(schema, /`quality_config` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_source_capability_parser`/);
});
test("local scrape schema links manufacturer mappings to selected source capabilities", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities`/);
assert.match(schema, /`target_strategy_override` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\) NULL/);
assert.match(schema, /`config_override` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_manufacturer_source_capability`/);
});
test("local scrape schema seeds Hudy and Idealo extraction capabilities", () => {
assert.match(schema, /'hudy-gallery'/);
assert.match(schema, /'hudy-product-texts'/);
assert.match(schema, /'hudy-product-params'/);
assert.match(schema, /'idealo-splide-gallery'/);
assert.match(schema, /'productGalleryOnly'/);
});
test/sql-catalog-maker.test.ts
+3
View File
@@ -59,6 +59,9 @@ test("mapping source queries are read-only", () => {
assert.doesNotThrow(() =>
assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })),
);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_manufacturer_sources/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_source_capabilities/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /capabilities/);
});
test("manufacturer refresh updates are narrowly allowlisted", () => {