rajch_ales/product-import-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move catalog source mapping to scrape database

Browse Source
This commit is contained in:
rajch_ales
2026-08-05 23:06:01 +02:00
parent e3caf25823
commit 5210dc2042
7 changed files with 198 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/migrate-mapping-to-scrape-db.ts
+99 -6
View File
@@ -69,11 +69,104 @@ try {
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) {
throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`);
const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
if (hasCatalogSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
fallback_source_type,
search_url_template,
fallback_url_template,
search_context_keys,
access_mode,
browser_engine,
browser_headless,
wait_after_load_ms,
note
)
SELECT
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
COALESCE(fallback_source_type, 'none'),
search_url_template,
fallback_url_template,
search_context_keys,
COALESCE(access_mode, 'browser'),
COALESCE(browser_engine, 'default'),
COALESCE(browser_headless, 0),
COALESCE(wait_after_load_ms, 3000),
note
FROM \`${catalogDb}\`.ps_product_catalog_source
ON DUPLICATE KEY UPDATE
source_key = VALUES(source_key),
source_name = VALUES(source_name),
source_type = VALUES(source_type),
base_url = VALUES(base_url),
enabled = VALUES(enabled),
priority = VALUES(priority),
search_primary_key = VALUES(search_primary_key),
search_fallback_keys = VALUES(search_fallback_keys),
fallback_source_type = VALUES(fallback_source_type),
search_url_template = VALUES(search_url_template),
fallback_url_template = VALUES(fallback_url_template),
search_context_keys = VALUES(search_context_keys),
access_mode = VALUES(access_mode),
browser_engine = VALUES(browser_engine),
browser_headless = VALUES(browser_headless),
wait_after_load_ms = VALUES(wait_after_load_ms),
note = VALUES(note)
`);
}
if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) {
throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`);
if (hasCatalogManufacturerSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
)
SELECT
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
ON DUPLICATE KEY UPDATE
id_manufacturer = VALUES(id_manufacturer),
id_product_catalog_source = VALUES(id_product_catalog_source),
enabled = VALUES(enabled),
priority = VALUES(priority),
note = VALUES(note)
`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
}
await connection.query(`
@@ -116,7 +209,7 @@ try {
COALESCE(browser_headless, 0),
COALESCE(wait_after_load_ms, 3000),
note
FROM \`${catalogDb}\`.ps_product_catalog_source
FROM \`${scrapeDb}\`.ps_product_catalog_source
ON DUPLICATE KEY UPDATE
source_key = VALUES(source_key),
source_name = VALUES(source_name),
@@ -149,7 +242,7 @@ try {
ss.id,
ms.enabled,
ms.priority
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms
FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
JOIN \`${scrapeDb}\`.scrape_sources ss
ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
ON DUPLICATE KEY UPDATE