From a72bdab9ab0b70004f6f4680aea2711e41aa8df3 Mon Sep 17 00:00:00 2001 From: rajch_ales Date: Wed, 5 Aug 2026 21:40:58 +0200 Subject: [PATCH] Move source mapping to scrape database --- agent.md | 9 +- package.json | 1 + scripts/migrate-mapping-to-scrape-db.ts | 165 ++++++++++++++++++++++++ sql/local-scrape-db.sql | 22 ++++ src/services/catalog-products.ts | 13 +- src/sql/catalog-maker.ts | 25 ++-- 6 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 scripts/migrate-mapping-to-scrape-db.ts diff --git a/agent.md b/agent.md index 2f8eb36..8063b35 100644 --- a/agent.md +++ b/agent.md @@ -10,7 +10,8 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N - Local DB may later allow controlled writes, but writes must remain explicit, logged and protected by transaction/dry-run settings. - Never print, commit or share endpoint tokens, usernames or passwords. - Never use Live DB as a fallback when Local DB is selected. If Local DB is unavailable, show an error and keep the UI running. -- The selected database mode is authoritative for every database call. When `Local 9bplus DB` is selected, manufacturers, languages, products, combinations, mapping, source configuration and all other catalog reads must use the local MariaDB connection; no endpoint or Live DB call is allowed. When `Live 9bplus DB` is selected, every catalog call must use the existing endpoint and remain read-only. Do not mix sources within one workflow. +- The selected catalog database mode is authoritative for catalog data calls. When `Local 9bplus DB` is selected, manufacturers, languages, products, combinations and all other catalog reads must use the local MariaDB connection; no endpoint or Live DB call is allowed. When `Live 9bplus DB` is selected, catalog reads use the existing endpoint and remain read-only. +- Mapping, source configuration, scraper settings, browser-engine choices and scraped draft data belong to `Local scrape DB` (`catalog_scrape`) and must be read from there, not from Live DB or the product catalog database. - Keep the development server on port `3404` unless the user explicitly requests another port. - Do not delete or revert user files or unrelated changes. @@ -102,9 +103,11 @@ There are three selectable data sources: ### Local scrape DB - Uses the separate local MariaDB database named `catalog_scrape`. -- Stores scraper source configuration, scrape runs, discovered products and assets. +- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets. +- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database. - It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added. - The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`. +- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`. ### Local MariaDB connection @@ -124,7 +127,7 @@ There are three selectable data sources: - The backend must resolve that header before every API database operation and route the complete request through the selected source. A Local DB request must use `queryMariaDb`; a Live DB request must use `queryEndpoint`; there is no silent fallback or mixed-source response. - Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint. - Backend data API routes reject a Live DB request when the project is configured only for MariaDB. -- Mapping endpoints follow the same rule as product endpoints and must read mapping tables from the selected database. +- Product endpoints read catalog data from the selected catalog database. Mapping endpoints read scraper source mapping from Local scrape DB while still using the selected catalog database only for catalog/product data. - Static HTML/CSS/JS files must remain available even when a database is not configured. - The top status shows the selected source, connection state, permissions and execution mode. - Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI. diff --git a/package.json b/package.json index b0ab817..dd704e5 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "tsx src/server.ts", "catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run", "db:create-scrape": "tsx scripts/create-local-scrape-db.ts", + "db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts", "test": "tsx --test test/*.test.ts", "test:ui": "playwright test", "test:e2e": "playwright test tests/ui/database.e2e.spec.ts", diff --git a/scripts/migrate-mapping-to-scrape-db.ts b/scripts/migrate-mapping-to-scrape-db.ts new file mode 100644 index 0000000..f3c84a6 --- /dev/null +++ b/scripts/migrate-mapping-to-scrape-db.ts @@ -0,0 +1,165 @@ +import fs from "node:fs/promises"; +import mariadb from "mariadb"; + +async function readEnv() { + return Object.fromEntries( + (await fs.readFile(".env", "utf8")) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#") && line.includes("=")) + .map((line) => { + const index = line.indexOf("="); + return [line.slice(0, index).trim(), line.slice(index + 1).trim()]; + }), + ); +} + +async function queryRows(connection, sql, params = []) { + const rows = await connection.query(sql, params); + return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : []; +} + +async function tableExists(connection, schema, table) { + const rows = await queryRows( + connection, + "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1", + [schema, table], + ); + return rows.length > 0; +} + +async function columnExists(connection, schema, table, column) { + const rows = await queryRows( + connection, + "SELECT 1 FROM information_schema.columns WHERE table_schema = ? AND table_name = ? AND column_name = ? LIMIT 1", + [schema, table, column], + ); + return rows.length > 0; +} + +async function addColumnIfMissing(connection, schema, table, column, definition) { + if (await columnExists(connection, schema, table, column)) return; + await connection.query(`ALTER TABLE \`${schema}\`.\`${table}\` ADD COLUMN \`${column}\` ${definition}`); +} + +const env = await readEnv(); +const catalogDb = env.DB_NAME || "9bplus"; +const scrapeDb = env.SCRAPE_DB_NAME || "catalog_scrape"; +const connection = await mariadb.createConnection({ + host: env.DB_HOST || "127.0.0.1", + port: Number(env.DB_PORT || 3306), + user: env.DB_USER || "root", + password: env.DB_PASSWORD || "", + multipleStatements: false, + bigIntAsNumber: true, +}); + +try { + await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`); + const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8"); + for (const statement of schemaSql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) { + await connection.query(statement); + } + + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "legacy_catalog_source_id", "BIGINT UNSIGNED NULL"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "fallback_source_type", "VARCHAR(64) NOT NULL DEFAULT 'none'"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "access_mode", "VARCHAR(64) NOT NULL DEFAULT 'browser'"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_engine", "VARCHAR(32) NOT NULL DEFAULT 'default'"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_headless", "TINYINT(1) NOT NULL DEFAULT 0"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000"); + await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL"); + + if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) { + throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`); + } + if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) { + throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`); + } + + await connection.query(` + INSERT INTO \`${scrapeDb}\`.scrape_sources ( + legacy_catalog_source_id, + source_key, + source_name, + source_type, + base_url, + enabled, + priority, + search_primary_key, + search_fallback_keys, + fallback_source_type, + search_url_template, + fallback_url_template, + search_context_keys, + access_mode, + browser_engine, + browser_headless, + wait_after_load_ms, + note + ) + SELECT + id_product_catalog_source, + source_key, + source_name, + source_type, + base_url, + enabled, + priority, + search_primary_key, + search_fallback_keys, + COALESCE(fallback_source_type, 'none'), + search_url_template, + fallback_url_template, + search_context_keys, + COALESCE(access_mode, 'browser'), + COALESCE(browser_engine, 'default'), + COALESCE(browser_headless, 0), + COALESCE(wait_after_load_ms, 3000), + note + FROM \`${catalogDb}\`.ps_product_catalog_source + ON DUPLICATE KEY UPDATE + source_key = VALUES(source_key), + source_name = VALUES(source_name), + source_type = VALUES(source_type), + base_url = VALUES(base_url), + enabled = VALUES(enabled), + priority = VALUES(priority), + search_primary_key = VALUES(search_primary_key), + search_fallback_keys = VALUES(search_fallback_keys), + fallback_source_type = VALUES(fallback_source_type), + search_url_template = VALUES(search_url_template), + fallback_url_template = VALUES(fallback_url_template), + search_context_keys = VALUES(search_context_keys), + access_mode = VALUES(access_mode), + browser_engine = VALUES(browser_engine), + browser_headless = VALUES(browser_headless), + wait_after_load_ms = VALUES(wait_after_load_ms), + note = VALUES(note) + `); + + await connection.query(` + INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources ( + manufacturer_id, + source_id, + enabled, + priority + ) + SELECT + ms.id_manufacturer, + ss.id, + ms.enabled, + ms.priority + FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms + JOIN \`${scrapeDb}\`.scrape_sources ss + ON ss.legacy_catalog_source_id = ms.id_product_catalog_source + ON DUPLICATE KEY UPDATE + enabled = VALUES(enabled), + priority = VALUES(priority) + `); + + const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`); + const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`); + console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`); +} finally { + await connection.end(); +} diff --git a/sql/local-scrape-db.sql b/sql/local-scrape-db.sql index 8f7ae4c..2b6d532 100644 --- a/sql/local-scrape-db.sql +++ b/sql/local-scrape-db.sql @@ -5,6 +5,7 @@ USE `catalog_scrape`; CREATE TABLE IF NOT EXISTS `scrape_sources` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `legacy_catalog_source_id` BIGINT UNSIGNED NULL, `source_key` VARCHAR(64) NOT NULL, `source_name` VARCHAR(128) NOT NULL, `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier', @@ -13,19 +14,40 @@ CREATE TABLE IF NOT EXISTS `scrape_sources` ( `priority` INT NOT NULL DEFAULT 100, `search_primary_key` VARCHAR(64) NULL, `search_fallback_keys` VARCHAR(255) NULL, + `fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none', `search_url_template` VARCHAR(2048) NULL, `fallback_url_template` VARCHAR(2048) NULL, `search_context_keys` VARCHAR(255) NULL, + `access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser', + `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default', + `browser_headless` TINYINT(1) NOT NULL DEFAULT 0, + `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `scraper_config` JSON NULL, `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `last_health_status` SMALLINT UNSIGNED NULL, `last_health_checked_at` DATETIME NULL, + `note` TEXT NULL, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), + UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`), UNIQUE KEY `uq_scrape_sources_key` (`source_key`) ) ENGINE=InnoDB; +CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `manufacturer_id` INT UNSIGNED NOT NULL, + `source_id` BIGINT UNSIGNED NOT NULL, + `enabled` TINYINT(1) NOT NULL DEFAULT 1, + `priority` INT NOT NULL DEFAULT 100, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`), + KEY `idx_scrape_manufacturer_source_source` (`source_id`), + CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) +) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS `scrape_products` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `manufacturer_id` INT UNSIGNED NULL, diff --git a/src/services/catalog-products.ts b/src/services/catalog-products.ts index 128d244..790acc9 100644 --- a/src/services/catalog-products.ts +++ b/src/services/catalog-products.ts @@ -129,16 +129,23 @@ export async function listSuggestedProducts( export async function listMappingSources(config, { manufacturerId, checkUrls = false }) { try { - const result = await queryEndpoint(config, mappingSourcesSql({ manufacturerId })); + const result = await queryEndpoint( + config, + mappingSourcesSql({ + manufacturerId, + scrapeDatabase: config.database?.scrapeName || "catalog_scrape", + }), + ); + const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.scrape_manufacturer_sources`; const items = result.items.map((row) => - normalizeMappingSource(row, "ps_product_catalog_manufacturer_source"), + normalizeMappingSource(row, mappingTableName), ); const checkedItems = checkUrls ? await checkMappingSourceUrls(items) : items; return { configured: result.configured, mapped: checkedItems.some((item) => item.enabled), - tableName: "ps_product_catalog_manufacturer_source", + tableName: mappingTableName, items: checkedItems, sourceCount: checkedItems.filter((item) => item.enabled).length, testedCount: checkedItems.filter((item) => item.enabled && item.health === "ok").length, diff --git a/src/sql/catalog-maker.ts b/src/sql/catalog-maker.ts index 3b8f42c..a5dae1b 100644 --- a/src/sql/catalog-maker.ts +++ b/src/sql/catalog-maker.ts @@ -382,14 +382,15 @@ export function mappingSourceTablesSql() { `; } -export function mappingSourcesSql({ manufacturerId }) { +export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_scrape" }) { const safeManufacturerId = parsePositiveInteger(manufacturerId, "manufacturerId"); + const scrapeSchema = escapeSqlIdentifier(scrapeDatabase); return ` SELECT - ms.id_product_catalog_manufacturer_source AS mapping_id, - ms.id_manufacturer, - ms.id_product_catalog_source, + ms.id AS mapping_id, + ms.manufacturer_id AS id_manufacturer, + s.id AS id_product_catalog_source, ms.enabled AS mapping_enabled, ms.priority AS mapping_priority, s.source_key, @@ -409,10 +410,10 @@ export function mappingSourcesSql({ manufacturerId }) { s.browser_headless, s.wait_after_load_ms, s.note - FROM ps_product_catalog_manufacturer_source ms - JOIN ps_product_catalog_source s - ON s.id_product_catalog_source = ms.id_product_catalog_source - WHERE ms.id_manufacturer = ${safeManufacturerId} + FROM ${scrapeSchema}.scrape_manufacturer_sources ms + JOIN ${scrapeSchema}.scrape_sources s + ON s.id = ms.source_id + WHERE ms.manufacturer_id = ${safeManufacturerId} ORDER BY ms.priority ASC, s.priority ASC, @@ -420,6 +421,14 @@ export function mappingSourcesSql({ manufacturerId }) { `; } +function escapeSqlIdentifier(value) { + const text = String(value ?? "").trim(); + if (!/^[a-zA-Z0-9_]+$/.test(text)) { + throw new Error("SQL identifier contains unsupported characters."); + } + return `\`${text}\``; +} + function escapeSqlString(value) { return String(value ?? "").replaceAll("\\", "\\\\").replaceAll("'", "''").trim(); }