Move source mapping to scrape database

agent.md

@@ -10,7 +10,8 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
 - Local DB may later allow controlled writes, but writes must remain explicit, logged and protected by transaction/dry-run settings.
 - Never print, commit or share endpoint tokens, usernames or passwords.
 - Never use Live DB as a fallback when Local DB is selected. If Local DB is unavailable, show an error and keep the UI running.
-- The selected database mode is authoritative for every database call. When `Local 9bplus DB` is selected, manufacturers, languages, products, combinations, mapping, source configuration and all other catalog reads must use the local MariaDB connection; no endpoint or Live DB call is allowed. When `Live 9bplus DB` is selected, every catalog call must use the existing endpoint and remain read-only. Do not mix sources within one workflow.
+- The selected catalog database mode is authoritative for catalog data calls. When `Local 9bplus DB` is selected, manufacturers, languages, products, combinations and all other catalog reads must use the local MariaDB connection; no endpoint or Live DB call is allowed. When `Live 9bplus DB` is selected, catalog reads use the existing endpoint and remain read-only.
+- Mapping, source configuration, scraper settings, browser-engine choices and scraped draft data belong to `Local scrape DB` (`catalog_scrape`) and must be read from there, not from Live DB or the product catalog database.
 - Keep the development server on port `3404` unless the user explicitly requests another port.
 - Do not delete or revert user files or unrelated changes.
 
@@ -102,9 +103,11 @@ There are three selectable data sources:
 ### Local scrape DB
 
 - Uses the separate local MariaDB database named `catalog_scrape`.
-- Stores scraper source configuration, scrape runs, discovered products and assets.
+- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
+- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
 - It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
 - The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
+- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
 
 ### Local MariaDB connection
 
@@ -124,7 +127,7 @@ There are three selectable data sources:
 - The backend must resolve that header before every API database operation and route the complete request through the selected source. A Local DB request must use `queryMariaDb`; a Live DB request must use `queryEndpoint`; there is no silent fallback or mixed-source response.
 - Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint.
 - Backend data API routes reject a Live DB request when the project is configured only for MariaDB.
-- Mapping endpoints follow the same rule as product endpoints and must read mapping tables from the selected database.
+- Product endpoints read catalog data from the selected catalog database. Mapping endpoints read scraper source mapping from Local scrape DB while still using the selected catalog database only for catalog/product data.
 - Static HTML/CSS/JS files must remain available even when a database is not configured.
 - The top status shows the selected source, connection state, permissions and execution mode.
 - Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI.

package.json

@@ -9,6 +9,7 @@
     "dev": "tsx src/server.ts",
     "catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
     "db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
+    "db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
     "test": "tsx --test test/*.test.ts",
     "test:ui": "playwright test",
     "test:e2e": "playwright test tests/ui/database.e2e.spec.ts",

scripts/migrate-mapping-to-scrape-db.ts

@@ -0,0 +1,165 @@
+import fs from "node:fs/promises";
+import mariadb from "mariadb";
+
+async function readEnv() {
+  return Object.fromEntries(
+    (await fs.readFile(".env", "utf8"))
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line && !line.startsWith("#") && line.includes("="))
+      .map((line) => {
+        const index = line.indexOf("=");
+        return [line.slice(0, index).trim(), line.slice(index + 1).trim()];
+      }),
+  );
+}
+
+async function queryRows(connection, sql, params = []) {
+  const rows = await connection.query(sql, params);
+  return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
+}
+
+async function tableExists(connection, schema, table) {
+  const rows = await queryRows(
+    connection,
+    "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1",
+    [schema, table],
+  );
+  return rows.length > 0;
+}
+
+async function columnExists(connection, schema, table, column) {
+  const rows = await queryRows(
+    connection,
+    "SELECT 1 FROM information_schema.columns WHERE table_schema = ? AND table_name = ? AND column_name = ? LIMIT 1",
+    [schema, table, column],
+  );
+  return rows.length > 0;
+}
+
+async function addColumnIfMissing(connection, schema, table, column, definition) {
+  if (await columnExists(connection, schema, table, column)) return;
+  await connection.query(`ALTER TABLE \`${schema}\`.\`${table}\` ADD COLUMN \`${column}\` ${definition}`);
+}
+
+const env = await readEnv();
+const catalogDb = env.DB_NAME || "9bplus";
+const scrapeDb = env.SCRAPE_DB_NAME || "catalog_scrape";
+const connection = await mariadb.createConnection({
+  host: env.DB_HOST || "127.0.0.1",
+  port: Number(env.DB_PORT || 3306),
+  user: env.DB_USER || "root",
+  password: env.DB_PASSWORD || "",
+  multipleStatements: false,
+  bigIntAsNumber: true,
+});
+
+try {
+  await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
+  const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
+  for (const statement of schemaSql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) {
+    await connection.query(statement);
+  }
+
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "legacy_catalog_source_id", "BIGINT UNSIGNED NULL");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "fallback_source_type", "VARCHAR(64) NOT NULL DEFAULT 'none'");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "access_mode", "VARCHAR(64) NOT NULL DEFAULT 'browser'");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_engine", "VARCHAR(32) NOT NULL DEFAULT 'default'");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_headless", "TINYINT(1) NOT NULL DEFAULT 0");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
+  await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
+
+  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) {
+    throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`);
+  }
+  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) {
+    throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`);
+  }
+
+  await connection.query(`
+    INSERT INTO \`${scrapeDb}\`.scrape_sources (
+      legacy_catalog_source_id,
+      source_key,
+      source_name,
+      source_type,
+      base_url,
+      enabled,
+      priority,
+      search_primary_key,
+      search_fallback_keys,
+      fallback_source_type,
+      search_url_template,
+      fallback_url_template,
+      search_context_keys,
+      access_mode,
+      browser_engine,
+      browser_headless,
+      wait_after_load_ms,
+      note
+    )
+    SELECT
+      id_product_catalog_source,
+      source_key,
+      source_name,
+      source_type,
+      base_url,
+      enabled,
+      priority,
+      search_primary_key,
+      search_fallback_keys,
+      COALESCE(fallback_source_type, 'none'),
+      search_url_template,
+      fallback_url_template,
+      search_context_keys,
+      COALESCE(access_mode, 'browser'),
+      COALESCE(browser_engine, 'default'),
+      COALESCE(browser_headless, 0),
+      COALESCE(wait_after_load_ms, 3000),
+      note
+    FROM \`${catalogDb}\`.ps_product_catalog_source
+    ON DUPLICATE KEY UPDATE
+      source_key = VALUES(source_key),
+      source_name = VALUES(source_name),
+      source_type = VALUES(source_type),
+      base_url = VALUES(base_url),
+      enabled = VALUES(enabled),
+      priority = VALUES(priority),
+      search_primary_key = VALUES(search_primary_key),
+      search_fallback_keys = VALUES(search_fallback_keys),
+      fallback_source_type = VALUES(fallback_source_type),
+      search_url_template = VALUES(search_url_template),
+      fallback_url_template = VALUES(fallback_url_template),
+      search_context_keys = VALUES(search_context_keys),
+      access_mode = VALUES(access_mode),
+      browser_engine = VALUES(browser_engine),
+      browser_headless = VALUES(browser_headless),
+      wait_after_load_ms = VALUES(wait_after_load_ms),
+      note = VALUES(note)
+  `);
+
+  await connection.query(`
+    INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources (
+      manufacturer_id,
+      source_id,
+      enabled,
+      priority
+    )
+    SELECT
+      ms.id_manufacturer,
+      ss.id,
+      ms.enabled,
+      ms.priority
+    FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms
+    JOIN \`${scrapeDb}\`.scrape_sources ss
+      ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
+    ON DUPLICATE KEY UPDATE
+      enabled = VALUES(enabled),
+      priority = VALUES(priority)
+  `);
+
+  const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`);
+  const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`);
+  console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`);
+} finally {
+  await connection.end();
+}

sql/local-scrape-db.sql

@@ -5,6 +5,7 @@ USE `catalog_scrape`;
 
 CREATE TABLE IF NOT EXISTS `scrape_sources` (
   `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `legacy_catalog_source_id` BIGINT UNSIGNED NULL,
   `source_key` VARCHAR(64) NOT NULL,
   `source_name` VARCHAR(128) NOT NULL,
   `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
@@ -13,19 +14,40 @@ CREATE TABLE IF NOT EXISTS `scrape_sources` (
   `priority` INT NOT NULL DEFAULT 100,
   `search_primary_key` VARCHAR(64) NULL,
   `search_fallback_keys` VARCHAR(255) NULL,
+  `fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none',
   `search_url_template` VARCHAR(2048) NULL,
   `fallback_url_template` VARCHAR(2048) NULL,
   `search_context_keys` VARCHAR(255) NULL,
+  `access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser',
+  `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
+  `browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
+  `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
   `scraper_config` JSON NULL,
   `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
   `last_health_status` SMALLINT UNSIGNED NULL,
   `last_health_checked_at` DATETIME NULL,
+  `note` TEXT NULL,
   `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`),
   UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
 ) ENGINE=InnoDB;
 
+CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `manufacturer_id` INT UNSIGNED NOT NULL,
+  `source_id` BIGINT UNSIGNED NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`),
+  KEY `idx_scrape_manufacturer_source_source` (`source_id`),
+  CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
+) ENGINE=InnoDB;
+
 CREATE TABLE IF NOT EXISTS `scrape_products` (
   `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
   `manufacturer_id` INT UNSIGNED NULL,

src/services/catalog-products.ts

@@ -129,16 +129,23 @@ export async function listSuggestedProducts(
 
 export async function listMappingSources(config, { manufacturerId, checkUrls = false }) {
   try {
-    const result = await queryEndpoint(config, mappingSourcesSql({ manufacturerId }));
+    const result = await queryEndpoint(
+      config,
+      mappingSourcesSql({
+        manufacturerId,
+        scrapeDatabase: config.database?.scrapeName || "catalog_scrape",
+      }),
+    );
+    const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.scrape_manufacturer_sources`;
     const items = result.items.map((row) =>
-      normalizeMappingSource(row, "ps_product_catalog_manufacturer_source"),
+      normalizeMappingSource(row, mappingTableName),
     );
     const checkedItems = checkUrls ? await checkMappingSourceUrls(items) : items;
 
     return {
       configured: result.configured,
       mapped: checkedItems.some((item) => item.enabled),
-      tableName: "ps_product_catalog_manufacturer_source",
+      tableName: mappingTableName,
       items: checkedItems,
       sourceCount: checkedItems.filter((item) => item.enabled).length,
       testedCount: checkedItems.filter((item) => item.enabled && item.health === "ok").length,

src/sql/catalog-maker.ts

@@ -382,14 +382,15 @@ export function mappingSourceTablesSql() {
   `;
 }
 
-export function mappingSourcesSql({ manufacturerId }) {
+export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_scrape" }) {
   const safeManufacturerId = parsePositiveInteger(manufacturerId, "manufacturerId");
+  const scrapeSchema = escapeSqlIdentifier(scrapeDatabase);
 
   return `
     SELECT
-      ms.id_product_catalog_manufacturer_source AS mapping_id,
-      ms.id_manufacturer,
-      ms.id_product_catalog_source,
+      ms.id AS mapping_id,
+      ms.manufacturer_id AS id_manufacturer,
+      s.id AS id_product_catalog_source,
       ms.enabled AS mapping_enabled,
       ms.priority AS mapping_priority,
       s.source_key,
@@ -409,10 +410,10 @@ export function mappingSourcesSql({ manufacturerId }) {
       s.browser_headless,
       s.wait_after_load_ms,
       s.note
-    FROM ps_product_catalog_manufacturer_source ms
-    JOIN ps_product_catalog_source s
-      ON s.id_product_catalog_source = ms.id_product_catalog_source
-    WHERE ms.id_manufacturer = ${safeManufacturerId}
+    FROM ${scrapeSchema}.scrape_manufacturer_sources ms
+    JOIN ${scrapeSchema}.scrape_sources s
+      ON s.id = ms.source_id
+    WHERE ms.manufacturer_id = ${safeManufacturerId}
     ORDER BY
       ms.priority ASC,
       s.priority ASC,
@@ -420,6 +421,14 @@ export function mappingSourcesSql({ manufacturerId }) {
   `;
 }
 
+function escapeSqlIdentifier(value) {
+  const text = String(value ?? "").trim();
+  if (!/^[a-zA-Z0-9_]+$/.test(text)) {
+    throw new Error("SQL identifier contains unsupported characters.");
+  }
+  return `\`${text}\``;
+}
+
 function escapeSqlString(value) {
   return String(value ?? "").replaceAll("\\", "\\\\").replaceAll("'", "''").trim();
 }