Move catalog source mapping to scrape database

agent.md

@@ -108,6 +108,7 @@ There are three selectable data sources:
 - On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source.
 - The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
 - The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`.
+- `npm run dbgate` must start local-only DBGate without auth prompts and with two predefined local MariaDB connections from `.env`: `Local 9bplus DB` (`DB_NAME`) and `Local scrape DB` (`SCRAPE_DB_NAME`). Do not hardcode or commit passwords; pass them through environment variables at process start.
 - Start Adminer only through `npm run adminer`; the script downloads the ignored local `tools/adminer/adminer.php` file when missing and runs it on the configured local Adminer URL. Adminer requires local PHP CLI.
 - Adminer login help may show non-secret local connection fields such as system, host, port, database and user, but must never render or print the actual password. Show password source as `.env DB_PASSWORD` instead.
 - It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.

package.json

@@ -11,7 +11,7 @@
     "db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
     "db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
     "adminer": "tsx scripts/start-adminer.ts",
-    "dbgate": "dbgate-serve",
+    "dbgate": "tsx scripts/start-dbgate.ts",
     "test": "tsx --test test/*.test.ts",
     "test:ui": "playwright test",
     "test:e2e": "playwright test tests/ui/database.e2e.spec.ts",

scripts/migrate-mapping-to-scrape-db.ts

@@ -69,16 +69,13 @@ try {
   await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
   await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
 
-  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) {
+  const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
-    throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`);
+  const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
-  }
-  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) {
-    throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`);
-  }
 
+  if (hasCatalogSourceTable) {
     await connection.query(`
-    INSERT INTO \`${scrapeDb}\`.scrape_sources (
+      INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
-      legacy_catalog_source_id,
+        id_product_catalog_source,
         source_key,
         source_name,
         source_type,
@@ -136,6 +133,102 @@ try {
         wait_after_load_ms = VALUES(wait_after_load_ms),
         note = VALUES(note)
     `);
+  }
+
+  if (hasCatalogManufacturerSourceTable) {
+    await connection.query(`
+      INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
+        id_product_catalog_manufacturer_source,
+        id_manufacturer,
+        id_product_catalog_source,
+        enabled,
+        priority,
+        note
+      )
+      SELECT
+        id_product_catalog_manufacturer_source,
+        id_manufacturer,
+        id_product_catalog_source,
+        enabled,
+        priority,
+        note
+      FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
+      ON DUPLICATE KEY UPDATE
+        id_manufacturer = VALUES(id_manufacturer),
+        id_product_catalog_source = VALUES(id_product_catalog_source),
+        enabled = VALUES(enabled),
+        priority = VALUES(priority),
+        note = VALUES(note)
+    `);
+  }
+
+  if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
+    throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
+  }
+  if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
+    throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
+  }
+
+  await connection.query(`
+    INSERT INTO \`${scrapeDb}\`.scrape_sources (
+      legacy_catalog_source_id,
+      source_key,
+      source_name,
+      source_type,
+      base_url,
+      enabled,
+      priority,
+      search_primary_key,
+      search_fallback_keys,
+      fallback_source_type,
+      search_url_template,
+      fallback_url_template,
+      search_context_keys,
+      access_mode,
+      browser_engine,
+      browser_headless,
+      wait_after_load_ms,
+      note
+    )
+    SELECT
+      id_product_catalog_source,
+      source_key,
+      source_name,
+      source_type,
+      base_url,
+      enabled,
+      priority,
+      search_primary_key,
+      search_fallback_keys,
+      COALESCE(fallback_source_type, 'none'),
+      search_url_template,
+      fallback_url_template,
+      search_context_keys,
+      COALESCE(access_mode, 'browser'),
+      COALESCE(browser_engine, 'default'),
+      COALESCE(browser_headless, 0),
+      COALESCE(wait_after_load_ms, 3000),
+      note
+    FROM \`${scrapeDb}\`.ps_product_catalog_source
+    ON DUPLICATE KEY UPDATE
+      source_key = VALUES(source_key),
+      source_name = VALUES(source_name),
+      source_type = VALUES(source_type),
+      base_url = VALUES(base_url),
+      enabled = VALUES(enabled),
+      priority = VALUES(priority),
+      search_primary_key = VALUES(search_primary_key),
+      search_fallback_keys = VALUES(search_fallback_keys),
+      fallback_source_type = VALUES(fallback_source_type),
+      search_url_template = VALUES(search_url_template),
+      fallback_url_template = VALUES(fallback_url_template),
+      search_context_keys = VALUES(search_context_keys),
+      access_mode = VALUES(access_mode),
+      browser_engine = VALUES(browser_engine),
+      browser_headless = VALUES(browser_headless),
+      wait_after_load_ms = VALUES(wait_after_load_ms),
+      note = VALUES(note)
+  `);
 
   await connection.query(`
     INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources (
@@ -149,7 +242,7 @@ try {
       ss.id,
       ms.enabled,
       ms.priority
-    FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms
+    FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
     JOIN \`${scrapeDb}\`.scrape_sources ss
       ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
     ON DUPLICATE KEY UPDATE

scripts/start-dbgate.ts

@@ -0,0 +1,54 @@
+import { existsSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { resolve } from "node:path";
+import { loadConfig } from "../src/config.ts";
+
+const config = loadConfig();
+const dbgateBin = resolve("node_modules/.bin/dbgate-serve.cmd");
+const command = existsSync(dbgateBin) ? dbgateBin : "dbgate-serve";
+const host = config.database.host || "127.0.0.1";
+const port = String(config.database.port || 3306);
+const user = config.database.user || "root";
+const password = config.database.password || "";
+const catalogDatabase = config.database.name || "9bplus";
+const scrapeDatabase = config.database.scrapeName || "catalog_scrape";
+const dbgateUrl = new URL(config.dbGate.url || "http://127.0.0.1:3000");
+
+const env = {
+  ...process.env,
+  PORT: dbgateUrl.port || "3000",
+  SKIP_ALL_AUTH: "1",
+  CONNECTIONS: "local_9bplus,local_scrape",
+  ENGINE_local_9bplus: "mysql@dbgate-plugin-mysql",
+  LABEL_local_9bplus: "Local 9bplus DB",
+  SERVER_local_9bplus: host,
+  PORT_local_9bplus: port,
+  USER_local_9bplus: user,
+  PASSWORD_local_9bplus: password,
+  DATABASE_local_9bplus: catalogDatabase,
+  ENGINE_local_scrape: "mysql@dbgate-plugin-mysql",
+  LABEL_local_scrape: "Local scrape DB",
+  SERVER_local_scrape: host,
+  PORT_local_scrape: port,
+  USER_local_scrape: user,
+  PASSWORD_local_scrape: password,
+  DATABASE_local_scrape: scrapeDatabase,
+};
+
+console.log(`Starting DBGate at ${dbgateUrl.origin}`);
+console.log(`Configured connections: Local 9bplus DB, Local scrape DB`);
+
+const child = spawn(command, [], {
+  env,
+  stdio: "inherit",
+  shell: true,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+
+  process.exit(code ?? 0);
+});

sql/local-scrape-db.sql

@@ -3,6 +3,41 @@ CREATE DATABASE IF NOT EXISTS `catalog_scrape`
 
 USE `catalog_scrape`;
 
+CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` (
+  `id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `source_key` VARCHAR(64) NOT NULL,
+  `source_name` VARCHAR(128) NOT NULL,
+  `source_type` VARCHAR(32) NOT NULL DEFAULT '',
+  `base_url` VARCHAR(512) NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean',
+  `search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '',
+  `fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none',
+  `search_url_template` VARCHAR(1000) NOT NULL,
+  `fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '',
+  `search_context_keys` VARCHAR(255) NOT NULL DEFAULT '',
+  `access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser',
+  `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
+  `browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
+  `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
+  `note` TEXT NULL,
+  PRIMARY KEY (`id_product_catalog_source`),
+  UNIQUE KEY `uq_product_catalog_source_key` (`source_key`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` (
+  `id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `id_manufacturer` INT UNSIGNED NOT NULL,
+  `id_product_catalog_source` INT UNSIGNED NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `note` TEXT NULL,
+  PRIMARY KEY (`id_product_catalog_manufacturer_source`),
+  UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`),
+  KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`)
+) ENGINE=InnoDB;
+
 CREATE TABLE IF NOT EXISTS `scrape_sources` (
   `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
   `legacy_catalog_source_id` BIGINT UNSIGNED NULL,

src/services/catalog-products.ts

@@ -136,7 +136,7 @@ export async function listMappingSources(config, { manufacturerId, checkUrls = f
         scrapeDatabase: config.database?.scrapeName || "catalog_scrape",
       }),
     );
-    const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.scrape_manufacturer_sources`;
+    const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.ps_product_catalog_manufacturer_source`;
     const items = result.items.map((row) =>
       normalizeMappingSource(row, mappingTableName),
     );

src/sql/catalog-maker.ts

@@ -388,9 +388,9 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
 
   return `
     SELECT
-      ms.id AS mapping_id,
+      ms.id_product_catalog_manufacturer_source AS mapping_id,
-      ms.manufacturer_id AS id_manufacturer,
+      ms.id_manufacturer,
-      s.id AS id_product_catalog_source,
+      s.id_product_catalog_source,
       ms.enabled AS mapping_enabled,
       ms.priority AS mapping_priority,
       s.source_key,
@@ -410,10 +410,10 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
       s.browser_headless,
       s.wait_after_load_ms,
       s.note
-    FROM ${scrapeSchema}.scrape_manufacturer_sources ms
+    FROM ${scrapeSchema}.ps_product_catalog_manufacturer_source ms
-    JOIN ${scrapeSchema}.scrape_sources s
+    JOIN ${scrapeSchema}.ps_product_catalog_source s
-      ON s.id = ms.source_id
+      ON s.id_product_catalog_source = ms.id_product_catalog_source
-    WHERE ms.manufacturer_id = ${safeManufacturerId}
+    WHERE ms.id_manufacturer = ${safeManufacturerId}
     ORDER BY
       ms.priority ASC,
       s.priority ASC,