Move catalog source mapping to scrape database

scripts/migrate-mapping-to-scrape-db.ts

@@ -69,11 +69,104 @@ try {
   await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
   await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
 
-  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) {
-    throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`);
+  const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
+  const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
+
+  if (hasCatalogSourceTable) {
+    await connection.query(`
+      INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
+        id_product_catalog_source,
+        source_key,
+        source_name,
+        source_type,
+        base_url,
+        enabled,
+        priority,
+        search_primary_key,
+        search_fallback_keys,
+        fallback_source_type,
+        search_url_template,
+        fallback_url_template,
+        search_context_keys,
+        access_mode,
+        browser_engine,
+        browser_headless,
+        wait_after_load_ms,
+        note
+      )
+      SELECT
+        id_product_catalog_source,
+        source_key,
+        source_name,
+        source_type,
+        base_url,
+        enabled,
+        priority,
+        search_primary_key,
+        search_fallback_keys,
+        COALESCE(fallback_source_type, 'none'),
+        search_url_template,
+        fallback_url_template,
+        search_context_keys,
+        COALESCE(access_mode, 'browser'),
+        COALESCE(browser_engine, 'default'),
+        COALESCE(browser_headless, 0),
+        COALESCE(wait_after_load_ms, 3000),
+        note
+      FROM \`${catalogDb}\`.ps_product_catalog_source
+      ON DUPLICATE KEY UPDATE
+        source_key = VALUES(source_key),
+        source_name = VALUES(source_name),
+        source_type = VALUES(source_type),
+        base_url = VALUES(base_url),
+        enabled = VALUES(enabled),
+        priority = VALUES(priority),
+        search_primary_key = VALUES(search_primary_key),
+        search_fallback_keys = VALUES(search_fallback_keys),
+        fallback_source_type = VALUES(fallback_source_type),
+        search_url_template = VALUES(search_url_template),
+        fallback_url_template = VALUES(fallback_url_template),
+        search_context_keys = VALUES(search_context_keys),
+        access_mode = VALUES(access_mode),
+        browser_engine = VALUES(browser_engine),
+        browser_headless = VALUES(browser_headless),
+        wait_after_load_ms = VALUES(wait_after_load_ms),
+        note = VALUES(note)
+    `);
   }
-  if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) {
-    throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`);
+
+  if (hasCatalogManufacturerSourceTable) {
+    await connection.query(`
+      INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
+        id_product_catalog_manufacturer_source,
+        id_manufacturer,
+        id_product_catalog_source,
+        enabled,
+        priority,
+        note
+      )
+      SELECT
+        id_product_catalog_manufacturer_source,
+        id_manufacturer,
+        id_product_catalog_source,
+        enabled,
+        priority,
+        note
+      FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
+      ON DUPLICATE KEY UPDATE
+        id_manufacturer = VALUES(id_manufacturer),
+        id_product_catalog_source = VALUES(id_product_catalog_source),
+        enabled = VALUES(enabled),
+        priority = VALUES(priority),
+        note = VALUES(note)
+    `);
+  }
+
+  if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
+    throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
+  }
+  if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
+    throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
   }
 
   await connection.query(`
@@ -116,7 +209,7 @@ try {
       COALESCE(browser_headless, 0),
       COALESCE(wait_after_load_ms, 3000),
       note
-    FROM \`${catalogDb}\`.ps_product_catalog_source
+    FROM \`${scrapeDb}\`.ps_product_catalog_source
     ON DUPLICATE KEY UPDATE
       source_key = VALUES(source_key),
       source_name = VALUES(source_name),
@@ -149,7 +242,7 @@ try {
       ss.id,
       ms.enabled,
       ms.priority
-    FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms
+    FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
     JOIN \`${scrapeDb}\`.scrape_sources ss
       ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
     ON DUPLICATE KEY UPDATE

scripts/start-dbgate.ts

@@ -0,0 +1,54 @@
+import { existsSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { resolve } from "node:path";
+import { loadConfig } from "../src/config.ts";
+
+const config = loadConfig();
+const dbgateBin = resolve("node_modules/.bin/dbgate-serve.cmd");
+const command = existsSync(dbgateBin) ? dbgateBin : "dbgate-serve";
+const host = config.database.host || "127.0.0.1";
+const port = String(config.database.port || 3306);
+const user = config.database.user || "root";
+const password = config.database.password || "";
+const catalogDatabase = config.database.name || "9bplus";
+const scrapeDatabase = config.database.scrapeName || "catalog_scrape";
+const dbgateUrl = new URL(config.dbGate.url || "http://127.0.0.1:3000");
+
+const env = {
+  ...process.env,
+  PORT: dbgateUrl.port || "3000",
+  SKIP_ALL_AUTH: "1",
+  CONNECTIONS: "local_9bplus,local_scrape",
+  ENGINE_local_9bplus: "mysql@dbgate-plugin-mysql",
+  LABEL_local_9bplus: "Local 9bplus DB",
+  SERVER_local_9bplus: host,
+  PORT_local_9bplus: port,
+  USER_local_9bplus: user,
+  PASSWORD_local_9bplus: password,
+  DATABASE_local_9bplus: catalogDatabase,
+  ENGINE_local_scrape: "mysql@dbgate-plugin-mysql",
+  LABEL_local_scrape: "Local scrape DB",
+  SERVER_local_scrape: host,
+  PORT_local_scrape: port,
+  USER_local_scrape: user,
+  PASSWORD_local_scrape: password,
+  DATABASE_local_scrape: scrapeDatabase,
+};
+
+console.log(`Starting DBGate at ${dbgateUrl.origin}`);
+console.log(`Configured connections: Local 9bplus DB, Local scrape DB`);
+
+const child = spawn(command, [], {
+  env,
+  stdio: "inherit",
+  shell: true,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+
+  process.exit(code ?? 0);
+});