From 5210dc2042facda621f4a03b963edc22b827eb5d Mon Sep 17 00:00:00 2001 From: rajch_ales Date: Wed, 5 Aug 2026 23:06:01 +0200 Subject: [PATCH] Move catalog source mapping to scrape database --- agent.md | 1 + package.json | 2 +- scripts/migrate-mapping-to-scrape-db.ts | 105 ++++++++++++++++++++++-- scripts/start-dbgate.ts | 54 ++++++++++++ sql/local-scrape-db.sql | 35 ++++++++ src/services/catalog-products.ts | 2 +- src/sql/catalog-maker.ts | 14 ++-- 7 files changed, 198 insertions(+), 15 deletions(-) create mode 100644 scripts/start-dbgate.ts diff --git a/agent.md b/agent.md index 17d1344..e02d019 100644 --- a/agent.md +++ b/agent.md @@ -108,6 +108,7 @@ There are three selectable data sources: - On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source. - The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases. - The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`. +- `npm run dbgate` must start local-only DBGate without auth prompts and with two predefined local MariaDB connections from `.env`: `Local 9bplus DB` (`DB_NAME`) and `Local scrape DB` (`SCRAPE_DB_NAME`). Do not hardcode or commit passwords; pass them through environment variables at process start. - Start Adminer only through `npm run adminer`; the script downloads the ignored local `tools/adminer/adminer.php` file when missing and runs it on the configured local Adminer URL. Adminer requires local PHP CLI. - Adminer login help may show non-secret local connection fields such as system, host, port, database and user, but must never render or print the actual password. Show password source as `.env DB_PASSWORD` instead. - It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added. diff --git a/package.json b/package.json index 8e6a55e..1c1a142 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "db:create-scrape": "tsx scripts/create-local-scrape-db.ts", "db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts", "adminer": "tsx scripts/start-adminer.ts", - "dbgate": "dbgate-serve", + "dbgate": "tsx scripts/start-dbgate.ts", "test": "tsx --test test/*.test.ts", "test:ui": "playwright test", "test:e2e": "playwright test tests/ui/database.e2e.spec.ts", diff --git a/scripts/migrate-mapping-to-scrape-db.ts b/scripts/migrate-mapping-to-scrape-db.ts index f3c84a6..518d5ed 100644 --- a/scripts/migrate-mapping-to-scrape-db.ts +++ b/scripts/migrate-mapping-to-scrape-db.ts @@ -69,11 +69,104 @@ try { await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000"); await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL"); - if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) { - throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`); + const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source"); + const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"); + + if (hasCatalogSourceTable) { + await connection.query(` + INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source ( + id_product_catalog_source, + source_key, + source_name, + source_type, + base_url, + enabled, + priority, + search_primary_key, + search_fallback_keys, + fallback_source_type, + search_url_template, + fallback_url_template, + search_context_keys, + access_mode, + browser_engine, + browser_headless, + wait_after_load_ms, + note + ) + SELECT + id_product_catalog_source, + source_key, + source_name, + source_type, + base_url, + enabled, + priority, + search_primary_key, + search_fallback_keys, + COALESCE(fallback_source_type, 'none'), + search_url_template, + fallback_url_template, + search_context_keys, + COALESCE(access_mode, 'browser'), + COALESCE(browser_engine, 'default'), + COALESCE(browser_headless, 0), + COALESCE(wait_after_load_ms, 3000), + note + FROM \`${catalogDb}\`.ps_product_catalog_source + ON DUPLICATE KEY UPDATE + source_key = VALUES(source_key), + source_name = VALUES(source_name), + source_type = VALUES(source_type), + base_url = VALUES(base_url), + enabled = VALUES(enabled), + priority = VALUES(priority), + search_primary_key = VALUES(search_primary_key), + search_fallback_keys = VALUES(search_fallback_keys), + fallback_source_type = VALUES(fallback_source_type), + search_url_template = VALUES(search_url_template), + fallback_url_template = VALUES(fallback_url_template), + search_context_keys = VALUES(search_context_keys), + access_mode = VALUES(access_mode), + browser_engine = VALUES(browser_engine), + browser_headless = VALUES(browser_headless), + wait_after_load_ms = VALUES(wait_after_load_ms), + note = VALUES(note) + `); } - if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) { - throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`); + + if (hasCatalogManufacturerSourceTable) { + await connection.query(` + INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ( + id_product_catalog_manufacturer_source, + id_manufacturer, + id_product_catalog_source, + enabled, + priority, + note + ) + SELECT + id_product_catalog_manufacturer_source, + id_manufacturer, + id_product_catalog_source, + enabled, + priority, + note + FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source + ON DUPLICATE KEY UPDATE + id_manufacturer = VALUES(id_manufacturer), + id_product_catalog_source = VALUES(id_product_catalog_source), + enabled = VALUES(enabled), + priority = VALUES(priority), + note = VALUES(note) + `); + } + + if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) { + throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`); + } + if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) { + throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`); } await connection.query(` @@ -116,7 +209,7 @@ try { COALESCE(browser_headless, 0), COALESCE(wait_after_load_ms, 3000), note - FROM \`${catalogDb}\`.ps_product_catalog_source + FROM \`${scrapeDb}\`.ps_product_catalog_source ON DUPLICATE KEY UPDATE source_key = VALUES(source_key), source_name = VALUES(source_name), @@ -149,7 +242,7 @@ try { ss.id, ms.enabled, ms.priority - FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms + FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms JOIN \`${scrapeDb}\`.scrape_sources ss ON ss.legacy_catalog_source_id = ms.id_product_catalog_source ON DUPLICATE KEY UPDATE diff --git a/scripts/start-dbgate.ts b/scripts/start-dbgate.ts new file mode 100644 index 0000000..b765d41 --- /dev/null +++ b/scripts/start-dbgate.ts @@ -0,0 +1,54 @@ +import { existsSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { resolve } from "node:path"; +import { loadConfig } from "../src/config.ts"; + +const config = loadConfig(); +const dbgateBin = resolve("node_modules/.bin/dbgate-serve.cmd"); +const command = existsSync(dbgateBin) ? dbgateBin : "dbgate-serve"; +const host = config.database.host || "127.0.0.1"; +const port = String(config.database.port || 3306); +const user = config.database.user || "root"; +const password = config.database.password || ""; +const catalogDatabase = config.database.name || "9bplus"; +const scrapeDatabase = config.database.scrapeName || "catalog_scrape"; +const dbgateUrl = new URL(config.dbGate.url || "http://127.0.0.1:3000"); + +const env = { + ...process.env, + PORT: dbgateUrl.port || "3000", + SKIP_ALL_AUTH: "1", + CONNECTIONS: "local_9bplus,local_scrape", + ENGINE_local_9bplus: "mysql@dbgate-plugin-mysql", + LABEL_local_9bplus: "Local 9bplus DB", + SERVER_local_9bplus: host, + PORT_local_9bplus: port, + USER_local_9bplus: user, + PASSWORD_local_9bplus: password, + DATABASE_local_9bplus: catalogDatabase, + ENGINE_local_scrape: "mysql@dbgate-plugin-mysql", + LABEL_local_scrape: "Local scrape DB", + SERVER_local_scrape: host, + PORT_local_scrape: port, + USER_local_scrape: user, + PASSWORD_local_scrape: password, + DATABASE_local_scrape: scrapeDatabase, +}; + +console.log(`Starting DBGate at ${dbgateUrl.origin}`); +console.log(`Configured connections: Local 9bplus DB, Local scrape DB`); + +const child = spawn(command, [], { + env, + stdio: "inherit", + shell: true, +}); + +child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + + process.exit(code ?? 0); +}); diff --git a/sql/local-scrape-db.sql b/sql/local-scrape-db.sql index 2b6d532..7087883 100644 --- a/sql/local-scrape-db.sql +++ b/sql/local-scrape-db.sql @@ -3,6 +3,41 @@ CREATE DATABASE IF NOT EXISTS `catalog_scrape` USE `catalog_scrape`; +CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` ( + `id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `source_key` VARCHAR(64) NOT NULL, + `source_name` VARCHAR(128) NOT NULL, + `source_type` VARCHAR(32) NOT NULL DEFAULT '', + `base_url` VARCHAR(512) NOT NULL, + `enabled` TINYINT(1) NOT NULL DEFAULT 1, + `priority` INT NOT NULL DEFAULT 100, + `search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean', + `search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '', + `fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none', + `search_url_template` VARCHAR(1000) NOT NULL, + `fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '', + `search_context_keys` VARCHAR(255) NOT NULL DEFAULT '', + `access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser', + `browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default', + `browser_headless` TINYINT(1) NOT NULL DEFAULT 0, + `wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000, + `note` TEXT NULL, + PRIMARY KEY (`id_product_catalog_source`), + UNIQUE KEY `uq_product_catalog_source_key` (`source_key`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` ( + `id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `id_manufacturer` INT UNSIGNED NOT NULL, + `id_product_catalog_source` INT UNSIGNED NOT NULL, + `enabled` TINYINT(1) NOT NULL DEFAULT 1, + `priority` INT NOT NULL DEFAULT 100, + `note` TEXT NULL, + PRIMARY KEY (`id_product_catalog_manufacturer_source`), + UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`), + KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`) +) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS `scrape_sources` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `legacy_catalog_source_id` BIGINT UNSIGNED NULL, diff --git a/src/services/catalog-products.ts b/src/services/catalog-products.ts index 790acc9..a7aab45 100644 --- a/src/services/catalog-products.ts +++ b/src/services/catalog-products.ts @@ -136,7 +136,7 @@ export async function listMappingSources(config, { manufacturerId, checkUrls = f scrapeDatabase: config.database?.scrapeName || "catalog_scrape", }), ); - const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.scrape_manufacturer_sources`; + const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.ps_product_catalog_manufacturer_source`; const items = result.items.map((row) => normalizeMappingSource(row, mappingTableName), ); diff --git a/src/sql/catalog-maker.ts b/src/sql/catalog-maker.ts index a5dae1b..b3b64b1 100644 --- a/src/sql/catalog-maker.ts +++ b/src/sql/catalog-maker.ts @@ -388,9 +388,9 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc return ` SELECT - ms.id AS mapping_id, - ms.manufacturer_id AS id_manufacturer, - s.id AS id_product_catalog_source, + ms.id_product_catalog_manufacturer_source AS mapping_id, + ms.id_manufacturer, + s.id_product_catalog_source, ms.enabled AS mapping_enabled, ms.priority AS mapping_priority, s.source_key, @@ -410,10 +410,10 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc s.browser_headless, s.wait_after_load_ms, s.note - FROM ${scrapeSchema}.scrape_manufacturer_sources ms - JOIN ${scrapeSchema}.scrape_sources s - ON s.id = ms.source_id - WHERE ms.manufacturer_id = ${safeManufacturerId} + FROM ${scrapeSchema}.ps_product_catalog_manufacturer_source ms + JOIN ${scrapeSchema}.ps_product_catalog_source s + ON s.id_product_catalog_source = ms.id_product_catalog_source + WHERE ms.id_manufacturer = ${safeManufacturerId} ORDER BY ms.priority ASC, s.priority ASC,