rajch_ales/product-import-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move catalog source mapping to scrape database

Browse Source
This commit is contained in:
rajch_ales
2026-08-05 23:06:01 +02:00
parent e3caf25823
commit 5210dc2042
7 changed files with 198 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent.md
+1
View File
@@ -108,6 +108,7 @@ There are three selectable data sources:
- On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source.
- The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
- The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`.
- `npm run dbgate` must start local-only DBGate without auth prompts and with two predefined local MariaDB connections from `.env`: `Local 9bplus DB` (`DB_NAME`) and `Local scrape DB` (`SCRAPE_DB_NAME`). Do not hardcode or commit passwords; pass them through environment variables at process start.
- Start Adminer only through `npm run adminer`; the script downloads the ignored local `tools/adminer/adminer.php` file when missing and runs it on the configured local Adminer URL. Adminer requires local PHP CLI.
- Adminer login help may show non-secret local connection fields such as system, host, port, database and user, but must never render or print the actual password. Show password source as `.env DB_PASSWORD` instead.
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
"db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
"adminer": "tsx scripts/start-adminer.ts",
"dbgate": "dbgate-serve",
"dbgate": "tsx scripts/start-dbgate.ts",
"test": "tsx --test test/*.test.ts",
"test:ui": "playwright test",
"test:e2e": "playwright test tests/ui/database.e2e.spec.ts",
scripts/migrate-mapping-to-scrape-db.ts
+102 -9
View File
@@ -69,16 +69,13 @@ try {
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
if (!(await tableExists(connection, catalogDb, "ps_product_catalog_source"))) {
throw new Error(`Source table ${catalogDb}.ps_product_catalog_source does not exist.`);
}
if (!(await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source"))) {
throw new Error(`Mapping table ${catalogDb}.ps_product_catalog_manufacturer_source does not exist.`);
}
const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
if (hasCatalogSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.scrape_sources (
legacy_catalog_source_id,
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
id_product_catalog_source,
source_key,
source_name,
source_type,
@@ -136,6 +133,102 @@ try {
wait_after_load_ms = VALUES(wait_after_load_ms),
note = VALUES(note)
`);
}
if (hasCatalogManufacturerSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
)
SELECT
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
ON DUPLICATE KEY UPDATE
id_manufacturer = VALUES(id_manufacturer),
id_product_catalog_source = VALUES(id_product_catalog_source),
enabled = VALUES(enabled),
priority = VALUES(priority),
note = VALUES(note)
`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
}
await connection.query(`
INSERT INTO \`${scrapeDb}\`.scrape_sources (
legacy_catalog_source_id,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
fallback_source_type,
search_url_template,
fallback_url_template,
search_context_keys,
access_mode,
browser_engine,
browser_headless,
wait_after_load_ms,
note
)
SELECT
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
COALESCE(fallback_source_type, 'none'),
search_url_template,
fallback_url_template,
search_context_keys,
COALESCE(access_mode, 'browser'),
COALESCE(browser_engine, 'default'),
COALESCE(browser_headless, 0),
COALESCE(wait_after_load_ms, 3000),
note
FROM \`${scrapeDb}\`.ps_product_catalog_source
ON DUPLICATE KEY UPDATE
source_key = VALUES(source_key),
source_name = VALUES(source_name),
source_type = VALUES(source_type),
base_url = VALUES(base_url),
enabled = VALUES(enabled),
priority = VALUES(priority),
search_primary_key = VALUES(search_primary_key),
search_fallback_keys = VALUES(search_fallback_keys),
fallback_source_type = VALUES(fallback_source_type),
search_url_template = VALUES(search_url_template),
fallback_url_template = VALUES(fallback_url_template),
search_context_keys = VALUES(search_context_keys),
access_mode = VALUES(access_mode),
browser_engine = VALUES(browser_engine),
browser_headless = VALUES(browser_headless),
wait_after_load_ms = VALUES(wait_after_load_ms),
note = VALUES(note)
`);
await connection.query(`
INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources (
@@ -149,7 +242,7 @@ try {
ss.id,
ms.enabled,
ms.priority
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source ms
FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
JOIN \`${scrapeDb}\`.scrape_sources ss
ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
ON DUPLICATE KEY UPDATE
scripts/start-dbgate.ts
+54
View File
@@ -0,0 +1,54 @@
import { existsSync } from "node:fs";
import { spawn } from "node:child_process";
import { resolve } from "node:path";
import { loadConfig } from "../src/config.ts";
const config = loadConfig();
const dbgateBin = resolve("node_modules/.bin/dbgate-serve.cmd");
const command = existsSync(dbgateBin) ? dbgateBin : "dbgate-serve";
const host = config.database.host || "127.0.0.1";
const port = String(config.database.port || 3306);
const user = config.database.user || "root";
const password = config.database.password || "";
const catalogDatabase = config.database.name || "9bplus";
const scrapeDatabase = config.database.scrapeName || "catalog_scrape";
const dbgateUrl = new URL(config.dbGate.url || "http://127.0.0.1:3000");
const env = {
...process.env,
PORT: dbgateUrl.port || "3000",
SKIP_ALL_AUTH: "1",
CONNECTIONS: "local_9bplus,local_scrape",
ENGINE_local_9bplus: "mysql@dbgate-plugin-mysql",
LABEL_local_9bplus: "Local 9bplus DB",
SERVER_local_9bplus: host,
PORT_local_9bplus: port,
USER_local_9bplus: user,
PASSWORD_local_9bplus: password,
DATABASE_local_9bplus: catalogDatabase,
ENGINE_local_scrape: "mysql@dbgate-plugin-mysql",
LABEL_local_scrape: "Local scrape DB",
SERVER_local_scrape: host,
PORT_local_scrape: port,
USER_local_scrape: user,
PASSWORD_local_scrape: password,
DATABASE_local_scrape: scrapeDatabase,
};
console.log(`Starting DBGate at ${dbgateUrl.origin}`);
console.log(`Configured connections: Local 9bplus DB, Local scrape DB`);
const child = spawn(command, [], {
env,
stdio: "inherit",
shell: true,
});
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
process.exit(code ?? 0);
});
sql/local-scrape-db.sql
+35
View File
@@ -3,6 +3,41 @@ CREATE DATABASE IF NOT EXISTS `catalog_scrape`
USE `catalog_scrape`;
CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` (
`id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_key` VARCHAR(64) NOT NULL,
`source_name` VARCHAR(128) NOT NULL,
`source_type` VARCHAR(32) NOT NULL DEFAULT '',
`base_url` VARCHAR(512) NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean',
`search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '',
`fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none',
`search_url_template` VARCHAR(1000) NOT NULL,
`fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '',
`search_context_keys` VARCHAR(255) NOT NULL DEFAULT '',
`access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser',
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`note` TEXT NULL,
PRIMARY KEY (`id_product_catalog_source`),
UNIQUE KEY `uq_product_catalog_source_key` (`source_key`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` (
`id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
`id_manufacturer` INT UNSIGNED NOT NULL,
`id_product_catalog_source` INT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`note` TEXT NULL,
PRIMARY KEY (`id_product_catalog_manufacturer_source`),
UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`),
KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`legacy_catalog_source_id` BIGINT UNSIGNED NULL,
src/services/catalog-products.ts
+1 -1
View File
@@ -136,7 +136,7 @@ export async function listMappingSources(config, { manufacturerId, checkUrls = f
scrapeDatabase: config.database?.scrapeName || "catalog_scrape",
}),
);
const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.scrape_manufacturer_sources`;
const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.ps_product_catalog_manufacturer_source`;
const items = result.items.map((row) =>
normalizeMappingSource(row, mappingTableName),
);
src/sql/catalog-maker.ts
+7 -7
View File
@@ -388,9 +388,9 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
return `
SELECT
ms.id AS mapping_id,
ms.manufacturer_id AS id_manufacturer,
s.id AS id_product_catalog_source,
ms.id_product_catalog_manufacturer_source AS mapping_id,
ms.id_manufacturer,
s.id_product_catalog_source,
ms.enabled AS mapping_enabled,
ms.priority AS mapping_priority,
s.source_key,
@@ -410,10 +410,10 @@ export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_sc
s.browser_headless,
s.wait_after_load_ms,
s.note
FROM ${scrapeSchema}.scrape_manufacturer_sources ms
JOIN ${scrapeSchema}.scrape_sources s
ON s.id = ms.source_id
WHERE ms.manufacturer_id = ${safeManufacturerId}
FROM ${scrapeSchema}.ps_product_catalog_manufacturer_source ms
JOIN ${scrapeSchema}.ps_product_catalog_source s
ON s.id_product_catalog_source = ms.id_product_catalog_source
WHERE ms.id_manufacturer = ${safeManufacturerId}
ORDER BY
ms.priority ASC,
s.priority ASC,