diff --git a/.gitignore b/.gitignore index 7f6c135..bbe2bc7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ tmp/ *.log *.sql *.sql.gz +!sql/local-scrape-db.sql test-results/ playwright-report/ diff --git a/agent.md b/agent.md index 432f31c..150a1d8 100644 --- a/agent.md +++ b/agent.md @@ -65,15 +65,28 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N ## Database architecture -There are two selectable data sources: +There are three selectable data sources: -### Live DB +### Local 9bplus DB + +- Uses the imported local MariaDB database named `9bplus`. +- This is the only database allowed to receive explicit catalog writes. +- Writes remain disabled unless the user enables the local permission switches. + +### Live 9bplus DB - Uses the existing 9b-plus API endpoint configured in `config/local.json`. - Credentials are secret and must stay server-side. - Must remain read-only. -### Local DB +### Local scrape DB + +- Uses the separate local MariaDB database named `catalog_scrape`. +- Stores scraper source configuration, scrape runs, discovered products and assets. +- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added. +- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`. + +### Local MariaDB connection - MariaDB is installed locally as service `MariaDB`. - Host: `localhost` / `127.0.0.1`. @@ -86,12 +99,13 @@ There are two selectable data sources: ## Database safety behavior -- The UI has `Local DB` and `Live DB` settings. +- The UI has `Local 9bplus DB`, `Live 9bplus DB` and `Local scrape DB` settings. - The UI sends the selected mode using the `X-Database-Mode` request header. - Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint. - Backend data API routes reject a Live DB request when the project is configured only for MariaDB. - Static HTML/CSS/JS files must remain available even when a database is not configured. - The top status shows the selected source, connection state, permissions and execution mode. +- Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI. ## Important current files diff --git a/config/local.example.json b/config/local.example.json index 63647af..5327aa6 100644 --- a/config/local.example.json +++ b/config/local.example.json @@ -19,7 +19,8 @@ "allowWrites": false, "host": "127.0.0.1", "port": 3306, - "name": "catalog_maker_test", + "name": "9bplus", + "scrapeName": "catalog_scrape", "user": "catalog_maker", "password": "", "connectionLimit": 5 diff --git a/package.json b/package.json index d3504e0..b0ab817 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "tsx src/cli.ts", "dev": "tsx src/server.ts", "catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run", + "db:create-scrape": "tsx scripts/create-local-scrape-db.ts", "test": "tsx --test test/*.test.ts", "test:ui": "playwright test", "test:e2e": "playwright test tests/ui/database.e2e.spec.ts", diff --git a/public/app.js b/public/app.js index 00da89e..f845614 100644 --- a/public/app.js +++ b/public/app.js @@ -59,7 +59,10 @@ window.fetch = (input, init = {}) => { if (!url.startsWith("/api/")) return nativeFetch(input, init); const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined)); - headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local"); + const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local"; + // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus. + headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local"); + headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0"); return nativeFetch(input, { ...init, headers }); }; let currentPosition = 1; @@ -940,7 +943,7 @@ function loadDatabaseMode() { refreshDatabaseStatusLabel(); } function saveDatabaseMode(event) { - const mode = event.target.value === "live" ? "live" : "local"; + const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local"; localStorage.setItem("catalog-maker:database-mode", mode); syncDatabasePermissions(mode); syncLocalDbFields(mode); @@ -987,10 +990,10 @@ function syncLocalDbFields(mode) { localDbFields.hidden = mode !== "local"; } function syncDatabasePermissions(mode) { - const isLive = mode === "live"; + const isReadOnly = mode === "live" || mode === "scrape"; for (const input of permissionInputs) { - input.disabled = isLive; - if (isLive) + input.disabled = isReadOnly; + if (isReadOnly) input.checked = false; } } @@ -1008,7 +1011,7 @@ function refreshDatabaseStatusLabel() { const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction"; const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}"); const writesEnabled = mode === "local" && (permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle); - const dbLabel = mode === "live" ? "Live DB" : "Local DB"; + const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB"; const configuredDriver = databaseStatus.dataset.driver; const connectionError = (mode === "local" && configuredDriver !== "mariadb") || (mode === "live" && configuredDriver !== "endpoint"); @@ -1022,8 +1025,10 @@ function refreshDatabaseStatusLabel() { } function renderDatabaseModeNote(mode) { databaseModeNote.textContent = mode === "live" - ? "Live DB selected for preparation. The current project still stays read-only." - : "Local test database selected."; + ? "Live 9bplus DB selected for read-only data." + : mode === "scrape" + ? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB." + : "Local 9bplus DB selected."; } function renderProduct(product) { const productKey = `${manufacturerSelect.value}:${product.idProductCatalog || product.supplierReference || product.name || ""}`; diff --git a/public/app.ts b/public/app.ts index f0391a7..c0bd70b 100644 --- a/public/app.ts +++ b/public/app.ts @@ -60,7 +60,10 @@ window.fetch = (input, init = {}) => { if (!url.startsWith("/api/")) return nativeFetch(input, init); const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined)); - headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local"); + const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local"; + // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus. + headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local"); + headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0"); return nativeFetch(input, { ...init, headers }); }; @@ -1034,7 +1037,7 @@ function loadDatabaseMode() { } function saveDatabaseMode(event) { - const mode = event.target.value === "live" ? "live" : "local"; + const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local"; localStorage.setItem("catalog-maker:database-mode", mode); syncDatabasePermissions(mode); syncLocalDbFields(mode); @@ -1082,10 +1085,10 @@ function syncLocalDbFields(mode) { } function syncDatabasePermissions(mode) { - const isLive = mode === "live"; + const isReadOnly = mode === "live" || mode === "scrape"; for (const input of permissionInputs) { - input.disabled = isLive; - if (isLive) input.checked = false; + input.disabled = isReadOnly; + if (isReadOnly) input.checked = false; } } @@ -1105,7 +1108,7 @@ function refreshDatabaseStatusLabel() { const writesEnabled = mode === "local" && ( permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle ); - const dbLabel = mode === "live" ? "Live DB" : "Local DB"; + const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB"; const configuredDriver = databaseStatus.dataset.driver; const connectionError = (mode === "local" && configuredDriver !== "mariadb") || (mode === "live" && configuredDriver !== "endpoint"); @@ -1120,8 +1123,10 @@ function refreshDatabaseStatusLabel() { function renderDatabaseModeNote(mode) { databaseModeNote.textContent = mode === "live" - ? "Live DB selected for preparation. The current project still stays read-only." - : "Local test database selected."; + ? "Live 9bplus DB selected for read-only data." + : mode === "scrape" + ? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB." + : "Local 9bplus DB selected."; } function renderProduct(product) { diff --git a/public/index.html b/public/index.html index a58596d..e19aa7b 100644 --- a/public/index.html +++ b/public/index.html @@ -151,11 +151,15 @@ Database +
-

Local test database selected.

+

Local 9bplus database selected.

diff --git a/scripts/create-local-scrape-db.ts b/scripts/create-local-scrape-db.ts new file mode 100644 index 0000000..2439e34 --- /dev/null +++ b/scripts/create-local-scrape-db.ts @@ -0,0 +1,33 @@ +import fs from "node:fs/promises"; +import mariadb from "mariadb"; + +async function readEnv() { + return Object.fromEntries( + (await fs.readFile(".env", "utf8")) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#") && line.includes("=")) + .map((line) => { + const index = line.indexOf("="); + return [line.slice(0, index), line.slice(index + 1)]; + }), + ); +} + +const env = await readEnv(); +const connection = await mariadb.createConnection({ + host: env.DB_HOST || "127.0.0.1", + port: Number(env.DB_PORT || 3306), + user: env.DB_USER || "root", + password: env.DB_PASSWORD || "", +}); + +try { + const sql = await fs.readFile("sql/local-scrape-db.sql", "utf8"); + for (const statement of sql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) { + await connection.query(statement); + } + console.log("Local scrape database and schema are ready."); +} finally { + await connection.end(); +} diff --git a/sql/local-scrape-db.sql b/sql/local-scrape-db.sql new file mode 100644 index 0000000..8f7ae4c --- /dev/null +++ b/sql/local-scrape-db.sql @@ -0,0 +1,88 @@ +CREATE DATABASE IF NOT EXISTS `catalog_scrape` + CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +USE `catalog_scrape`; + +CREATE TABLE IF NOT EXISTS `scrape_sources` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `source_key` VARCHAR(64) NOT NULL, + `source_name` VARCHAR(128) NOT NULL, + `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier', + `base_url` VARCHAR(1024) NOT NULL, + `enabled` TINYINT(1) NOT NULL DEFAULT 1, + `priority` INT NOT NULL DEFAULT 100, + `search_primary_key` VARCHAR(64) NULL, + `search_fallback_keys` VARCHAR(255) NULL, + `search_url_template` VARCHAR(2048) NULL, + `fallback_url_template` VARCHAR(2048) NULL, + `search_context_keys` VARCHAR(255) NULL, + `scraper_config` JSON NULL, + `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000, + `last_health_status` SMALLINT UNSIGNED NULL, + `last_health_checked_at` DATETIME NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_scrape_sources_key` (`source_key`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `scrape_products` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `manufacturer_id` INT UNSIGNED NULL, + `id_product` BIGINT UNSIGNED NULL, + `supplier_reference` VARCHAR(255) NULL, + `ean13` VARCHAR(32) NULL, + `color` VARCHAR(255) NULL, + `source_id` BIGINT UNSIGNED NOT NULL, + `source_url` VARCHAR(2048) NOT NULL, + `match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean', + `source_product_key` VARCHAR(255) NULL, + `product_name` TEXT NULL, + `raw_data` JSON NULL, + `parsed_data` JSON NULL, + `status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found', + `error_message` TEXT NULL, + `fetched_at` DATETIME NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_scrape_products_ean` (`ean13`), + KEY `idx_scrape_products_reference` (`supplier_reference`), + KEY `idx_scrape_products_source` (`source_id`), + CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `scrape_assets` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `scrape_product_id` BIGINT UNSIGNED NOT NULL, + `asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image', + `asset_url` VARCHAR(2048) NOT NULL, + `local_path` VARCHAR(1024) NULL, + `content_hash` CHAR(64) NULL, + `position` INT UNSIGNED NOT NULL DEFAULT 0, + `alt_text` VARCHAR(512) NULL, + `status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered', + `error_message` TEXT NULL, + `fetched_at` DATETIME NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_scrape_assets_product` (`scrape_product_id`), + CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) +) ENGINE=InnoDB; + +CREATE TABLE IF NOT EXISTS `scrape_runs` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `source_id` BIGINT UNSIGNED NULL, + `manufacturer_id` INT UNSIGNED NULL, + `started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `finished_at` DATETIME NULL, + `status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running', + `request_count` INT UNSIGNED NOT NULL DEFAULT 0, + `success_count` INT UNSIGNED NOT NULL DEFAULT 0, + `error_count` INT UNSIGNED NOT NULL DEFAULT 0, + `notes` TEXT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_scrape_runs_source` (`source_id`), + CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) +) ENGINE=InnoDB; diff --git a/src/config.ts b/src/config.ts index 80a48d5..05c6720 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,7 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) { name: env.DB_NAME || config.database?.name || "catalog_maker_test", user: env.DB_USER || config.database?.user || "catalog_maker", password: env.DB_PASSWORD || config.database?.password || "", + scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape", connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5), }, }; diff --git a/src/server.ts b/src/server.ts index ab6ad81..d066818 100644 --- a/src/server.ts +++ b/src/server.ts @@ -195,6 +195,7 @@ function getStatus() { database: { driver: config.database.driver, mode: config.database.mode, + scrapeName: config.database.scrapeName, allowWrites: config.database.driver === "mariadb" && config.database.mode === "local" && config.database.allowWrites, }, workbook: path.basename(config.sourceWorkbook), @@ -212,7 +213,8 @@ function getStatus() { } function getRequestConfig(request) { - const selectedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase(); + const requestedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase(); + const selectedMode = requestedMode === "scrape" ? "local" : requestedMode; if (selectedMode === "local" && config.database.driver !== "mariadb") { throw new Error("Local DB is selected, but MariaDB is not configured. Live DB was not used."); diff --git a/tests/ui/database.e2e.spec.ts b/tests/ui/database.e2e.spec.ts index c656f63..0eb3228 100644 --- a/tests/ui/database.e2e.spec.ts +++ b/tests/ui/database.e2e.spec.ts @@ -17,6 +17,19 @@ test.describe("database connection", () => { await expect(page.locator("#localDbTestResult")).toContainText("Connected:"); }); + test("keeps the scrape database as a separate read-only mode", async ({ page }) => { + await page.goto("/"); + await page.locator("#settingsButton").click(); + + await expect(page.locator('input[name="databaseMode"][value="local"] + span')).toHaveText("Local 9bplus DB"); + await expect(page.locator('input[name="databaseMode"][value="live"] + span')).toHaveText("Live 9bplus DB"); + await page.locator('input[name="databaseMode"][value="scrape"]').check(); + await expect(page.locator("#databaseModeNote")).toContainText("Local scrape DB selected"); + await expect(page.locator("#allowInsertToggle")).toBeDisabled(); + await expect(page.locator("#allowUpdateToggle")).toBeDisabled(); + await expect(page.locator("#allowDeleteToggle")).toBeDisabled(); + }); + test("loads manufacturers from Local DB without Live endpoint fallback", async ({ page }) => { const manufacturersResponse = page.waitForResponse("**/api/manufacturers"); await page.goto("/");