Add local scrape database mode and schema

.gitignore

@@ -9,6 +9,7 @@ tmp/
 *.log
 *.sql
 *.sql.gz
+!sql/local-scrape-db.sql
 test-results/
 playwright-report/
 

agent.md

@@ -65,15 +65,28 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
 
 ## Database architecture
 
-There are two selectable data sources:
+There are three selectable data sources:
 
-### Live DB
+### Local 9bplus DB
+
+- Uses the imported local MariaDB database named `9bplus`.
+- This is the only database allowed to receive explicit catalog writes.
+- Writes remain disabled unless the user enables the local permission switches.
+
+### Live 9bplus DB
 
 - Uses the existing 9b-plus API endpoint configured in `config/local.json`.
 - Credentials are secret and must stay server-side.
 - Must remain read-only.
 
-### Local DB
+### Local scrape DB
+
+- Uses the separate local MariaDB database named `catalog_scrape`.
+- Stores scraper source configuration, scrape runs, discovered products and assets.
+- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
+- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
+
+### Local MariaDB connection
 
 - MariaDB is installed locally as service `MariaDB`.
 - Host: `localhost` / `127.0.0.1`.
@@ -86,12 +99,13 @@ There are two selectable data sources:
 
 ## Database safety behavior
 
-- The UI has `Local DB` and `Live DB` settings.
+- The UI has `Local 9bplus DB`, `Live 9bplus DB` and `Local scrape DB` settings.
 - The UI sends the selected mode using the `X-Database-Mode` request header.
 - Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint.
 - Backend data API routes reject a Live DB request when the project is configured only for MariaDB.
 - Static HTML/CSS/JS files must remain available even when a database is not configured.
 - The top status shows the selected source, connection state, permissions and execution mode.
+- Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI.
 
 ## Important current files
 

config/local.example.json

@@ -19,7 +19,8 @@
     "allowWrites": false,
     "host": "127.0.0.1",
     "port": 3306,
-    "name": "catalog_maker_test",
+    "name": "9bplus",
+    "scrapeName": "catalog_scrape",
     "user": "catalog_maker",
     "password": "",
     "connectionLimit": 5

package.json

@@ -8,6 +8,7 @@
     "start": "tsx src/cli.ts",
     "dev": "tsx src/server.ts",
     "catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
+    "db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
     "test": "tsx --test test/*.test.ts",
     "test:ui": "playwright test",
     "test:e2e": "playwright test tests/ui/database.e2e.spec.ts",

public/app.js

@@ -59,7 +59,10 @@ window.fetch = (input, init = {}) => {
     if (!url.startsWith("/api/"))
         return nativeFetch(input, init);
     const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
-    headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local");
+    const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
+    // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
+    headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
+    headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
     return nativeFetch(input, { ...init, headers });
 };
 let currentPosition = 1;
@@ -940,7 +943,7 @@ function loadDatabaseMode() {
     refreshDatabaseStatusLabel();
 }
 function saveDatabaseMode(event) {
-    const mode = event.target.value === "live" ? "live" : "local";
+    const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
     localStorage.setItem("catalog-maker:database-mode", mode);
     syncDatabasePermissions(mode);
     syncLocalDbFields(mode);
@@ -987,10 +990,10 @@ function syncLocalDbFields(mode) {
     localDbFields.hidden = mode !== "local";
 }
 function syncDatabasePermissions(mode) {
-    const isLive = mode === "live";
+    const isReadOnly = mode === "live" || mode === "scrape";
     for (const input of permissionInputs) {
-        input.disabled = isLive;
+        input.disabled = isReadOnly;
-        if (isLive)
+        if (isReadOnly)
             input.checked = false;
     }
 }
@@ -1008,7 +1011,7 @@ function refreshDatabaseStatusLabel() {
     const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
     const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}");
     const writesEnabled = mode === "local" && (permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle);
-    const dbLabel = mode === "live" ? "Live DB" : "Local DB";
+    const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
     const configuredDriver = databaseStatus.dataset.driver;
     const connectionError = (mode === "local" && configuredDriver !== "mariadb")
         || (mode === "live" && configuredDriver !== "endpoint");
@@ -1022,8 +1025,10 @@ function refreshDatabaseStatusLabel() {
 }
 function renderDatabaseModeNote(mode) {
     databaseModeNote.textContent = mode === "live"
-        ? "Live DB selected for preparation. The current project still stays read-only."
+        ? "Live 9bplus DB selected for read-only data."
-        : "Local test database selected.";
+        : mode === "scrape"
+            ? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB."
+            : "Local 9bplus DB selected.";
 }
 function renderProduct(product) {
     const productKey = `${manufacturerSelect.value}:${product.idProductCatalog || product.supplierReference || product.name || ""}`;

public/app.ts

@@ -60,7 +60,10 @@ window.fetch = (input, init = {}) => {
   if (!url.startsWith("/api/")) return nativeFetch(input, init);
 
   const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
-  headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local");
+  const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
+  // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
+  headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
+  headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
   return nativeFetch(input, { ...init, headers });
 };
 
@@ -1034,7 +1037,7 @@ function loadDatabaseMode() {
 }
 
 function saveDatabaseMode(event) {
-  const mode = event.target.value === "live" ? "live" : "local";
+  const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
   localStorage.setItem("catalog-maker:database-mode", mode);
   syncDatabasePermissions(mode);
   syncLocalDbFields(mode);
@@ -1082,10 +1085,10 @@ function syncLocalDbFields(mode) {
 }
 
 function syncDatabasePermissions(mode) {
-  const isLive = mode === "live";
+  const isReadOnly = mode === "live" || mode === "scrape";
   for (const input of permissionInputs) {
-    input.disabled = isLive;
+    input.disabled = isReadOnly;
-    if (isLive) input.checked = false;
+    if (isReadOnly) input.checked = false;
   }
 }
 
@@ -1105,7 +1108,7 @@ function refreshDatabaseStatusLabel() {
   const writesEnabled = mode === "local" && (
     permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle
   );
-  const dbLabel = mode === "live" ? "Live DB" : "Local DB";
+  const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
   const configuredDriver = databaseStatus.dataset.driver;
   const connectionError = (mode === "local" && configuredDriver !== "mariadb")
     || (mode === "live" && configuredDriver !== "endpoint");
@@ -1120,8 +1123,10 @@ function refreshDatabaseStatusLabel() {
 
 function renderDatabaseModeNote(mode) {
   databaseModeNote.textContent = mode === "live"
-    ? "Live DB selected for preparation. The current project still stays read-only."
+    ? "Live 9bplus DB selected for read-only data."
-    : "Local test database selected.";
+    : mode === "scrape"
+      ? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB."
+      : "Local 9bplus DB selected.";
 }
 
 function renderProduct(product) {

public/index.html

@@ -151,11 +151,15 @@
             <legend>Database</legend>
             <label class="settings-option">
               <input type="radio" name="databaseMode" value="local" checked />
-              <span>Local DB</span>
+              <span>Local 9bplus DB</span>
             </label>
             <label class="settings-option">
               <input type="radio" name="databaseMode" value="live" />
-              <span>Live DB</span>
+              <span>Live 9bplus DB</span>
+            </label>
+            <label class="settings-option">
+              <input type="radio" name="databaseMode" value="scrape" />
+              <span>Local scrape DB</span>
             </label>
             <div class="local-db-fields" id="localDbFields">
               <label class="settings-field">
@@ -181,7 +185,7 @@
               <button class="settings-test-button" id="testLocalDbButton" type="button">Test connection</button>
               <p class="settings-note" id="localDbTestResult">Connection not tested.</p>
             </div>
-            <p class="settings-note" id="databaseModeNote">Local test database selected.</p>
+            <p class="settings-note" id="databaseModeNote">Local 9bplus database selected.</p>
           </fieldset>
 
           <fieldset class="settings-group">

scripts/create-local-scrape-db.ts

@@ -0,0 +1,33 @@
+import fs from "node:fs/promises";
+import mariadb from "mariadb";
+
+async function readEnv() {
+  return Object.fromEntries(
+    (await fs.readFile(".env", "utf8"))
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line && !line.startsWith("#") && line.includes("="))
+      .map((line) => {
+        const index = line.indexOf("=");
+        return [line.slice(0, index), line.slice(index + 1)];
+      }),
+  );
+}
+
+const env = await readEnv();
+const connection = await mariadb.createConnection({
+  host: env.DB_HOST || "127.0.0.1",
+  port: Number(env.DB_PORT || 3306),
+  user: env.DB_USER || "root",
+  password: env.DB_PASSWORD || "",
+});
+
+try {
+  const sql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
+  for (const statement of sql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) {
+    await connection.query(statement);
+  }
+  console.log("Local scrape database and schema are ready.");
+} finally {
+  await connection.end();
+}

sql/local-scrape-db.sql

@@ -0,0 +1,88 @@
+CREATE DATABASE IF NOT EXISTS `catalog_scrape`
+  CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+USE `catalog_scrape`;
+
+CREATE TABLE IF NOT EXISTS `scrape_sources` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `source_key` VARCHAR(64) NOT NULL,
+  `source_name` VARCHAR(128) NOT NULL,
+  `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
+  `base_url` VARCHAR(1024) NOT NULL,
+  `enabled` TINYINT(1) NOT NULL DEFAULT 1,
+  `priority` INT NOT NULL DEFAULT 100,
+  `search_primary_key` VARCHAR(64) NULL,
+  `search_fallback_keys` VARCHAR(255) NULL,
+  `search_url_template` VARCHAR(2048) NULL,
+  `fallback_url_template` VARCHAR(2048) NULL,
+  `search_context_keys` VARCHAR(255) NULL,
+  `scraper_config` JSON NULL,
+  `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
+  `last_health_status` SMALLINT UNSIGNED NULL,
+  `last_health_checked_at` DATETIME NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_products` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `manufacturer_id` INT UNSIGNED NULL,
+  `id_product` BIGINT UNSIGNED NULL,
+  `supplier_reference` VARCHAR(255) NULL,
+  `ean13` VARCHAR(32) NULL,
+  `color` VARCHAR(255) NULL,
+  `source_id` BIGINT UNSIGNED NOT NULL,
+  `source_url` VARCHAR(2048) NOT NULL,
+  `match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean',
+  `source_product_key` VARCHAR(255) NULL,
+  `product_name` TEXT NULL,
+  `raw_data` JSON NULL,
+  `parsed_data` JSON NULL,
+  `status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found',
+  `error_message` TEXT NULL,
+  `fetched_at` DATETIME NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `idx_scrape_products_ean` (`ean13`),
+  KEY `idx_scrape_products_reference` (`supplier_reference`),
+  KEY `idx_scrape_products_source` (`source_id`),
+  CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_assets` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `scrape_product_id` BIGINT UNSIGNED NOT NULL,
+  `asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image',
+  `asset_url` VARCHAR(2048) NOT NULL,
+  `local_path` VARCHAR(1024) NULL,
+  `content_hash` CHAR(64) NULL,
+  `position` INT UNSIGNED NOT NULL DEFAULT 0,
+  `alt_text` VARCHAR(512) NULL,
+  `status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered',
+  `error_message` TEXT NULL,
+  `fetched_at` DATETIME NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `idx_scrape_assets_product` (`scrape_product_id`),
+  CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS `scrape_runs` (
+  `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `source_id` BIGINT UNSIGNED NULL,
+  `manufacturer_id` INT UNSIGNED NULL,
+  `started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `finished_at` DATETIME NULL,
+  `status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running',
+  `request_count` INT UNSIGNED NOT NULL DEFAULT 0,
+  `success_count` INT UNSIGNED NOT NULL DEFAULT 0,
+  `error_count` INT UNSIGNED NOT NULL DEFAULT 0,
+  `notes` TEXT NULL,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `idx_scrape_runs_source` (`source_id`),
+  CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
+) ENGINE=InnoDB;

src/config.ts

@@ -36,6 +36,7 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
       name: env.DB_NAME || config.database?.name || "catalog_maker_test",
       user: env.DB_USER || config.database?.user || "catalog_maker",
       password: env.DB_PASSWORD || config.database?.password || "",
+      scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
       connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
     },
   };

src/server.ts

@@ -195,6 +195,7 @@ function getStatus() {
     database: {
       driver: config.database.driver,
       mode: config.database.mode,
+      scrapeName: config.database.scrapeName,
       allowWrites: config.database.driver === "mariadb" && config.database.mode === "local" && config.database.allowWrites,
     },
     workbook: path.basename(config.sourceWorkbook),
@@ -212,7 +213,8 @@ function getStatus() {
 }
 
 function getRequestConfig(request) {
-  const selectedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase();
+  const requestedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase();
+  const selectedMode = requestedMode === "scrape" ? "local" : requestedMode;
 
   if (selectedMode === "local" && config.database.driver !== "mariadb") {
     throw new Error("Local DB is selected, but MariaDB is not configured. Live DB was not used.");

tests/ui/database.e2e.spec.ts

@@ -17,6 +17,19 @@ test.describe("database connection", () => {
     await expect(page.locator("#localDbTestResult")).toContainText("Connected:");
   });
 
+  test("keeps the scrape database as a separate read-only mode", async ({ page }) => {
+    await page.goto("/");
+    await page.locator("#settingsButton").click();
+
+    await expect(page.locator('input[name="databaseMode"][value="local"] + span')).toHaveText("Local 9bplus DB");
+    await expect(page.locator('input[name="databaseMode"][value="live"] + span')).toHaveText("Live 9bplus DB");
+    await page.locator('input[name="databaseMode"][value="scrape"]').check();
+    await expect(page.locator("#databaseModeNote")).toContainText("Local scrape DB selected");
+    await expect(page.locator("#allowInsertToggle")).toBeDisabled();
+    await expect(page.locator("#allowUpdateToggle")).toBeDisabled();
+    await expect(page.locator("#allowDeleteToggle")).toBeDisabled();
+  });
+
   test("loads manufacturers from Local DB without Live endpoint fallback", async ({ page }) => {
     const manufacturersResponse = page.waitForResponse("**/api/manufacturers");
     await page.goto("/");