Add local scrape database mode and schema
This commit is contained in:
@@ -9,6 +9,7 @@ tmp/
|
||||
*.log
|
||||
*.sql
|
||||
*.sql.gz
|
||||
!sql/local-scrape-db.sql
|
||||
test-results/
|
||||
playwright-report/
|
||||
|
||||
|
||||
@@ -65,15 +65,28 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
|
||||
|
||||
## Database architecture
|
||||
|
||||
There are two selectable data sources:
|
||||
There are three selectable data sources:
|
||||
|
||||
### Live DB
|
||||
### Local 9bplus DB
|
||||
|
||||
- Uses the imported local MariaDB database named `9bplus`.
|
||||
- This is the only database allowed to receive explicit catalog writes.
|
||||
- Writes remain disabled unless the user enables the local permission switches.
|
||||
|
||||
### Live 9bplus DB
|
||||
|
||||
- Uses the existing 9b-plus API endpoint configured in `config/local.json`.
|
||||
- Credentials are secret and must stay server-side.
|
||||
- Must remain read-only.
|
||||
|
||||
### Local DB
|
||||
### Local scrape DB
|
||||
|
||||
- Uses the separate local MariaDB database named `catalog_scrape`.
|
||||
- Stores scraper source configuration, scrape runs, discovered products and assets.
|
||||
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
|
||||
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
|
||||
|
||||
### Local MariaDB connection
|
||||
|
||||
- MariaDB is installed locally as service `MariaDB`.
|
||||
- Host: `localhost` / `127.0.0.1`.
|
||||
@@ -86,12 +99,13 @@ There are two selectable data sources:
|
||||
|
||||
## Database safety behavior
|
||||
|
||||
- The UI has `Local DB` and `Live DB` settings.
|
||||
- The UI has `Local 9bplus DB`, `Live 9bplus DB` and `Local scrape DB` settings.
|
||||
- The UI sends the selected mode using the `X-Database-Mode` request header.
|
||||
- Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint.
|
||||
- Backend data API routes reject a Live DB request when the project is configured only for MariaDB.
|
||||
- Static HTML/CSS/JS files must remain available even when a database is not configured.
|
||||
- The top status shows the selected source, connection state, permissions and execution mode.
|
||||
- Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI.
|
||||
|
||||
## Important current files
|
||||
|
||||
|
||||
@@ -19,7 +19,8 @@
|
||||
"allowWrites": false,
|
||||
"host": "127.0.0.1",
|
||||
"port": 3306,
|
||||
"name": "catalog_maker_test",
|
||||
"name": "9bplus",
|
||||
"scrapeName": "catalog_scrape",
|
||||
"user": "catalog_maker",
|
||||
"password": "",
|
||||
"connectionLimit": 5
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"start": "tsx src/cli.ts",
|
||||
"dev": "tsx src/server.ts",
|
||||
"catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
|
||||
"db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
|
||||
"test": "tsx --test test/*.test.ts",
|
||||
"test:ui": "playwright test",
|
||||
"test:e2e": "playwright test tests/ui/database.e2e.spec.ts",
|
||||
|
||||
+13
-8
@@ -59,7 +59,10 @@ window.fetch = (input, init = {}) => {
|
||||
if (!url.startsWith("/api/"))
|
||||
return nativeFetch(input, init);
|
||||
const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
|
||||
headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||
const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
|
||||
// Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
|
||||
headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
|
||||
headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
|
||||
return nativeFetch(input, { ...init, headers });
|
||||
};
|
||||
let currentPosition = 1;
|
||||
@@ -940,7 +943,7 @@ function loadDatabaseMode() {
|
||||
refreshDatabaseStatusLabel();
|
||||
}
|
||||
function saveDatabaseMode(event) {
|
||||
const mode = event.target.value === "live" ? "live" : "local";
|
||||
const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
|
||||
localStorage.setItem("catalog-maker:database-mode", mode);
|
||||
syncDatabasePermissions(mode);
|
||||
syncLocalDbFields(mode);
|
||||
@@ -987,10 +990,10 @@ function syncLocalDbFields(mode) {
|
||||
localDbFields.hidden = mode !== "local";
|
||||
}
|
||||
function syncDatabasePermissions(mode) {
|
||||
const isLive = mode === "live";
|
||||
const isReadOnly = mode === "live" || mode === "scrape";
|
||||
for (const input of permissionInputs) {
|
||||
input.disabled = isLive;
|
||||
if (isLive)
|
||||
input.disabled = isReadOnly;
|
||||
if (isReadOnly)
|
||||
input.checked = false;
|
||||
}
|
||||
}
|
||||
@@ -1008,7 +1011,7 @@ function refreshDatabaseStatusLabel() {
|
||||
const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
|
||||
const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}");
|
||||
const writesEnabled = mode === "local" && (permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle);
|
||||
const dbLabel = mode === "live" ? "Live DB" : "Local DB";
|
||||
const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
|
||||
const configuredDriver = databaseStatus.dataset.driver;
|
||||
const connectionError = (mode === "local" && configuredDriver !== "mariadb")
|
||||
|| (mode === "live" && configuredDriver !== "endpoint");
|
||||
@@ -1022,8 +1025,10 @@ function refreshDatabaseStatusLabel() {
|
||||
}
|
||||
function renderDatabaseModeNote(mode) {
|
||||
databaseModeNote.textContent = mode === "live"
|
||||
? "Live DB selected for preparation. The current project still stays read-only."
|
||||
: "Local test database selected.";
|
||||
? "Live 9bplus DB selected for read-only data."
|
||||
: mode === "scrape"
|
||||
? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB."
|
||||
: "Local 9bplus DB selected.";
|
||||
}
|
||||
function renderProduct(product) {
|
||||
const productKey = `${manufacturerSelect.value}:${product.idProductCatalog || product.supplierReference || product.name || ""}`;
|
||||
|
||||
+13
-8
@@ -60,7 +60,10 @@ window.fetch = (input, init = {}) => {
|
||||
if (!url.startsWith("/api/")) return nativeFetch(input, init);
|
||||
|
||||
const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
|
||||
headers.set("X-Database-Mode", localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||
const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
|
||||
// Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
|
||||
headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
|
||||
headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
|
||||
return nativeFetch(input, { ...init, headers });
|
||||
};
|
||||
|
||||
@@ -1034,7 +1037,7 @@ function loadDatabaseMode() {
|
||||
}
|
||||
|
||||
function saveDatabaseMode(event) {
|
||||
const mode = event.target.value === "live" ? "live" : "local";
|
||||
const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
|
||||
localStorage.setItem("catalog-maker:database-mode", mode);
|
||||
syncDatabasePermissions(mode);
|
||||
syncLocalDbFields(mode);
|
||||
@@ -1082,10 +1085,10 @@ function syncLocalDbFields(mode) {
|
||||
}
|
||||
|
||||
function syncDatabasePermissions(mode) {
|
||||
const isLive = mode === "live";
|
||||
const isReadOnly = mode === "live" || mode === "scrape";
|
||||
for (const input of permissionInputs) {
|
||||
input.disabled = isLive;
|
||||
if (isLive) input.checked = false;
|
||||
input.disabled = isReadOnly;
|
||||
if (isReadOnly) input.checked = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1105,7 +1108,7 @@ function refreshDatabaseStatusLabel() {
|
||||
const writesEnabled = mode === "local" && (
|
||||
permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle
|
||||
);
|
||||
const dbLabel = mode === "live" ? "Live DB" : "Local DB";
|
||||
const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
|
||||
const configuredDriver = databaseStatus.dataset.driver;
|
||||
const connectionError = (mode === "local" && configuredDriver !== "mariadb")
|
||||
|| (mode === "live" && configuredDriver !== "endpoint");
|
||||
@@ -1120,8 +1123,10 @@ function refreshDatabaseStatusLabel() {
|
||||
|
||||
function renderDatabaseModeNote(mode) {
|
||||
databaseModeNote.textContent = mode === "live"
|
||||
? "Live DB selected for preparation. The current project still stays read-only."
|
||||
: "Local test database selected.";
|
||||
? "Live 9bplus DB selected for read-only data."
|
||||
: mode === "scrape"
|
||||
? "Local scrape DB selected for scraper settings and scraped data. Catalog reads stay on Local 9bplus DB."
|
||||
: "Local 9bplus DB selected.";
|
||||
}
|
||||
|
||||
function renderProduct(product) {
|
||||
|
||||
+7
-3
@@ -151,11 +151,15 @@
|
||||
<legend>Database</legend>
|
||||
<label class="settings-option">
|
||||
<input type="radio" name="databaseMode" value="local" checked />
|
||||
<span>Local DB</span>
|
||||
<span>Local 9bplus DB</span>
|
||||
</label>
|
||||
<label class="settings-option">
|
||||
<input type="radio" name="databaseMode" value="live" />
|
||||
<span>Live DB</span>
|
||||
<span>Live 9bplus DB</span>
|
||||
</label>
|
||||
<label class="settings-option">
|
||||
<input type="radio" name="databaseMode" value="scrape" />
|
||||
<span>Local scrape DB</span>
|
||||
</label>
|
||||
<div class="local-db-fields" id="localDbFields">
|
||||
<label class="settings-field">
|
||||
@@ -181,7 +185,7 @@
|
||||
<button class="settings-test-button" id="testLocalDbButton" type="button">Test connection</button>
|
||||
<p class="settings-note" id="localDbTestResult">Connection not tested.</p>
|
||||
</div>
|
||||
<p class="settings-note" id="databaseModeNote">Local test database selected.</p>
|
||||
<p class="settings-note" id="databaseModeNote">Local 9bplus database selected.</p>
|
||||
</fieldset>
|
||||
|
||||
<fieldset class="settings-group">
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import fs from "node:fs/promises";
|
||||
import mariadb from "mariadb";
|
||||
|
||||
async function readEnv() {
|
||||
return Object.fromEntries(
|
||||
(await fs.readFile(".env", "utf8"))
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line && !line.startsWith("#") && line.includes("="))
|
||||
.map((line) => {
|
||||
const index = line.indexOf("=");
|
||||
return [line.slice(0, index), line.slice(index + 1)];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const env = await readEnv();
|
||||
const connection = await mariadb.createConnection({
|
||||
host: env.DB_HOST || "127.0.0.1",
|
||||
port: Number(env.DB_PORT || 3306),
|
||||
user: env.DB_USER || "root",
|
||||
password: env.DB_PASSWORD || "",
|
||||
});
|
||||
|
||||
try {
|
||||
const sql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
|
||||
for (const statement of sql.split(/;\s*(?:\r?\n|$)/).map((item) => item.trim()).filter(Boolean)) {
|
||||
await connection.query(statement);
|
||||
}
|
||||
console.log("Local scrape database and schema are ready.");
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
CREATE DATABASE IF NOT EXISTS `catalog_scrape`
|
||||
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
|
||||
|
||||
USE `catalog_scrape`;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_sources` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`source_key` VARCHAR(64) NOT NULL,
|
||||
`source_name` VARCHAR(128) NOT NULL,
|
||||
`source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
|
||||
`base_url` VARCHAR(1024) NOT NULL,
|
||||
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
|
||||
`priority` INT NOT NULL DEFAULT 100,
|
||||
`search_primary_key` VARCHAR(64) NULL,
|
||||
`search_fallback_keys` VARCHAR(255) NULL,
|
||||
`search_url_template` VARCHAR(2048) NULL,
|
||||
`fallback_url_template` VARCHAR(2048) NULL,
|
||||
`search_context_keys` VARCHAR(255) NULL,
|
||||
`scraper_config` JSON NULL,
|
||||
`request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
|
||||
`last_health_status` SMALLINT UNSIGNED NULL,
|
||||
`last_health_checked_at` DATETIME NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_products` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`manufacturer_id` INT UNSIGNED NULL,
|
||||
`id_product` BIGINT UNSIGNED NULL,
|
||||
`supplier_reference` VARCHAR(255) NULL,
|
||||
`ean13` VARCHAR(32) NULL,
|
||||
`color` VARCHAR(255) NULL,
|
||||
`source_id` BIGINT UNSIGNED NOT NULL,
|
||||
`source_url` VARCHAR(2048) NOT NULL,
|
||||
`match_method` ENUM('ean', 'reference', 'name', 'manual') NOT NULL DEFAULT 'ean',
|
||||
`source_product_key` VARCHAR(255) NULL,
|
||||
`product_name` TEXT NULL,
|
||||
`raw_data` JSON NULL,
|
||||
`parsed_data` JSON NULL,
|
||||
`status` ENUM('found', 'parsed', 'error', 'manual') NOT NULL DEFAULT 'found',
|
||||
`error_message` TEXT NULL,
|
||||
`fetched_at` DATETIME NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_scrape_products_ean` (`ean13`),
|
||||
KEY `idx_scrape_products_reference` (`supplier_reference`),
|
||||
KEY `idx_scrape_products_source` (`source_id`),
|
||||
CONSTRAINT `fk_scrape_products_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_assets` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`scrape_product_id` BIGINT UNSIGNED NOT NULL,
|
||||
`asset_type` ENUM('image', 'document', 'other') NOT NULL DEFAULT 'image',
|
||||
`asset_url` VARCHAR(2048) NOT NULL,
|
||||
`local_path` VARCHAR(1024) NULL,
|
||||
`content_hash` CHAR(64) NULL,
|
||||
`position` INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
`alt_text` VARCHAR(512) NULL,
|
||||
`status` ENUM('discovered', 'downloaded', 'error') NOT NULL DEFAULT 'discovered',
|
||||
`error_message` TEXT NULL,
|
||||
`fetched_at` DATETIME NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_scrape_assets_product` (`scrape_product_id`),
|
||||
CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `scrape_runs` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
`source_id` BIGINT UNSIGNED NULL,
|
||||
`manufacturer_id` INT UNSIGNED NULL,
|
||||
`started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
`finished_at` DATETIME NULL,
|
||||
`status` ENUM('running', 'completed', 'failed', 'cancelled') NOT NULL DEFAULT 'running',
|
||||
`request_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
`success_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
`error_count` INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
`notes` TEXT NULL,
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_scrape_runs_source` (`source_id`),
|
||||
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
|
||||
) ENGINE=InnoDB;
|
||||
@@ -36,6 +36,7 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
|
||||
name: env.DB_NAME || config.database?.name || "catalog_maker_test",
|
||||
user: env.DB_USER || config.database?.user || "catalog_maker",
|
||||
password: env.DB_PASSWORD || config.database?.password || "",
|
||||
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
|
||||
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
|
||||
},
|
||||
};
|
||||
|
||||
+3
-1
@@ -195,6 +195,7 @@ function getStatus() {
|
||||
database: {
|
||||
driver: config.database.driver,
|
||||
mode: config.database.mode,
|
||||
scrapeName: config.database.scrapeName,
|
||||
allowWrites: config.database.driver === "mariadb" && config.database.mode === "local" && config.database.allowWrites,
|
||||
},
|
||||
workbook: path.basename(config.sourceWorkbook),
|
||||
@@ -212,7 +213,8 @@ function getStatus() {
|
||||
}
|
||||
|
||||
function getRequestConfig(request) {
|
||||
const selectedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase();
|
||||
const requestedMode = String(request.headers["x-database-mode"] || config.database.mode).toLowerCase();
|
||||
const selectedMode = requestedMode === "scrape" ? "local" : requestedMode;
|
||||
|
||||
if (selectedMode === "local" && config.database.driver !== "mariadb") {
|
||||
throw new Error("Local DB is selected, but MariaDB is not configured. Live DB was not used.");
|
||||
|
||||
@@ -17,6 +17,19 @@ test.describe("database connection", () => {
|
||||
await expect(page.locator("#localDbTestResult")).toContainText("Connected:");
|
||||
});
|
||||
|
||||
test("keeps the scrape database as a separate read-only mode", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
await page.locator("#settingsButton").click();
|
||||
|
||||
await expect(page.locator('input[name="databaseMode"][value="local"] + span')).toHaveText("Local 9bplus DB");
|
||||
await expect(page.locator('input[name="databaseMode"][value="live"] + span')).toHaveText("Live 9bplus DB");
|
||||
await page.locator('input[name="databaseMode"][value="scrape"]').check();
|
||||
await expect(page.locator("#databaseModeNote")).toContainText("Local scrape DB selected");
|
||||
await expect(page.locator("#allowInsertToggle")).toBeDisabled();
|
||||
await expect(page.locator("#allowUpdateToggle")).toBeDisabled();
|
||||
await expect(page.locator("#allowDeleteToggle")).toBeDisabled();
|
||||
});
|
||||
|
||||
test("loads manufacturers from Local DB without Live endpoint fallback", async ({ page }) => {
|
||||
const manufacturersResponse = page.waitForResponse("**/api/manufacturers");
|
||||
await page.goto("/");
|
||||
|
||||
Reference in New Issue
Block a user