Replace DBGate with local Adminer
This commit is contained in:
+2
-2
@@ -5,8 +5,8 @@ DB_HOST=127.0.0.1
|
|||||||
DB_PORT=3306
|
DB_PORT=3306
|
||||||
DB_NAME=9bplus
|
DB_NAME=9bplus
|
||||||
SCRAPE_DB_NAME=catalog_scrape
|
SCRAPE_DB_NAME=catalog_scrape
|
||||||
DBGATE_URL=http://127.0.0.1:3000
|
ADMINER_URL=http://127.0.0.1:8080
|
||||||
DBGATE_BROWSER_ENGINE=chromium
|
ADMINER_BROWSER_ENGINE=chromium
|
||||||
DB_USER=catalog_maker
|
DB_USER=catalog_maker
|
||||||
DB_PASSWORD=
|
DB_PASSWORD=
|
||||||
DB_CONNECTION_LIMIT=5
|
DB_CONNECTION_LIMIT=5
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ config/local.json
|
|||||||
outputs/
|
outputs/
|
||||||
logs/
|
logs/
|
||||||
tmp/
|
tmp/
|
||||||
|
tools/adminer/
|
||||||
|
tools/php/
|
||||||
*.log
|
*.log
|
||||||
*.sql
|
*.sql
|
||||||
*.sql.gz
|
*.sql.gz
|
||||||
|
|||||||
@@ -105,8 +105,8 @@ There are three selectable data sources:
|
|||||||
- Uses the separate local MariaDB database named `catalog_scrape`.
|
- Uses the separate local MariaDB database named `catalog_scrape`.
|
||||||
- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
|
- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
|
||||||
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
|
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
|
||||||
- The Settings modal should expose an `Open DBGate` action beside `Local scrape DB`; it opens the local DBGate URL from `DBGATE_URL` / `databaseGate.url` for inspecting `catalog_scrape`.
|
- The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
|
||||||
- Start DBGate only through `npm run dbgate`; the script reads `.env` and preconfigures `Local 9bplus DB` plus `Local scrape DB` as DBGate connections. Do not start raw `dbgate-serve` because it will not receive the project database profiles.
|
- Start Adminer only through `npm run adminer`; the script downloads the ignored local `tools/adminer/adminer.php` file when missing and runs it on the configured local Adminer URL. Adminer requires local PHP CLI.
|
||||||
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
|
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
|
||||||
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
|
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
|
||||||
- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
|
- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
|
||||||
|
|||||||
@@ -25,8 +25,8 @@
|
|||||||
"password": "",
|
"password": "",
|
||||||
"connectionLimit": 5
|
"connectionLimit": 5
|
||||||
},
|
},
|
||||||
"databaseGate": {
|
"databaseTool": {
|
||||||
"url": "http://127.0.0.1:3000",
|
"url": "http://127.0.0.1:8080",
|
||||||
"browserEngine": "chromium"
|
"browserEngine": "chromium"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
-11571
File diff suppressed because it is too large
Load Diff
+1
-2
@@ -10,7 +10,7 @@
|
|||||||
"catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
|
"catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
|
||||||
"db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
|
"db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
|
||||||
"db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
|
"db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
|
||||||
"dbgate": "tsx scripts/start-dbgate.ts",
|
"adminer": "tsx scripts/start-adminer.ts",
|
||||||
"test": "tsx --test test/*.test.ts",
|
"test": "tsx --test test/*.test.ts",
|
||||||
"test:ui": "playwright test",
|
"test:ui": "playwright test",
|
||||||
"test:e2e": "playwright test tests/ui/database.e2e.spec.ts",
|
"test:e2e": "playwright test tests/ui/database.e2e.spec.ts",
|
||||||
@@ -29,7 +29,6 @@
|
|||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@tailwindcss/cli": "^4.3.3",
|
"@tailwindcss/cli": "^4.3.3",
|
||||||
"@types/node": "^26.1.2",
|
"@types/node": "^26.1.2",
|
||||||
"dbgate-serve": "^7.2.3",
|
|
||||||
"tailwindcss": "^4.3.3",
|
"tailwindcss": "^4.3.3",
|
||||||
"tsx": "^4.23.7",
|
"tsx": "^4.23.7",
|
||||||
"typescript": "^7.0.2"
|
"typescript": "^7.0.2"
|
||||||
|
|||||||
+7
-7
@@ -1062,24 +1062,24 @@ async function testLocalDbConnection() {
|
|||||||
async function openDbGate() {
|
async function openDbGate() {
|
||||||
openDbGateButton.disabled = true;
|
openDbGateButton.disabled = true;
|
||||||
const previousNote = databaseModeNote.textContent;
|
const previousNote = databaseModeNote.textContent;
|
||||||
databaseModeNote.textContent = "Opening DBGate...";
|
databaseModeNote.textContent = "Opening Adminer...";
|
||||||
try {
|
try {
|
||||||
const response = await fetch("/api/tools/open-dbgate");
|
const response = await fetch("/api/tools/open-adminer");
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok || !data.opened)
|
if (!response.ok || !data.opened)
|
||||||
throw new Error(data.message || "DBGate could not be opened.");
|
throw new Error(data.message || "Adminer could not be opened.");
|
||||||
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
|
databaseModeNote.textContent = `Adminer opened: ${data.url}`;
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
databaseModeNote.textContent = `DBGate error: ${error.message}`;
|
databaseModeNote.textContent = `Adminer error: ${error.message}`;
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
openDbGateButton.disabled = false;
|
openDbGateButton.disabled = false;
|
||||||
window.setTimeout(() => {
|
window.setTimeout(() => {
|
||||||
if (databaseModeNote.textContent.startsWith("DBGate")) {
|
if (databaseModeNote.textContent.startsWith("Adminer")) {
|
||||||
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||||
}
|
}
|
||||||
else if (databaseModeNote.textContent === "Opening DBGate...") {
|
else if (databaseModeNote.textContent === "Opening Adminer...") {
|
||||||
databaseModeNote.textContent = previousNote;
|
databaseModeNote.textContent = previousNote;
|
||||||
}
|
}
|
||||||
}, 3500);
|
}, 3500);
|
||||||
|
|||||||
+7
-7
@@ -1156,21 +1156,21 @@ async function testLocalDbConnection() {
|
|||||||
async function openDbGate() {
|
async function openDbGate() {
|
||||||
openDbGateButton.disabled = true;
|
openDbGateButton.disabled = true;
|
||||||
const previousNote = databaseModeNote.textContent;
|
const previousNote = databaseModeNote.textContent;
|
||||||
databaseModeNote.textContent = "Opening DBGate...";
|
databaseModeNote.textContent = "Opening Adminer...";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch("/api/tools/open-dbgate");
|
const response = await fetch("/api/tools/open-adminer");
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok || !data.opened) throw new Error(data.message || "DBGate could not be opened.");
|
if (!response.ok || !data.opened) throw new Error(data.message || "Adminer could not be opened.");
|
||||||
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
|
databaseModeNote.textContent = `Adminer opened: ${data.url}`;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
databaseModeNote.textContent = `DBGate error: ${error.message}`;
|
databaseModeNote.textContent = `Adminer error: ${error.message}`;
|
||||||
} finally {
|
} finally {
|
||||||
openDbGateButton.disabled = false;
|
openDbGateButton.disabled = false;
|
||||||
window.setTimeout(() => {
|
window.setTimeout(() => {
|
||||||
if (databaseModeNote.textContent.startsWith("DBGate")) {
|
if (databaseModeNote.textContent.startsWith("Adminer")) {
|
||||||
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||||
} else if (databaseModeNote.textContent === "Opening DBGate...") {
|
} else if (databaseModeNote.textContent === "Opening Adminer...") {
|
||||||
databaseModeNote.textContent = previousNote;
|
databaseModeNote.textContent = previousNote;
|
||||||
}
|
}
|
||||||
}, 3500);
|
}, 3500);
|
||||||
|
|||||||
+1
-1
@@ -177,7 +177,7 @@
|
|||||||
<span>Local scrape DB</span>
|
<span>Local scrape DB</span>
|
||||||
<button class="settings-inline-button" id="openDbGateButton" type="button">
|
<button class="settings-inline-button" id="openDbGateButton" type="button">
|
||||||
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#external"></use></svg>
|
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#external"></use></svg>
|
||||||
<span>Open DBGate</span>
|
<span>Open Adminer</span>
|
||||||
</button>
|
</button>
|
||||||
</label>
|
</label>
|
||||||
<div class="local-db-fields" id="localDbFields">
|
<div class="local-db-fields" id="localDbFields">
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import { createWriteStream, existsSync, mkdirSync } from "node:fs";
|
||||||
|
import { get } from "node:https";
|
||||||
|
import { dirname, resolve } from "node:path";
|
||||||
|
import { spawn, spawnSync } from "node:child_process";
|
||||||
|
|
||||||
|
const adminerFile = resolve("tools/adminer/adminer.php");
|
||||||
|
const localPhp = resolve("tools/php/php.exe");
|
||||||
|
const host = process.env.ADMINER_HOST || "127.0.0.1";
|
||||||
|
const port = process.env.ADMINER_PORT || "8080";
|
||||||
|
const adminerDownloadUrl = process.env.ADMINER_DOWNLOAD_URL || "https://www.adminer.org/latest.php";
|
||||||
|
const phpCommand = existsSync(localPhp) ? localPhp : "php";
|
||||||
|
|
||||||
|
if (!commandExists(phpCommand)) {
|
||||||
|
console.error("Adminer needs PHP CLI. Install PHP first, then run npm run adminer again.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!existsSync(adminerFile)) {
|
||||||
|
mkdirSync(dirname(adminerFile), { recursive: true });
|
||||||
|
await downloadFile(adminerDownloadUrl, adminerFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
const child = spawn(phpCommand, ["-S", `${host}:${port}`, adminerFile], {
|
||||||
|
stdio: "inherit",
|
||||||
|
shell: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
child.on("exit", (code, signal) => {
|
||||||
|
if (signal) {
|
||||||
|
process.kill(process.pid, signal);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
process.exit(code ?? 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
function commandExists(command: string): boolean {
|
||||||
|
const result = spawnSync(command, ["-v"], { stdio: "ignore", shell: true });
|
||||||
|
return result.status === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function downloadFile(url: string, target: string): Promise<void> {
|
||||||
|
return new Promise((resolveDownload, rejectDownload) => {
|
||||||
|
const request = get(url, (response) => {
|
||||||
|
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
||||||
|
downloadFile(response.headers.location, target).then(resolveDownload, rejectDownload);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.statusCode !== 200) {
|
||||||
|
rejectDownload(new Error(`Adminer download failed with HTTP ${response.statusCode}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = createWriteStream(target);
|
||||||
|
response.pipe(file);
|
||||||
|
file.on("finish", () => {
|
||||||
|
file.close();
|
||||||
|
resolveDownload();
|
||||||
|
});
|
||||||
|
file.on("error", rejectDownload);
|
||||||
|
});
|
||||||
|
|
||||||
|
request.on("error", rejectDownload);
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
import { spawn } from "node:child_process";
|
|
||||||
import fs from "node:fs";
|
|
||||||
|
|
||||||
const envFile = ".env";
|
|
||||||
const localEnv = fs.existsSync(envFile) ? readEnvFile(envFile) : {};
|
|
||||||
|
|
||||||
const host = localEnv.DB_HOST || "127.0.0.1";
|
|
||||||
const port = localEnv.DB_PORT || "3306";
|
|
||||||
const user = localEnv.DB_USER || "root";
|
|
||||||
const password = localEnv.DB_PASSWORD || "";
|
|
||||||
const catalogDatabase = localEnv.DB_NAME || "9bplus";
|
|
||||||
const scrapeDatabase = localEnv.SCRAPE_DB_NAME || "catalog_scrape";
|
|
||||||
|
|
||||||
const dbgateEnv = {
|
|
||||||
...process.env,
|
|
||||||
CONNECTIONS: "local9bplus,localscrape",
|
|
||||||
|
|
||||||
ENGINE_local9bplus: "mariadb@dbgate-plugin-mysql",
|
|
||||||
SERVER_local9bplus: host,
|
|
||||||
PORT_local9bplus: port,
|
|
||||||
USER_local9bplus: user,
|
|
||||||
PASSWORD_local9bplus: password,
|
|
||||||
DATABASE_local9bplus: catalogDatabase,
|
|
||||||
LABEL_local9bplus: "Local 9bplus DB",
|
|
||||||
READONLY_local9bplus: "1",
|
|
||||||
|
|
||||||
ENGINE_localscrape: "mariadb@dbgate-plugin-mysql",
|
|
||||||
SERVER_localscrape: host,
|
|
||||||
PORT_localscrape: port,
|
|
||||||
USER_localscrape: user,
|
|
||||||
PASSWORD_localscrape: password,
|
|
||||||
DATABASE_localscrape: scrapeDatabase,
|
|
||||||
LABEL_localscrape: "Local scrape DB",
|
|
||||||
READONLY_localscrape: "0",
|
|
||||||
};
|
|
||||||
|
|
||||||
const child = spawn("dbgate-serve", {
|
|
||||||
stdio: "inherit",
|
|
||||||
shell: true,
|
|
||||||
env: dbgateEnv,
|
|
||||||
});
|
|
||||||
|
|
||||||
child.on("exit", (code, signal) => {
|
|
||||||
if (signal) {
|
|
||||||
process.kill(process.pid, signal);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
process.exit(code ?? 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
function readEnvFile(filePath: string): Record<string, string> {
|
|
||||||
return Object.fromEntries(
|
|
||||||
fs.readFileSync(filePath, "utf8")
|
|
||||||
.split(/\r?\n/)
|
|
||||||
.map((line) => line.trim())
|
|
||||||
.filter((line) => line && !line.startsWith("#") && line.includes("="))
|
|
||||||
.map((line) => {
|
|
||||||
const separator = line.indexOf("=");
|
|
||||||
return [line.slice(0, separator).trim(), line.slice(separator + 1).trim()];
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
+11
-4
@@ -42,9 +42,14 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
|
|||||||
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
|
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
|
||||||
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
|
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
|
||||||
},
|
},
|
||||||
databaseGate: {
|
databaseTool: {
|
||||||
url: env.DBGATE_URL || config.databaseGate?.url || "http://127.0.0.1:3000",
|
url: env.ADMINER_URL || config.databaseTool?.url || config.databaseGate?.url || "http://127.0.0.1:8080",
|
||||||
browserEngine: String(env.DBGATE_BROWSER_ENGINE || config.databaseGate?.browserEngine || "chromium").toLowerCase(),
|
browserEngine: String(
|
||||||
|
env.ADMINER_BROWSER_ENGINE
|
||||||
|
|| config.databaseTool?.browserEngine
|
||||||
|
|| config.databaseGate?.browserEngine
|
||||||
|
|| "chromium",
|
||||||
|
).toLowerCase(),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -93,8 +98,10 @@ function mergeConfig(base, override) {
|
|||||||
...base.database,
|
...base.database,
|
||||||
...override.database,
|
...override.database,
|
||||||
},
|
},
|
||||||
databaseGate: {
|
databaseTool: {
|
||||||
|
...base.databaseTool,
|
||||||
...base.databaseGate,
|
...base.databaseGate,
|
||||||
|
...override.databaseTool,
|
||||||
...override.databaseGate,
|
...override.databaseGate,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
+4
-4
@@ -136,12 +136,12 @@ const server = http.createServer(async (request, response) => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (url.pathname === "/api/tools/open-dbgate") {
|
if (url.pathname === "/api/tools/open-adminer") {
|
||||||
await openInBrowser(config.databaseGate.url, { engine: config.databaseGate.browserEngine });
|
await openInBrowser(config.databaseTool.url, { engine: config.databaseTool.browserEngine });
|
||||||
await sendJson(response, {
|
await sendJson(response, {
|
||||||
opened: true,
|
opened: true,
|
||||||
url: config.databaseGate.url,
|
url: config.databaseTool.url,
|
||||||
browserEngine: config.databaseGate.browserEngine,
|
browserEngine: config.databaseTool.browserEngine,
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user