Add DBGate opener for scrape database
This commit is contained in:
@@ -5,6 +5,8 @@ DB_HOST=127.0.0.1
|
|||||||
DB_PORT=3306
|
DB_PORT=3306
|
||||||
DB_NAME=9bplus
|
DB_NAME=9bplus
|
||||||
SCRAPE_DB_NAME=catalog_scrape
|
SCRAPE_DB_NAME=catalog_scrape
|
||||||
|
DBGATE_URL=http://127.0.0.1:3000
|
||||||
|
DBGATE_BROWSER_ENGINE=chromium
|
||||||
DB_USER=catalog_maker
|
DB_USER=catalog_maker
|
||||||
DB_PASSWORD=
|
DB_PASSWORD=
|
||||||
DB_CONNECTION_LIMIT=5
|
DB_CONNECTION_LIMIT=5
|
||||||
|
|||||||
@@ -105,6 +105,7 @@ There are three selectable data sources:
|
|||||||
- Uses the separate local MariaDB database named `catalog_scrape`.
|
- Uses the separate local MariaDB database named `catalog_scrape`.
|
||||||
- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
|
- Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
|
||||||
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
|
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
|
||||||
|
- The Settings modal should expose an `Open DBGate` action beside `Local scrape DB`; it opens the local DBGate URL from `DBGATE_URL` / `databaseGate.url` for inspecting `catalog_scrape`.
|
||||||
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
|
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
|
||||||
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
|
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
|
||||||
- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
|
- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
|
||||||
|
|||||||
@@ -24,5 +24,9 @@
|
|||||||
"user": "catalog_maker",
|
"user": "catalog_maker",
|
||||||
"password": "",
|
"password": "",
|
||||||
"connectionLimit": 5
|
"connectionLimit": 5
|
||||||
|
},
|
||||||
|
"databaseGate": {
|
||||||
|
"url": "http://127.0.0.1:3000",
|
||||||
|
"browserEngine": "chromium"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ const localDbFields = document.querySelector("#localDbFields");
|
|||||||
const localDbInputs = document.querySelectorAll("#localDbFields input");
|
const localDbInputs = document.querySelectorAll("#localDbFields input");
|
||||||
const testLocalDbButton = document.querySelector("#testLocalDbButton");
|
const testLocalDbButton = document.querySelector("#testLocalDbButton");
|
||||||
const localDbTestResult = document.querySelector("#localDbTestResult");
|
const localDbTestResult = document.querySelector("#localDbTestResult");
|
||||||
|
const openDbGateButton = document.querySelector("#openDbGateButton");
|
||||||
const lightContrastSelect = document.querySelector("#lightContrastSelect");
|
const lightContrastSelect = document.querySelector("#lightContrastSelect");
|
||||||
const darkContrastSelect = document.querySelector("#darkContrastSelect");
|
const darkContrastSelect = document.querySelector("#darkContrastSelect");
|
||||||
const catalogScreen = document.querySelector(".catalog-screen");
|
const catalogScreen = document.querySelector(".catalog-screen");
|
||||||
@@ -239,6 +240,7 @@ for (const input of transactionModeInputs)
|
|||||||
for (const input of localDbInputs)
|
for (const input of localDbInputs)
|
||||||
input.addEventListener("change", saveLocalDbSettings);
|
input.addEventListener("change", saveLocalDbSettings);
|
||||||
testLocalDbButton.addEventListener("click", testLocalDbConnection);
|
testLocalDbButton.addEventListener("click", testLocalDbConnection);
|
||||||
|
openDbGateButton.addEventListener("click", openDbGate);
|
||||||
lightContrastSelect.addEventListener("change", saveAppearance);
|
lightContrastSelect.addEventListener("change", saveAppearance);
|
||||||
darkContrastSelect.addEventListener("change", saveAppearance);
|
darkContrastSelect.addEventListener("change", saveAppearance);
|
||||||
loadDatabaseMode();
|
loadDatabaseMode();
|
||||||
@@ -1057,6 +1059,32 @@ async function testLocalDbConnection() {
|
|||||||
testLocalDbButton.disabled = false;
|
testLocalDbButton.disabled = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
async function openDbGate() {
|
||||||
|
openDbGateButton.disabled = true;
|
||||||
|
const previousNote = databaseModeNote.textContent;
|
||||||
|
databaseModeNote.textContent = "Opening DBGate...";
|
||||||
|
try {
|
||||||
|
const response = await fetch("/api/tools/open-dbgate");
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok || !data.opened)
|
||||||
|
throw new Error(data.message || "DBGate could not be opened.");
|
||||||
|
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
databaseModeNote.textContent = `DBGate error: ${error.message}`;
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
openDbGateButton.disabled = false;
|
||||||
|
window.setTimeout(() => {
|
||||||
|
if (databaseModeNote.textContent.startsWith("DBGate")) {
|
||||||
|
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||||
|
}
|
||||||
|
else if (databaseModeNote.textContent === "Opening DBGate...") {
|
||||||
|
databaseModeNote.textContent = previousNote;
|
||||||
|
}
|
||||||
|
}, 3500);
|
||||||
|
}
|
||||||
|
}
|
||||||
function syncLocalDbFields(mode) {
|
function syncLocalDbFields(mode) {
|
||||||
localDbFields.hidden = mode !== "local";
|
localDbFields.hidden = mode !== "local";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ const localDbFields = document.querySelector("#localDbFields");
|
|||||||
const localDbInputs = document.querySelectorAll("#localDbFields input");
|
const localDbInputs = document.querySelectorAll("#localDbFields input");
|
||||||
const testLocalDbButton = document.querySelector("#testLocalDbButton");
|
const testLocalDbButton = document.querySelector("#testLocalDbButton");
|
||||||
const localDbTestResult = document.querySelector("#localDbTestResult");
|
const localDbTestResult = document.querySelector("#localDbTestResult");
|
||||||
|
const openDbGateButton = document.querySelector("#openDbGateButton");
|
||||||
const lightContrastSelect = document.querySelector("#lightContrastSelect");
|
const lightContrastSelect = document.querySelector("#lightContrastSelect");
|
||||||
const darkContrastSelect = document.querySelector("#darkContrastSelect");
|
const darkContrastSelect = document.querySelector("#darkContrastSelect");
|
||||||
const catalogScreen = document.querySelector(".catalog-screen");
|
const catalogScreen = document.querySelector(".catalog-screen");
|
||||||
@@ -237,6 +238,7 @@ for (const input of permissionInputs) input.addEventListener("change", saveSetti
|
|||||||
for (const input of transactionModeInputs) input.addEventListener("change", saveSettings);
|
for (const input of transactionModeInputs) input.addEventListener("change", saveSettings);
|
||||||
for (const input of localDbInputs) input.addEventListener("change", saveLocalDbSettings);
|
for (const input of localDbInputs) input.addEventListener("change", saveLocalDbSettings);
|
||||||
testLocalDbButton.addEventListener("click", testLocalDbConnection);
|
testLocalDbButton.addEventListener("click", testLocalDbConnection);
|
||||||
|
openDbGateButton.addEventListener("click", openDbGate);
|
||||||
lightContrastSelect.addEventListener("change", saveAppearance);
|
lightContrastSelect.addEventListener("change", saveAppearance);
|
||||||
darkContrastSelect.addEventListener("change", saveAppearance);
|
darkContrastSelect.addEventListener("change", saveAppearance);
|
||||||
|
|
||||||
@@ -1151,6 +1153,30 @@ async function testLocalDbConnection() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function openDbGate() {
|
||||||
|
openDbGateButton.disabled = true;
|
||||||
|
const previousNote = databaseModeNote.textContent;
|
||||||
|
databaseModeNote.textContent = "Opening DBGate...";
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch("/api/tools/open-dbgate");
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok || !data.opened) throw new Error(data.message || "DBGate could not be opened.");
|
||||||
|
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
|
||||||
|
} catch (error) {
|
||||||
|
databaseModeNote.textContent = `DBGate error: ${error.message}`;
|
||||||
|
} finally {
|
||||||
|
openDbGateButton.disabled = false;
|
||||||
|
window.setTimeout(() => {
|
||||||
|
if (databaseModeNote.textContent.startsWith("DBGate")) {
|
||||||
|
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
|
||||||
|
} else if (databaseModeNote.textContent === "Opening DBGate...") {
|
||||||
|
databaseModeNote.textContent = previousNote;
|
||||||
|
}
|
||||||
|
}, 3500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function syncLocalDbFields(mode) {
|
function syncLocalDbFields(mode) {
|
||||||
localDbFields.hidden = mode !== "local";
|
localDbFields.hidden = mode !== "local";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,4 +16,5 @@
|
|||||||
<symbol id="tag" viewBox="0 0 24 24"><path d="m4 5 8.5-.5L20 12l-8 8-7.5-7.5L4 5Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="1.8"/><circle cx="8" cy="8" r="1.2" fill="currentColor"/></symbol>
|
<symbol id="tag" viewBox="0 0 24 24"><path d="m4 5 8.5-.5L20 12l-8 8-7.5-7.5L4 5Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="1.8"/><circle cx="8" cy="8" r="1.2" fill="currentColor"/></symbol>
|
||||||
<symbol id="close" viewBox="0 0 24 24"><path d="m6 6 12 12M18 6 6 18" fill="none" stroke="currentColor" stroke-linecap="round" stroke-width="1.8"/></symbol>
|
<symbol id="close" viewBox="0 0 24 24"><path d="m6 6 12 12M18 6 6 18" fill="none" stroke="currentColor" stroke-linecap="round" stroke-width="1.8"/></symbol>
|
||||||
<symbol id="refresh" viewBox="0 0 24 24"><path d="M20 11a8 8 0 0 0-14.8-3L3 11m0 0V5m0 6h6M4 13a8 8 0 0 0 14.8 3L21 13m0 0v6m0-6h-6" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8"/></symbol>
|
<symbol id="refresh" viewBox="0 0 24 24"><path d="M20 11a8 8 0 0 0-14.8-3L3 11m0 0V5m0 6h6M4 13a8 8 0 0 0 14.8 3L21 13m0 0v6m0-6h-6" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8"/></symbol>
|
||||||
|
<symbol id="external" viewBox="0 0 24 24"><path d="M14 4h6v6M13 11l7-7M20 14v5a1 1 0 0 1-1 1H5a1 1 0 0 1-1-1V5a1 1 0 0 1 1-1h5" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8"/></symbol>
|
||||||
</svg>
|
</svg>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 3.9 KiB After Width: | Height: | Size: 4.1 KiB |
@@ -175,6 +175,10 @@
|
|||||||
<label class="settings-option">
|
<label class="settings-option">
|
||||||
<input type="radio" name="databaseMode" value="scrape" />
|
<input type="radio" name="databaseMode" value="scrape" />
|
||||||
<span>Local scrape DB</span>
|
<span>Local scrape DB</span>
|
||||||
|
<button class="settings-inline-button" id="openDbGateButton" type="button">
|
||||||
|
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#external"></use></svg>
|
||||||
|
<span>Open DBGate</span>
|
||||||
|
</button>
|
||||||
</label>
|
</label>
|
||||||
<div class="local-db-fields" id="localDbFields">
|
<div class="local-db-fields" id="localDbFields">
|
||||||
<label class="settings-field">
|
<label class="settings-field">
|
||||||
|
|||||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -42,6 +42,10 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
|
|||||||
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
|
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
|
||||||
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
|
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
|
||||||
},
|
},
|
||||||
|
databaseGate: {
|
||||||
|
url: env.DBGATE_URL || config.databaseGate?.url || "http://127.0.0.1:3000",
|
||||||
|
browserEngine: String(env.DBGATE_BROWSER_ENGINE || config.databaseGate?.browserEngine || "chromium").toLowerCase(),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,5 +93,9 @@ function mergeConfig(base, override) {
|
|||||||
...base.database,
|
...base.database,
|
||||||
...override.database,
|
...override.database,
|
||||||
},
|
},
|
||||||
|
databaseGate: {
|
||||||
|
...base.databaseGate,
|
||||||
|
...override.databaseGate,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
getProductPictures,
|
getProductPictures,
|
||||||
openProductSource,
|
openProductSource,
|
||||||
} from "./services/catalog-products.ts";
|
} from "./services/catalog-products.ts";
|
||||||
|
import { openInBrowser } from "./services/browser-adapter.ts";
|
||||||
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||||
const publicDir = path.resolve(__dirname, "../public");
|
const publicDir = path.resolve(__dirname, "../public");
|
||||||
@@ -135,6 +136,16 @@ const server = http.createServer(async (request, response) => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (url.pathname === "/api/tools/open-dbgate") {
|
||||||
|
await openInBrowser(config.databaseGate.url, { engine: config.databaseGate.browserEngine });
|
||||||
|
await sendJson(response, {
|
||||||
|
opened: true,
|
||||||
|
url: config.databaseGate.url,
|
||||||
|
browserEngine: config.databaseGate.browserEngine,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (url.pathname === "/api/catalog-maker/product-info") {
|
if (url.pathname === "/api/catalog-maker/product-info") {
|
||||||
await sendJson(
|
await sendJson(
|
||||||
response,
|
response,
|
||||||
|
|||||||
+4
-2
@@ -188,13 +188,14 @@
|
|||||||
.settings-dialog { @apply max-w-md; }
|
.settings-dialog { @apply max-w-md; }
|
||||||
.suggested-header { @apply flex items-center justify-between border-b border-slate-300 bg-slate-200 px-3 py-2; }
|
.suggested-header { @apply flex items-center justify-between border-b border-slate-300 bg-slate-200 px-3 py-2; }
|
||||||
.suggested-header h2 { @apply m-0 text-sm font-semibold text-slate-800; }
|
.suggested-header h2 { @apply m-0 text-sm font-semibold text-slate-800; }
|
||||||
.modal-close, .settings-test-button { @apply inline-flex items-center justify-center gap-1.5 rounded border border-slate-300 bg-white px-2 py-1 text-[11px] text-slate-800 shadow-sm hover:bg-slate-100; }
|
.modal-close, .settings-test-button, .settings-inline-button { @apply inline-flex items-center justify-center gap-1.5 rounded border border-slate-300 bg-white px-2 py-1 text-[11px] text-slate-800 shadow-sm hover:bg-slate-100; }
|
||||||
.modal-close .sidebar-icon, .settings-test-button .sidebar-icon { @apply h-3.5 w-3.5 text-slate-500; }
|
.modal-close .sidebar-icon, .settings-test-button .sidebar-icon, .settings-inline-button .sidebar-icon { @apply h-3.5 w-3.5 text-slate-500; }
|
||||||
.suggested-list, .source-list, .settings-content { @apply max-h-[calc(90vh-48px)] overflow-y-auto p-2; }
|
.suggested-list, .source-list, .settings-content { @apply max-h-[calc(90vh-48px)] overflow-y-auto p-2; }
|
||||||
.settings-content { @apply grid gap-2; }
|
.settings-content { @apply grid gap-2; }
|
||||||
.settings-group { @apply m-0 rounded border border-slate-300 p-2; }
|
.settings-group { @apply m-0 rounded border border-slate-300 p-2; }
|
||||||
.settings-group legend { @apply px-1 text-xs font-semibold text-slate-700; }
|
.settings-group legend { @apply px-1 text-xs font-semibold text-slate-700; }
|
||||||
.settings-option { @apply flex items-center gap-1 py-0.5 text-[11px] text-slate-700; }
|
.settings-option { @apply flex items-center gap-1 py-0.5 text-[11px] text-slate-700; }
|
||||||
|
.settings-option .settings-inline-button { @apply ml-auto py-0.5 text-[10px]; }
|
||||||
.settings-option input { @apply accent-catalog-focus; }
|
.settings-option input { @apply accent-catalog-focus; }
|
||||||
.settings-option input:disabled + span { @apply text-slate-400; }
|
.settings-option input:disabled + span { @apply text-slate-400; }
|
||||||
.local-db-fields { @apply grid gap-1; }
|
.local-db-fields { @apply grid gap-1; }
|
||||||
@@ -341,6 +342,7 @@
|
|||||||
html.dark .settings-field input { @apply border-slate-600 bg-slate-800 text-slate-100; }
|
html.dark .settings-field input { @apply border-slate-600 bg-slate-800 text-slate-100; }
|
||||||
html.dark .modal-close,
|
html.dark .modal-close,
|
||||||
html.dark .settings-test-button,
|
html.dark .settings-test-button,
|
||||||
|
html.dark .settings-inline-button,
|
||||||
html.dark .suggested-pick,
|
html.dark .suggested-pick,
|
||||||
html.dark .source-row-action,
|
html.dark .source-row-action,
|
||||||
html.dark .source-picture-action { @apply border-slate-600 bg-slate-800 text-slate-100 hover:bg-slate-700; }
|
html.dark .source-picture-action { @apply border-slate-600 bg-slate-800 text-slate-100 hover:bg-slate-700; }
|
||||||
|
|||||||
Reference in New Issue
Block a user