271 lines
9.4 KiB
TypeScript
271 lines
9.4 KiB
TypeScript
import fs from "node:fs/promises";
|
|
import mariadb from "mariadb";
|
|
|
|
async function readEnv() {
|
|
return Object.fromEntries(
|
|
(await fs.readFile(".env", "utf8"))
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter((line) => line && !line.startsWith("#") && line.includes("="))
|
|
.map((line) => {
|
|
const index = line.indexOf("=");
|
|
return [line.slice(0, index).trim(), line.slice(index + 1).trim()];
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function queryRows(connection, sql, params = []) {
|
|
const rows = await connection.query(sql, params);
|
|
return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
|
|
}
|
|
|
|
async function tableExists(connection, schema, table) {
|
|
const rows = await queryRows(
|
|
connection,
|
|
"SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1",
|
|
[schema, table],
|
|
);
|
|
return rows.length > 0;
|
|
}
|
|
|
|
async function columnExists(connection, schema, table, column) {
|
|
const rows = await queryRows(
|
|
connection,
|
|
"SELECT 1 FROM information_schema.columns WHERE table_schema = ? AND table_name = ? AND column_name = ? LIMIT 1",
|
|
[schema, table, column],
|
|
);
|
|
return rows.length > 0;
|
|
}
|
|
|
|
async function addColumnIfMissing(connection, schema, table, column, definition) {
|
|
if (await columnExists(connection, schema, table, column)) return;
|
|
await connection.query(`ALTER TABLE \`${schema}\`.\`${table}\` ADD COLUMN \`${column}\` ${definition}`);
|
|
}
|
|
|
|
const env = await readEnv();
|
|
const catalogDb = env.DB_NAME || "9bplus";
|
|
const scrapeDb = env.SCRAPE_DB_NAME || "catalog_scrape";
|
|
const connection = await mariadb.createConnection({
|
|
host: env.DB_HOST || "127.0.0.1",
|
|
port: Number(env.DB_PORT || 3306),
|
|
user: env.DB_USER || "root",
|
|
password: env.DB_PASSWORD || "",
|
|
multipleStatements: false,
|
|
bigIntAsNumber: true,
|
|
});
|
|
|
|
try {
|
|
await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
|
|
const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
|
|
const schemaStatements = schemaSql
|
|
.split(/;\s*(?:\r?\n|$)/)
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
for (const statement of schemaStatements) {
|
|
await connection.query(statement);
|
|
}
|
|
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "legacy_catalog_source_id", "BIGINT UNSIGNED NULL");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "fallback_source_type", "VARCHAR(64) NOT NULL DEFAULT 'none'");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "access_mode", "VARCHAR(64) NOT NULL DEFAULT 'browser'");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_engine", "VARCHAR(32) NOT NULL DEFAULT 'default'");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_headless", "TINYINT(1) NOT NULL DEFAULT 0");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
|
|
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
|
|
|
|
const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
|
|
const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
|
|
|
|
if (hasCatalogSourceTable) {
|
|
await connection.query(`
|
|
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
|
|
id_product_catalog_source,
|
|
source_key,
|
|
source_name,
|
|
source_type,
|
|
base_url,
|
|
enabled,
|
|
priority,
|
|
search_primary_key,
|
|
search_fallback_keys,
|
|
fallback_source_type,
|
|
search_url_template,
|
|
fallback_url_template,
|
|
search_context_keys,
|
|
access_mode,
|
|
browser_engine,
|
|
browser_headless,
|
|
wait_after_load_ms,
|
|
note
|
|
)
|
|
SELECT
|
|
id_product_catalog_source,
|
|
source_key,
|
|
source_name,
|
|
source_type,
|
|
base_url,
|
|
enabled,
|
|
priority,
|
|
search_primary_key,
|
|
search_fallback_keys,
|
|
COALESCE(fallback_source_type, 'none'),
|
|
search_url_template,
|
|
fallback_url_template,
|
|
search_context_keys,
|
|
COALESCE(access_mode, 'browser'),
|
|
COALESCE(browser_engine, 'default'),
|
|
COALESCE(browser_headless, 0),
|
|
COALESCE(wait_after_load_ms, 3000),
|
|
note
|
|
FROM \`${catalogDb}\`.ps_product_catalog_source
|
|
ON DUPLICATE KEY UPDATE
|
|
source_key = VALUES(source_key),
|
|
source_name = VALUES(source_name),
|
|
source_type = VALUES(source_type),
|
|
base_url = VALUES(base_url),
|
|
enabled = VALUES(enabled),
|
|
priority = VALUES(priority),
|
|
search_primary_key = VALUES(search_primary_key),
|
|
search_fallback_keys = VALUES(search_fallback_keys),
|
|
fallback_source_type = VALUES(fallback_source_type),
|
|
search_url_template = VALUES(search_url_template),
|
|
fallback_url_template = VALUES(fallback_url_template),
|
|
search_context_keys = VALUES(search_context_keys),
|
|
access_mode = VALUES(access_mode),
|
|
browser_engine = VALUES(browser_engine),
|
|
browser_headless = VALUES(browser_headless),
|
|
wait_after_load_ms = VALUES(wait_after_load_ms),
|
|
note = VALUES(note)
|
|
`);
|
|
}
|
|
|
|
if (hasCatalogManufacturerSourceTable) {
|
|
await connection.query(`
|
|
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
|
|
id_product_catalog_manufacturer_source,
|
|
id_manufacturer,
|
|
id_product_catalog_source,
|
|
enabled,
|
|
priority,
|
|
note
|
|
)
|
|
SELECT
|
|
id_product_catalog_manufacturer_source,
|
|
id_manufacturer,
|
|
id_product_catalog_source,
|
|
enabled,
|
|
priority,
|
|
note
|
|
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
|
|
ON DUPLICATE KEY UPDATE
|
|
id_manufacturer = VALUES(id_manufacturer),
|
|
id_product_catalog_source = VALUES(id_product_catalog_source),
|
|
enabled = VALUES(enabled),
|
|
priority = VALUES(priority),
|
|
note = VALUES(note)
|
|
`);
|
|
}
|
|
|
|
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
|
|
throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
|
|
}
|
|
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
|
|
throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
|
|
}
|
|
|
|
await connection.query(`
|
|
INSERT INTO \`${scrapeDb}\`.scrape_sources (
|
|
legacy_catalog_source_id,
|
|
source_key,
|
|
source_name,
|
|
source_type,
|
|
base_url,
|
|
enabled,
|
|
priority,
|
|
search_primary_key,
|
|
search_fallback_keys,
|
|
fallback_source_type,
|
|
search_url_template,
|
|
fallback_url_template,
|
|
search_context_keys,
|
|
access_mode,
|
|
browser_engine,
|
|
browser_headless,
|
|
wait_after_load_ms,
|
|
note
|
|
)
|
|
SELECT
|
|
id_product_catalog_source,
|
|
source_key,
|
|
source_name,
|
|
source_type,
|
|
base_url,
|
|
enabled,
|
|
priority,
|
|
search_primary_key,
|
|
search_fallback_keys,
|
|
COALESCE(fallback_source_type, 'none'),
|
|
search_url_template,
|
|
fallback_url_template,
|
|
search_context_keys,
|
|
COALESCE(access_mode, 'browser'),
|
|
COALESCE(browser_engine, 'default'),
|
|
COALESCE(browser_headless, 0),
|
|
COALESCE(wait_after_load_ms, 3000),
|
|
note
|
|
FROM \`${scrapeDb}\`.ps_product_catalog_source
|
|
ON DUPLICATE KEY UPDATE
|
|
source_key = VALUES(source_key),
|
|
source_name = VALUES(source_name),
|
|
source_type = VALUES(source_type),
|
|
base_url = VALUES(base_url),
|
|
enabled = VALUES(enabled),
|
|
priority = VALUES(priority),
|
|
search_primary_key = VALUES(search_primary_key),
|
|
search_fallback_keys = VALUES(search_fallback_keys),
|
|
fallback_source_type = VALUES(fallback_source_type),
|
|
search_url_template = VALUES(search_url_template),
|
|
fallback_url_template = VALUES(fallback_url_template),
|
|
search_context_keys = VALUES(search_context_keys),
|
|
access_mode = VALUES(access_mode),
|
|
browser_engine = VALUES(browser_engine),
|
|
browser_headless = VALUES(browser_headless),
|
|
wait_after_load_ms = VALUES(wait_after_load_ms),
|
|
note = VALUES(note)
|
|
`);
|
|
|
|
await connection.query(`
|
|
INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources (
|
|
manufacturer_id,
|
|
source_id,
|
|
enabled,
|
|
priority
|
|
)
|
|
SELECT
|
|
ms.id_manufacturer,
|
|
ss.id,
|
|
ms.enabled,
|
|
ms.priority
|
|
FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
|
|
JOIN \`${scrapeDb}\`.scrape_sources ss
|
|
ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
|
|
ON DUPLICATE KEY UPDATE
|
|
enabled = VALUES(enabled),
|
|
priority = VALUES(priority)
|
|
`);
|
|
|
|
for (const statement of schemaStatements.filter(isCapabilitySeedStatement)) {
|
|
await connection.query(statement);
|
|
}
|
|
|
|
const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`);
|
|
const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`);
|
|
console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`);
|
|
} finally {
|
|
await connection.end();
|
|
}
|
|
|
|
function isCapabilitySeedStatement(statement) {
|
|
return /^INSERT INTO `scrape_(source_capabilities|manufacturer_source_capabilities)`/i.test(statement);
|
|
}
|