Store selected picture candidates in scrape DB
This commit is contained in:
@@ -0,0 +1,319 @@
|
||||
// @ts-nocheck
|
||||
import crypto from "node:crypto";
|
||||
import mariadb from "mariadb";
|
||||
|
||||
export async function saveScrapePictureCandidate(config, picture) {
|
||||
if (config.database?.driver !== "mariadb") {
|
||||
throw new Error("Local scrape DB needs MariaDB configuration.");
|
||||
}
|
||||
if (!config.database?.allowLocalWrites) {
|
||||
throw new Error("Local scrape DB writes are disabled. Enable writes in Settings first.");
|
||||
}
|
||||
|
||||
const imageUrl = normalizeRequiredUrl(picture.imageUrl, "imageUrl");
|
||||
const sourceKey = normalizeText(picture.sourceKey) || "unknown";
|
||||
const sourceName = normalizeText(picture.sourceName) || sourceKey;
|
||||
const sourcePageUrl = normalizeOptionalText(picture.pageUrl);
|
||||
const image = await downloadImage(imageUrl);
|
||||
const sha256 = crypto.createHash("sha256").update(image.buffer).digest("hex");
|
||||
const scrapeDb = config.database.scrapeName || "catalog_scrape";
|
||||
|
||||
const connection = await mariadb.createConnection({
|
||||
host: config.database.host,
|
||||
port: Number(config.database.port),
|
||||
database: scrapeDb,
|
||||
user: config.database.user,
|
||||
password: config.database.password,
|
||||
bigIntAsNumber: true,
|
||||
});
|
||||
|
||||
try {
|
||||
await connection.beginTransaction();
|
||||
const sourceId = await upsertSource(connection, {
|
||||
sourceKey,
|
||||
sourceName,
|
||||
imageUrl,
|
||||
sourceType: normalizeText(picture.sourceType) || "supplier",
|
||||
});
|
||||
const imageBlobId = await upsertImageBlob(connection, {
|
||||
sha256,
|
||||
mimeType: image.mimeType,
|
||||
width: normalizePositiveNumber(picture.width),
|
||||
height: normalizePositiveNumber(picture.height),
|
||||
sizeBytes: image.buffer.byteLength,
|
||||
buffer: image.buffer,
|
||||
});
|
||||
const duplicate = await findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl);
|
||||
const candidateId = await upsertPictureCandidate(connection, {
|
||||
imageBlobId,
|
||||
sourceId,
|
||||
manufacturerId: normalizePositiveNumber(picture.manufacturerId),
|
||||
idProductCatalog: normalizePositiveNumber(picture.idProductCatalog),
|
||||
idProduct: normalizePositiveNumber(picture.idProduct),
|
||||
supplierReference: normalizeOptionalText(picture.supplierReference),
|
||||
ean13: normalizeOptionalText(picture.ean13),
|
||||
color: normalizeOptionalText(picture.color),
|
||||
sizeValue: normalizeOptionalText(picture.sizeValue),
|
||||
combi: normalizeOptionalText(picture.combi),
|
||||
sourcePageUrl,
|
||||
sourceImageUrl: imageUrl,
|
||||
sourcePosition: normalizePositiveNumber(picture.sourcePosition) || 0,
|
||||
targetStrategy: normalizeTargetStrategy(picture.targetStrategy),
|
||||
targetColor: normalizeOptionalText(picture.targetColor),
|
||||
targetSize: normalizeOptionalText(picture.targetSize),
|
||||
targetCombi: normalizeOptionalText(picture.targetCombi),
|
||||
targetEan13: normalizeOptionalText(picture.targetEan13),
|
||||
targetIdProductCatalog: normalizePositiveNumber(picture.targetIdProductCatalog),
|
||||
matchConfidence: normalizeConfidence(picture.matchConfidence),
|
||||
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
|
||||
duplicateOfCandidateId: duplicate?.id || null,
|
||||
});
|
||||
|
||||
await connection.commit();
|
||||
return {
|
||||
saved: true,
|
||||
candidateId,
|
||||
imageBlobId,
|
||||
sha256,
|
||||
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
|
||||
duplicateOfCandidateId: duplicate?.id || null,
|
||||
mimeType: image.mimeType,
|
||||
width: normalizePositiveNumber(picture.width) || null,
|
||||
height: normalizePositiveNumber(picture.height) || null,
|
||||
sizeBytes: image.buffer.byteLength,
|
||||
sourceKey,
|
||||
sourceName,
|
||||
};
|
||||
} catch (error) {
|
||||
await connection.rollback();
|
||||
throw error;
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(url) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 20000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Image download failed with HTTP ${response.status}.`);
|
||||
}
|
||||
|
||||
const mimeType = String(response.headers.get("content-type") || "application/octet-stream")
|
||||
.split(";")[0]
|
||||
.toLowerCase();
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
throw new Error(`Downloaded content is not an image (${mimeType}).`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (!buffer.length) {
|
||||
throw new Error("Downloaded image is empty.");
|
||||
}
|
||||
if (buffer.byteLength > 20 * 1024 * 1024) {
|
||||
throw new Error("Downloaded image is larger than 20 MB.");
|
||||
}
|
||||
|
||||
return { buffer, mimeType };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertSource(connection, source) {
|
||||
const baseUrl = new URL(source.imageUrl).origin;
|
||||
const sourceType = ["supplier", "manufacturer", "backup", "other"].includes(source.sourceType)
|
||||
? source.sourceType
|
||||
: "supplier";
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_sources (
|
||||
source_key,
|
||||
source_name,
|
||||
source_type,
|
||||
base_url,
|
||||
enabled
|
||||
)
|
||||
VALUES (?, ?, ?, ?, 1)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
source_name = VALUES(source_name),
|
||||
base_url = VALUES(base_url),
|
||||
enabled = VALUES(enabled)
|
||||
`,
|
||||
[source.sourceKey, source.sourceName, sourceType, baseUrl],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
async function upsertImageBlob(connection, image) {
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_image_blobs (
|
||||
sha256,
|
||||
mime_type,
|
||||
width,
|
||||
height,
|
||||
size_bytes,
|
||||
image_data
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
mime_type = VALUES(mime_type),
|
||||
width = COALESCE(VALUES(width), width),
|
||||
height = COALESCE(VALUES(height), height),
|
||||
size_bytes = VALUES(size_bytes)
|
||||
`,
|
||||
[image.sha256, image.mimeType, image.width, image.height, image.sizeBytes, image.buffer],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
async function findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl) {
|
||||
const rows = await connection.query(
|
||||
`
|
||||
SELECT id
|
||||
FROM scrape_picture_candidates
|
||||
WHERE image_blob_id = ?
|
||||
AND NOT (source_id = ? AND source_image_url = ?)
|
||||
ORDER BY id ASC
|
||||
LIMIT 1
|
||||
`,
|
||||
[imageBlobId, sourceId, imageUrl],
|
||||
);
|
||||
const items = normalizeRows(rows);
|
||||
return items[0] || null;
|
||||
}
|
||||
|
||||
async function upsertPictureCandidate(connection, candidate) {
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_picture_candidates (
|
||||
image_blob_id,
|
||||
source_id,
|
||||
manufacturer_id,
|
||||
id_product_catalog,
|
||||
id_product,
|
||||
supplier_reference,
|
||||
ean13,
|
||||
color,
|
||||
size_value,
|
||||
combi,
|
||||
source_page_url,
|
||||
source_image_url,
|
||||
source_position,
|
||||
target_strategy,
|
||||
target_color,
|
||||
target_size,
|
||||
target_combi,
|
||||
target_ean13,
|
||||
target_id_product_catalog,
|
||||
match_confidence,
|
||||
duplicate_status,
|
||||
duplicate_of_candidate_id,
|
||||
status
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
image_blob_id = VALUES(image_blob_id),
|
||||
manufacturer_id = VALUES(manufacturer_id),
|
||||
id_product_catalog = VALUES(id_product_catalog),
|
||||
id_product = VALUES(id_product),
|
||||
supplier_reference = VALUES(supplier_reference),
|
||||
ean13 = VALUES(ean13),
|
||||
color = VALUES(color),
|
||||
size_value = VALUES(size_value),
|
||||
combi = VALUES(combi),
|
||||
source_page_url = VALUES(source_page_url),
|
||||
source_position = VALUES(source_position),
|
||||
target_strategy = VALUES(target_strategy),
|
||||
target_color = VALUES(target_color),
|
||||
target_size = VALUES(target_size),
|
||||
target_combi = VALUES(target_combi),
|
||||
target_ean13 = VALUES(target_ean13),
|
||||
target_id_product_catalog = VALUES(target_id_product_catalog),
|
||||
match_confidence = VALUES(match_confidence),
|
||||
duplicate_status = VALUES(duplicate_status),
|
||||
duplicate_of_candidate_id = VALUES(duplicate_of_candidate_id),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`,
|
||||
[
|
||||
candidate.imageBlobId,
|
||||
candidate.sourceId,
|
||||
candidate.manufacturerId,
|
||||
candidate.idProductCatalog,
|
||||
candidate.idProduct,
|
||||
candidate.supplierReference,
|
||||
candidate.ean13,
|
||||
candidate.color,
|
||||
candidate.sizeValue,
|
||||
candidate.combi,
|
||||
candidate.sourcePageUrl,
|
||||
candidate.sourceImageUrl,
|
||||
candidate.sourcePosition,
|
||||
candidate.targetStrategy,
|
||||
candidate.targetColor,
|
||||
candidate.targetSize,
|
||||
candidate.targetCombi,
|
||||
candidate.targetEan13,
|
||||
candidate.targetIdProductCatalog,
|
||||
candidate.matchConfidence,
|
||||
candidate.duplicateStatus,
|
||||
candidate.duplicateOfCandidateId,
|
||||
],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
function normalizeRows(rows) {
|
||||
return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
|
||||
}
|
||||
|
||||
function normalizeRequiredUrl(value, name) {
|
||||
const text = normalizeText(value);
|
||||
if (!text) throw new Error(`${name} is required.`);
|
||||
const url = new URL(text);
|
||||
if (!["http:", "https:"].includes(url.protocol)) {
|
||||
throw new Error(`${name} must be an HTTP URL.`);
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function normalizeText(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function normalizeOptionalText(value) {
|
||||
const text = normalizeText(value);
|
||||
return text || null;
|
||||
}
|
||||
|
||||
function normalizePositiveNumber(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.trunc(number) : null;
|
||||
}
|
||||
|
||||
function normalizeConfidence(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return 0;
|
||||
return Math.max(0, Math.min(100, number));
|
||||
}
|
||||
|
||||
function normalizeTargetStrategy(value) {
|
||||
const strategy = normalizeText(value).toLowerCase();
|
||||
return ["unknown", "color", "size", "combination", "manual"].includes(strategy)
|
||||
? strategy
|
||||
: "unknown";
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
loadProductByEan,
|
||||
loadProductInfo,
|
||||
getProductPictures,
|
||||
saveProductPictureCandidate,
|
||||
openProductSource,
|
||||
} from "./services/catalog-products.ts";
|
||||
import { openInBrowser } from "./services/browser-adapter.ts";
|
||||
@@ -124,6 +125,12 @@ const server = http.createServer(async (request, response) => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/catalog-maker/save-picture" && request.method === "POST") {
|
||||
const body = await readJsonBody(request);
|
||||
await sendJson(response, await saveProductPictureCandidate(requestConfig, body));
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/catalog-maker/open-source") {
|
||||
await sendJson(
|
||||
response,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// @ts-nocheck
|
||||
import { executeEndpointWrite, queryEndpoint } from "../endpoint/endpoint-client.ts";
|
||||
import { saveScrapePictureCandidate } from "../db/scrape-image-store.ts";
|
||||
import { openInBrowser, scrapePage, searchInBrowser } from "./browser-adapter.ts";
|
||||
import {
|
||||
prepareManufacturerCatalogSqls,
|
||||
@@ -247,6 +248,10 @@ export async function getProductPictures(
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveProductPictureCandidate(config, picture) {
|
||||
return saveScrapePictureCandidate(config, picture);
|
||||
}
|
||||
|
||||
export async function openProductSource(config, { manufacturerId, sourceKey, url }) {
|
||||
const sourceUrl = normalizeSourceUrl(url);
|
||||
const mapping = await listMappingSources(config, { manufacturerId });
|
||||
|
||||
+14
-3
@@ -227,9 +227,15 @@
|
||||
.source-status, .source-row-action { @apply shrink-0 rounded px-1 text-[10px]; }
|
||||
.source-status.is-ok { @apply bg-emerald-100 text-emerald-700; }
|
||||
.source-status.is-muted { @apply bg-slate-200 text-slate-600; }
|
||||
.picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)] items-center gap-2 border-b border-slate-200 p-1 text-[10px]; }
|
||||
.picture-candidate img { @apply h-16 w-[72px] object-contain; }
|
||||
.picture-candidate span { @apply min-w-0 truncate; }
|
||||
.picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)_auto] items-center gap-2 border-b border-slate-200 p-2 text-[10px]; }
|
||||
.picture-candidate-preview { @apply flex h-16 w-[72px] items-center justify-center bg-white; }
|
||||
.picture-candidate img { @apply max-h-16 w-[72px] object-contain; }
|
||||
.picture-candidate-meta { @apply grid min-w-0 gap-0.5; }
|
||||
.picture-candidate-title { @apply min-w-0 truncate font-semibold text-slate-800; }
|
||||
.picture-candidate-dimensions,
|
||||
.picture-candidate-target,
|
||||
.picture-candidate-duplicate { @apply min-w-0 truncate text-slate-500; }
|
||||
.picture-candidate-save { @apply min-w-[76px]; }
|
||||
@media (max-width: 980px) {
|
||||
.catalog-screen { @apply flex flex-col pl-0; }
|
||||
.catalog-screen.sidebar-collapsed { @apply pl-0; }
|
||||
@@ -358,5 +364,10 @@
|
||||
html.dark .source-row { @apply border-slate-700 bg-slate-900 text-slate-100 hover:bg-slate-800; }
|
||||
html.dark .suggested-row,
|
||||
html.dark .picture-candidate { @apply border-slate-700; }
|
||||
html.dark .picture-candidate-preview { @apply bg-slate-800; }
|
||||
html.dark .picture-candidate-title { @apply text-slate-100; }
|
||||
html.dark .picture-candidate-dimensions,
|
||||
html.dark .picture-candidate-target,
|
||||
html.dark .picture-candidate-duplicate { @apply text-slate-400; }
|
||||
html.dark .suggested-row:nth-child(even) { @apply bg-slate-800; }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user