Store selected picture candidates in scrape DB
This commit is contained in:
@@ -166,6 +166,7 @@ There are three selectable data sources:
|
||||
- Each source row has its own `Get pictures` action.
|
||||
- Picture candidates must show source, image dimensions and the proposed target assignment. The target assignment is not always color: supported strategies are `color`, `size`, `combination` and `manual`. Start by using manual corrections from the user, then learn reusable rules in Local scrape DB.
|
||||
- Scraped pictures and their metadata belong in Local scrape DB, not browser localStorage. Store the binary image in `scrape_image_blobs` with `sha256`, `perceptual_hash`, mime type, dimensions and size. Store each source/target proposal in `scrape_picture_candidates`.
|
||||
- Every picture candidate row must have its own save action. Saving one candidate stores only that selected image candidate into Local scrape DB and never saves the whole source result list.
|
||||
- Use `sha256` to remove exact duplicate image files and a perceptual hash to flag visually similar images from different sources or sizes. Exact duplicates may be collapsed automatically; visual duplicates should remain reviewable until the user approves the rule.
|
||||
- Product source cache is cleared when a new product loads.
|
||||
- Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images.
|
||||
|
||||
+122
-6
@@ -856,21 +856,62 @@ function renderSourceCandidates(items) {
|
||||
}
|
||||
function renderPictureCandidates(items) {
|
||||
const rows = [];
|
||||
let position = 0;
|
||||
for (const item of items) {
|
||||
for (const candidate of item.candidates || []) {
|
||||
for (const picture of candidate.pictures || []) {
|
||||
const row = document.createElement("a");
|
||||
position += 1;
|
||||
const target = buildPictureTargetInfo();
|
||||
const row = document.createElement("div");
|
||||
row.className = "picture-candidate";
|
||||
row.href = picture;
|
||||
row.target = "_blank";
|
||||
row.rel = "noreferrer";
|
||||
const preview = document.createElement("a");
|
||||
preview.href = picture;
|
||||
preview.target = "_blank";
|
||||
preview.rel = "noreferrer";
|
||||
preview.className = "picture-candidate-preview";
|
||||
const image = document.createElement("img");
|
||||
image.src = picture;
|
||||
image.alt = candidate.rawTitle || item.source?.name || "Product picture";
|
||||
image.loading = "lazy";
|
||||
preview.append(image);
|
||||
const details = document.createElement("div");
|
||||
details.className = "picture-candidate-meta";
|
||||
const label = document.createElement("span");
|
||||
label.textContent = item.source?.name || item.source?.key || "Source";
|
||||
row.append(image, label);
|
||||
label.className = "picture-candidate-title";
|
||||
label.textContent = `${item.source?.name || item.source?.key || "Source"} picture #${position}`;
|
||||
const dimensions = document.createElement("span");
|
||||
dimensions.className = "picture-candidate-dimensions";
|
||||
dimensions.textContent = "Size: loading...";
|
||||
image.addEventListener("load", () => {
|
||||
dimensions.textContent = image.naturalWidth && image.naturalHeight
|
||||
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px`
|
||||
: "Size: unknown";
|
||||
});
|
||||
image.addEventListener("error", () => {
|
||||
dimensions.textContent = "Size: image preview failed";
|
||||
});
|
||||
const assignment = document.createElement("span");
|
||||
assignment.className = "picture-candidate-target";
|
||||
assignment.textContent = target.label;
|
||||
const duplicateHint = document.createElement("span");
|
||||
duplicateHint.className = "picture-candidate-duplicate";
|
||||
duplicateHint.textContent = "DB: not saved";
|
||||
details.append(label, dimensions, assignment, duplicateHint);
|
||||
const action = document.createElement("button");
|
||||
action.type = "button";
|
||||
action.className = "source-picture-action picture-candidate-save";
|
||||
action.textContent = "Save to DB";
|
||||
action.addEventListener("click", () => savePictureCandidateToDb({
|
||||
picture,
|
||||
item,
|
||||
candidate,
|
||||
image,
|
||||
position,
|
||||
target,
|
||||
action,
|
||||
duplicateHint,
|
||||
}));
|
||||
row.append(preview, details, action);
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
@@ -884,6 +925,81 @@ function renderPictureCandidates(items) {
|
||||
}
|
||||
sourceList.replaceChildren(...rows);
|
||||
}
|
||||
function buildPictureTargetInfo() {
|
||||
const supplierColorGroup = getFirstSupplierColorGroup();
|
||||
const firstSupplierRow = supplierColorGroup?.supplierRows?.[0] || currentCombinations[0] || {};
|
||||
const color = supplierColorGroup?.color || formatValue(firstSupplierRow.color);
|
||||
const ean13 = supplierColorGroup?.supplierEan || firstSupplierRow.ean13 || "";
|
||||
if (color) {
|
||||
return {
|
||||
strategy: "color",
|
||||
color,
|
||||
sizeValue: "",
|
||||
combi: "",
|
||||
ean13,
|
||||
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
label: `Target: color ${color}${ean13 ? ` / EAN ${ean13}` : ""}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
strategy: "unknown",
|
||||
color: "",
|
||||
sizeValue: "",
|
||||
combi: "",
|
||||
ean13,
|
||||
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
label: ean13 ? `Target: unknown / EAN ${ean13}` : "Target: needs manual assignment",
|
||||
};
|
||||
}
|
||||
async function savePictureCandidateToDb({ picture, item, candidate, image, position, target, action, duplicateHint, }) {
|
||||
action.disabled = true;
|
||||
action.textContent = "Saving...";
|
||||
duplicateHint.textContent = "DB: saving...";
|
||||
try {
|
||||
const payload = {
|
||||
sourceKey: item.source?.key || "",
|
||||
sourceName: item.source?.name || "",
|
||||
sourceType: item.source?.type || "",
|
||||
imageUrl: picture,
|
||||
pageUrl: candidate.url || item.searchUrl || "",
|
||||
sourcePosition: position,
|
||||
manufacturerId: manufacturerSelect.value,
|
||||
idProductCatalog: currentProduct?.idProductCatalog || target.idProductCatalog || "",
|
||||
idProduct: currentProduct?.idProduct || "",
|
||||
supplierReference: currentProduct?.supplierReference || "",
|
||||
ean13: target.ean13 || "",
|
||||
color: target.color || "",
|
||||
sizeValue: target.sizeValue || "",
|
||||
combi: target.combi || "",
|
||||
targetStrategy: target.strategy,
|
||||
targetColor: target.color || "",
|
||||
targetSize: target.sizeValue || "",
|
||||
targetCombi: target.combi || "",
|
||||
targetEan13: target.ean13 || "",
|
||||
targetIdProductCatalog: target.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
matchConfidence: target.strategy === "color" ? 70 : 0,
|
||||
width: image.naturalWidth || "",
|
||||
height: image.naturalHeight || "",
|
||||
};
|
||||
const response = await fetch("/api/catalog-maker/save-picture", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok)
|
||||
throw new Error(data.message || "Picture save failed.");
|
||||
action.textContent = "Saved";
|
||||
duplicateHint.textContent = data.duplicateStatus === "exact_duplicate"
|
||||
? `DB: duplicate of #${data.duplicateOfCandidateId || "known"}`
|
||||
: `DB: saved #${data.candidateId}`;
|
||||
}
|
||||
catch (error) {
|
||||
action.disabled = false;
|
||||
action.textContent = "Save to DB";
|
||||
duplicateHint.textContent = `DB: ${error.message || "save failed"}`;
|
||||
}
|
||||
}
|
||||
function collectPictureUrls(items) {
|
||||
return items.flatMap((item) => (item.candidates || []).flatMap((candidate) => candidate.pictures || [])).filter((url, index, all) => all.indexOf(url) === index);
|
||||
}
|
||||
|
||||
+145
-6
@@ -928,23 +928,74 @@ function renderSourceCandidates(items) {
|
||||
|
||||
function renderPictureCandidates(items) {
|
||||
const rows = [];
|
||||
let position = 0;
|
||||
for (const item of items) {
|
||||
for (const candidate of item.candidates || []) {
|
||||
for (const picture of candidate.pictures || []) {
|
||||
const row = document.createElement("a");
|
||||
position += 1;
|
||||
const target = buildPictureTargetInfo();
|
||||
const row = document.createElement("div");
|
||||
row.className = "picture-candidate";
|
||||
row.href = picture;
|
||||
row.target = "_blank";
|
||||
row.rel = "noreferrer";
|
||||
|
||||
const preview = document.createElement("a");
|
||||
preview.href = picture;
|
||||
preview.target = "_blank";
|
||||
preview.rel = "noreferrer";
|
||||
preview.className = "picture-candidate-preview";
|
||||
|
||||
const image = document.createElement("img");
|
||||
image.src = picture;
|
||||
image.alt = candidate.rawTitle || item.source?.name || "Product picture";
|
||||
image.loading = "lazy";
|
||||
preview.append(image);
|
||||
|
||||
const details = document.createElement("div");
|
||||
details.className = "picture-candidate-meta";
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = item.source?.name || item.source?.key || "Source";
|
||||
row.append(image, label);
|
||||
label.className = "picture-candidate-title";
|
||||
label.textContent = `${item.source?.name || item.source?.key || "Source"} picture #${position}`;
|
||||
|
||||
const dimensions = document.createElement("span");
|
||||
dimensions.className = "picture-candidate-dimensions";
|
||||
dimensions.textContent = "Size: loading...";
|
||||
image.addEventListener("load", () => {
|
||||
dimensions.textContent = image.naturalWidth && image.naturalHeight
|
||||
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px`
|
||||
: "Size: unknown";
|
||||
});
|
||||
image.addEventListener("error", () => {
|
||||
dimensions.textContent = "Size: image preview failed";
|
||||
});
|
||||
|
||||
const assignment = document.createElement("span");
|
||||
assignment.className = "picture-candidate-target";
|
||||
assignment.textContent = target.label;
|
||||
|
||||
const duplicateHint = document.createElement("span");
|
||||
duplicateHint.className = "picture-candidate-duplicate";
|
||||
duplicateHint.textContent = "DB: not saved";
|
||||
|
||||
details.append(label, dimensions, assignment, duplicateHint);
|
||||
|
||||
const action = document.createElement("button");
|
||||
action.type = "button";
|
||||
action.className = "source-picture-action picture-candidate-save";
|
||||
action.textContent = "Save to DB";
|
||||
action.addEventListener("click", () =>
|
||||
savePictureCandidateToDb({
|
||||
picture,
|
||||
item,
|
||||
candidate,
|
||||
image,
|
||||
position,
|
||||
target,
|
||||
action,
|
||||
duplicateHint,
|
||||
}),
|
||||
);
|
||||
|
||||
row.append(preview, details, action);
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
@@ -960,6 +1011,94 @@ function renderPictureCandidates(items) {
|
||||
sourceList.replaceChildren(...rows);
|
||||
}
|
||||
|
||||
function buildPictureTargetInfo() {
|
||||
const supplierColorGroup = getFirstSupplierColorGroup();
|
||||
const firstSupplierRow = supplierColorGroup?.supplierRows?.[0] || currentCombinations[0] || {};
|
||||
const color = supplierColorGroup?.color || formatValue(firstSupplierRow.color);
|
||||
const ean13 = supplierColorGroup?.supplierEan || firstSupplierRow.ean13 || "";
|
||||
|
||||
if (color) {
|
||||
return {
|
||||
strategy: "color",
|
||||
color,
|
||||
sizeValue: "",
|
||||
combi: "",
|
||||
ean13,
|
||||
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
label: `Target: color ${color}${ean13 ? ` / EAN ${ean13}` : ""}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
strategy: "unknown",
|
||||
color: "",
|
||||
sizeValue: "",
|
||||
combi: "",
|
||||
ean13,
|
||||
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
label: ean13 ? `Target: unknown / EAN ${ean13}` : "Target: needs manual assignment",
|
||||
};
|
||||
}
|
||||
|
||||
async function savePictureCandidateToDb({
|
||||
picture,
|
||||
item,
|
||||
candidate,
|
||||
image,
|
||||
position,
|
||||
target,
|
||||
action,
|
||||
duplicateHint,
|
||||
}) {
|
||||
action.disabled = true;
|
||||
action.textContent = "Saving...";
|
||||
duplicateHint.textContent = "DB: saving...";
|
||||
|
||||
try {
|
||||
const payload = {
|
||||
sourceKey: item.source?.key || "",
|
||||
sourceName: item.source?.name || "",
|
||||
sourceType: item.source?.type || "",
|
||||
imageUrl: picture,
|
||||
pageUrl: candidate.url || item.searchUrl || "",
|
||||
sourcePosition: position,
|
||||
manufacturerId: manufacturerSelect.value,
|
||||
idProductCatalog: currentProduct?.idProductCatalog || target.idProductCatalog || "",
|
||||
idProduct: currentProduct?.idProduct || "",
|
||||
supplierReference: currentProduct?.supplierReference || "",
|
||||
ean13: target.ean13 || "",
|
||||
color: target.color || "",
|
||||
sizeValue: target.sizeValue || "",
|
||||
combi: target.combi || "",
|
||||
targetStrategy: target.strategy,
|
||||
targetColor: target.color || "",
|
||||
targetSize: target.sizeValue || "",
|
||||
targetCombi: target.combi || "",
|
||||
targetEan13: target.ean13 || "",
|
||||
targetIdProductCatalog: target.idProductCatalog || currentProduct?.idProductCatalog || "",
|
||||
matchConfidence: target.strategy === "color" ? 70 : 0,
|
||||
width: image.naturalWidth || "",
|
||||
height: image.naturalHeight || "",
|
||||
};
|
||||
const response = await fetch("/api/catalog-maker/save-picture", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.message || "Picture save failed.");
|
||||
|
||||
action.textContent = "Saved";
|
||||
duplicateHint.textContent = data.duplicateStatus === "exact_duplicate"
|
||||
? `DB: duplicate of #${data.duplicateOfCandidateId || "known"}`
|
||||
: `DB: saved #${data.candidateId}`;
|
||||
} catch (error) {
|
||||
action.disabled = false;
|
||||
action.textContent = "Save to DB";
|
||||
duplicateHint.textContent = `DB: ${error.message || "save failed"}`;
|
||||
}
|
||||
}
|
||||
|
||||
function collectPictureUrls(items) {
|
||||
return items.flatMap((item) =>
|
||||
(item.candidates || []).flatMap((candidate) => candidate.pictures || []),
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,319 @@
|
||||
// @ts-nocheck
|
||||
import crypto from "node:crypto";
|
||||
import mariadb from "mariadb";
|
||||
|
||||
export async function saveScrapePictureCandidate(config, picture) {
|
||||
if (config.database?.driver !== "mariadb") {
|
||||
throw new Error("Local scrape DB needs MariaDB configuration.");
|
||||
}
|
||||
if (!config.database?.allowLocalWrites) {
|
||||
throw new Error("Local scrape DB writes are disabled. Enable writes in Settings first.");
|
||||
}
|
||||
|
||||
const imageUrl = normalizeRequiredUrl(picture.imageUrl, "imageUrl");
|
||||
const sourceKey = normalizeText(picture.sourceKey) || "unknown";
|
||||
const sourceName = normalizeText(picture.sourceName) || sourceKey;
|
||||
const sourcePageUrl = normalizeOptionalText(picture.pageUrl);
|
||||
const image = await downloadImage(imageUrl);
|
||||
const sha256 = crypto.createHash("sha256").update(image.buffer).digest("hex");
|
||||
const scrapeDb = config.database.scrapeName || "catalog_scrape";
|
||||
|
||||
const connection = await mariadb.createConnection({
|
||||
host: config.database.host,
|
||||
port: Number(config.database.port),
|
||||
database: scrapeDb,
|
||||
user: config.database.user,
|
||||
password: config.database.password,
|
||||
bigIntAsNumber: true,
|
||||
});
|
||||
|
||||
try {
|
||||
await connection.beginTransaction();
|
||||
const sourceId = await upsertSource(connection, {
|
||||
sourceKey,
|
||||
sourceName,
|
||||
imageUrl,
|
||||
sourceType: normalizeText(picture.sourceType) || "supplier",
|
||||
});
|
||||
const imageBlobId = await upsertImageBlob(connection, {
|
||||
sha256,
|
||||
mimeType: image.mimeType,
|
||||
width: normalizePositiveNumber(picture.width),
|
||||
height: normalizePositiveNumber(picture.height),
|
||||
sizeBytes: image.buffer.byteLength,
|
||||
buffer: image.buffer,
|
||||
});
|
||||
const duplicate = await findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl);
|
||||
const candidateId = await upsertPictureCandidate(connection, {
|
||||
imageBlobId,
|
||||
sourceId,
|
||||
manufacturerId: normalizePositiveNumber(picture.manufacturerId),
|
||||
idProductCatalog: normalizePositiveNumber(picture.idProductCatalog),
|
||||
idProduct: normalizePositiveNumber(picture.idProduct),
|
||||
supplierReference: normalizeOptionalText(picture.supplierReference),
|
||||
ean13: normalizeOptionalText(picture.ean13),
|
||||
color: normalizeOptionalText(picture.color),
|
||||
sizeValue: normalizeOptionalText(picture.sizeValue),
|
||||
combi: normalizeOptionalText(picture.combi),
|
||||
sourcePageUrl,
|
||||
sourceImageUrl: imageUrl,
|
||||
sourcePosition: normalizePositiveNumber(picture.sourcePosition) || 0,
|
||||
targetStrategy: normalizeTargetStrategy(picture.targetStrategy),
|
||||
targetColor: normalizeOptionalText(picture.targetColor),
|
||||
targetSize: normalizeOptionalText(picture.targetSize),
|
||||
targetCombi: normalizeOptionalText(picture.targetCombi),
|
||||
targetEan13: normalizeOptionalText(picture.targetEan13),
|
||||
targetIdProductCatalog: normalizePositiveNumber(picture.targetIdProductCatalog),
|
||||
matchConfidence: normalizeConfidence(picture.matchConfidence),
|
||||
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
|
||||
duplicateOfCandidateId: duplicate?.id || null,
|
||||
});
|
||||
|
||||
await connection.commit();
|
||||
return {
|
||||
saved: true,
|
||||
candidateId,
|
||||
imageBlobId,
|
||||
sha256,
|
||||
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
|
||||
duplicateOfCandidateId: duplicate?.id || null,
|
||||
mimeType: image.mimeType,
|
||||
width: normalizePositiveNumber(picture.width) || null,
|
||||
height: normalizePositiveNumber(picture.height) || null,
|
||||
sizeBytes: image.buffer.byteLength,
|
||||
sourceKey,
|
||||
sourceName,
|
||||
};
|
||||
} catch (error) {
|
||||
await connection.rollback();
|
||||
throw error;
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(url) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 20000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Image download failed with HTTP ${response.status}.`);
|
||||
}
|
||||
|
||||
const mimeType = String(response.headers.get("content-type") || "application/octet-stream")
|
||||
.split(";")[0]
|
||||
.toLowerCase();
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
throw new Error(`Downloaded content is not an image (${mimeType}).`);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (!buffer.length) {
|
||||
throw new Error("Downloaded image is empty.");
|
||||
}
|
||||
if (buffer.byteLength > 20 * 1024 * 1024) {
|
||||
throw new Error("Downloaded image is larger than 20 MB.");
|
||||
}
|
||||
|
||||
return { buffer, mimeType };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function upsertSource(connection, source) {
|
||||
const baseUrl = new URL(source.imageUrl).origin;
|
||||
const sourceType = ["supplier", "manufacturer", "backup", "other"].includes(source.sourceType)
|
||||
? source.sourceType
|
||||
: "supplier";
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_sources (
|
||||
source_key,
|
||||
source_name,
|
||||
source_type,
|
||||
base_url,
|
||||
enabled
|
||||
)
|
||||
VALUES (?, ?, ?, ?, 1)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
source_name = VALUES(source_name),
|
||||
base_url = VALUES(base_url),
|
||||
enabled = VALUES(enabled)
|
||||
`,
|
||||
[source.sourceKey, source.sourceName, sourceType, baseUrl],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
async function upsertImageBlob(connection, image) {
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_image_blobs (
|
||||
sha256,
|
||||
mime_type,
|
||||
width,
|
||||
height,
|
||||
size_bytes,
|
||||
image_data
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
mime_type = VALUES(mime_type),
|
||||
width = COALESCE(VALUES(width), width),
|
||||
height = COALESCE(VALUES(height), height),
|
||||
size_bytes = VALUES(size_bytes)
|
||||
`,
|
||||
[image.sha256, image.mimeType, image.width, image.height, image.sizeBytes, image.buffer],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
async function findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl) {
|
||||
const rows = await connection.query(
|
||||
`
|
||||
SELECT id
|
||||
FROM scrape_picture_candidates
|
||||
WHERE image_blob_id = ?
|
||||
AND NOT (source_id = ? AND source_image_url = ?)
|
||||
ORDER BY id ASC
|
||||
LIMIT 1
|
||||
`,
|
||||
[imageBlobId, sourceId, imageUrl],
|
||||
);
|
||||
const items = normalizeRows(rows);
|
||||
return items[0] || null;
|
||||
}
|
||||
|
||||
async function upsertPictureCandidate(connection, candidate) {
|
||||
const result = await connection.query(
|
||||
`
|
||||
INSERT INTO scrape_picture_candidates (
|
||||
image_blob_id,
|
||||
source_id,
|
||||
manufacturer_id,
|
||||
id_product_catalog,
|
||||
id_product,
|
||||
supplier_reference,
|
||||
ean13,
|
||||
color,
|
||||
size_value,
|
||||
combi,
|
||||
source_page_url,
|
||||
source_image_url,
|
||||
source_position,
|
||||
target_strategy,
|
||||
target_color,
|
||||
target_size,
|
||||
target_combi,
|
||||
target_ean13,
|
||||
target_id_product_catalog,
|
||||
match_confidence,
|
||||
duplicate_status,
|
||||
duplicate_of_candidate_id,
|
||||
status
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
id = LAST_INSERT_ID(id),
|
||||
image_blob_id = VALUES(image_blob_id),
|
||||
manufacturer_id = VALUES(manufacturer_id),
|
||||
id_product_catalog = VALUES(id_product_catalog),
|
||||
id_product = VALUES(id_product),
|
||||
supplier_reference = VALUES(supplier_reference),
|
||||
ean13 = VALUES(ean13),
|
||||
color = VALUES(color),
|
||||
size_value = VALUES(size_value),
|
||||
combi = VALUES(combi),
|
||||
source_page_url = VALUES(source_page_url),
|
||||
source_position = VALUES(source_position),
|
||||
target_strategy = VALUES(target_strategy),
|
||||
target_color = VALUES(target_color),
|
||||
target_size = VALUES(target_size),
|
||||
target_combi = VALUES(target_combi),
|
||||
target_ean13 = VALUES(target_ean13),
|
||||
target_id_product_catalog = VALUES(target_id_product_catalog),
|
||||
match_confidence = VALUES(match_confidence),
|
||||
duplicate_status = VALUES(duplicate_status),
|
||||
duplicate_of_candidate_id = VALUES(duplicate_of_candidate_id),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`,
|
||||
[
|
||||
candidate.imageBlobId,
|
||||
candidate.sourceId,
|
||||
candidate.manufacturerId,
|
||||
candidate.idProductCatalog,
|
||||
candidate.idProduct,
|
||||
candidate.supplierReference,
|
||||
candidate.ean13,
|
||||
candidate.color,
|
||||
candidate.sizeValue,
|
||||
candidate.combi,
|
||||
candidate.sourcePageUrl,
|
||||
candidate.sourceImageUrl,
|
||||
candidate.sourcePosition,
|
||||
candidate.targetStrategy,
|
||||
candidate.targetColor,
|
||||
candidate.targetSize,
|
||||
candidate.targetCombi,
|
||||
candidate.targetEan13,
|
||||
candidate.targetIdProductCatalog,
|
||||
candidate.matchConfidence,
|
||||
candidate.duplicateStatus,
|
||||
candidate.duplicateOfCandidateId,
|
||||
],
|
||||
);
|
||||
|
||||
return Number(result.insertId);
|
||||
}
|
||||
|
||||
function normalizeRows(rows) {
|
||||
return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
|
||||
}
|
||||
|
||||
function normalizeRequiredUrl(value, name) {
|
||||
const text = normalizeText(value);
|
||||
if (!text) throw new Error(`${name} is required.`);
|
||||
const url = new URL(text);
|
||||
if (!["http:", "https:"].includes(url.protocol)) {
|
||||
throw new Error(`${name} must be an HTTP URL.`);
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function normalizeText(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function normalizeOptionalText(value) {
|
||||
const text = normalizeText(value);
|
||||
return text || null;
|
||||
}
|
||||
|
||||
function normalizePositiveNumber(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.trunc(number) : null;
|
||||
}
|
||||
|
||||
function normalizeConfidence(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return 0;
|
||||
return Math.max(0, Math.min(100, number));
|
||||
}
|
||||
|
||||
function normalizeTargetStrategy(value) {
|
||||
const strategy = normalizeText(value).toLowerCase();
|
||||
return ["unknown", "color", "size", "combination", "manual"].includes(strategy)
|
||||
? strategy
|
||||
: "unknown";
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
loadProductByEan,
|
||||
loadProductInfo,
|
||||
getProductPictures,
|
||||
saveProductPictureCandidate,
|
||||
openProductSource,
|
||||
} from "./services/catalog-products.ts";
|
||||
import { openInBrowser } from "./services/browser-adapter.ts";
|
||||
@@ -124,6 +125,12 @@ const server = http.createServer(async (request, response) => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/catalog-maker/save-picture" && request.method === "POST") {
|
||||
const body = await readJsonBody(request);
|
||||
await sendJson(response, await saveProductPictureCandidate(requestConfig, body));
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/catalog-maker/open-source") {
|
||||
await sendJson(
|
||||
response,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// @ts-nocheck
|
||||
import { executeEndpointWrite, queryEndpoint } from "../endpoint/endpoint-client.ts";
|
||||
import { saveScrapePictureCandidate } from "../db/scrape-image-store.ts";
|
||||
import { openInBrowser, scrapePage, searchInBrowser } from "./browser-adapter.ts";
|
||||
import {
|
||||
prepareManufacturerCatalogSqls,
|
||||
@@ -247,6 +248,10 @@ export async function getProductPictures(
|
||||
};
|
||||
}
|
||||
|
||||
export async function saveProductPictureCandidate(config, picture) {
|
||||
return saveScrapePictureCandidate(config, picture);
|
||||
}
|
||||
|
||||
export async function openProductSource(config, { manufacturerId, sourceKey, url }) {
|
||||
const sourceUrl = normalizeSourceUrl(url);
|
||||
const mapping = await listMappingSources(config, { manufacturerId });
|
||||
|
||||
+14
-3
@@ -227,9 +227,15 @@
|
||||
.source-status, .source-row-action { @apply shrink-0 rounded px-1 text-[10px]; }
|
||||
.source-status.is-ok { @apply bg-emerald-100 text-emerald-700; }
|
||||
.source-status.is-muted { @apply bg-slate-200 text-slate-600; }
|
||||
.picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)] items-center gap-2 border-b border-slate-200 p-1 text-[10px]; }
|
||||
.picture-candidate img { @apply h-16 w-[72px] object-contain; }
|
||||
.picture-candidate span { @apply min-w-0 truncate; }
|
||||
.picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)_auto] items-center gap-2 border-b border-slate-200 p-2 text-[10px]; }
|
||||
.picture-candidate-preview { @apply flex h-16 w-[72px] items-center justify-center bg-white; }
|
||||
.picture-candidate img { @apply max-h-16 w-[72px] object-contain; }
|
||||
.picture-candidate-meta { @apply grid min-w-0 gap-0.5; }
|
||||
.picture-candidate-title { @apply min-w-0 truncate font-semibold text-slate-800; }
|
||||
.picture-candidate-dimensions,
|
||||
.picture-candidate-target,
|
||||
.picture-candidate-duplicate { @apply min-w-0 truncate text-slate-500; }
|
||||
.picture-candidate-save { @apply min-w-[76px]; }
|
||||
@media (max-width: 980px) {
|
||||
.catalog-screen { @apply flex flex-col pl-0; }
|
||||
.catalog-screen.sidebar-collapsed { @apply pl-0; }
|
||||
@@ -358,5 +364,10 @@
|
||||
html.dark .source-row { @apply border-slate-700 bg-slate-900 text-slate-100 hover:bg-slate-800; }
|
||||
html.dark .suggested-row,
|
||||
html.dark .picture-candidate { @apply border-slate-700; }
|
||||
html.dark .picture-candidate-preview { @apply bg-slate-800; }
|
||||
html.dark .picture-candidate-title { @apply text-slate-100; }
|
||||
html.dark .picture-candidate-dimensions,
|
||||
html.dark .picture-candidate-target,
|
||||
html.dark .picture-candidate-duplicate { @apply text-slate-400; }
|
||||
html.dark .suggested-row:nth-child(even) { @apply bg-slate-800; }
|
||||
}
|
||||
|
||||
@@ -138,6 +138,89 @@ test("main workspace keeps a bottom horizontal scrollbar for wide data", async (
|
||||
expect(scrollState.scrollWidth).toBeGreaterThan(scrollState.clientWidth);
|
||||
});
|
||||
|
||||
test("picture candidates show source, dimensions, target assignment and save action", async ({ page }) => {
|
||||
await page.route("**/api/manufacturers", async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({ items: [{ id: 13, name: "La Sportiva" }] }),
|
||||
});
|
||||
});
|
||||
await page.route("**/api/languages", async (route) => {
|
||||
await route.fulfill({ contentType: "application/json", body: JSON.stringify({ items: [] }) });
|
||||
});
|
||||
await page.route("**/api/catalog-maker/mapping-sources**", async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
mapped: true,
|
||||
items: [{ key: "hudy", name: "Hudy", enabled: true, health: "ok" }],
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route("**/api/catalog-maker/product**", async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
product: {
|
||||
idProductCatalog: 135520,
|
||||
idProduct: 110927,
|
||||
idManufacturer: 13,
|
||||
codeManufacture: "LAS",
|
||||
supplierReference: "ZFHS099",
|
||||
name: "Aequilibrium Trek Woman GTX",
|
||||
position: 1,
|
||||
total: 380,
|
||||
},
|
||||
combinations: [
|
||||
{
|
||||
idProductCatalog: 110923,
|
||||
status: "SS25/Boty",
|
||||
color: "Carbon/Malibu Blue",
|
||||
combination: "36",
|
||||
ean13: "8058428191284",
|
||||
supplierStock: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route("**/api/catalog-maker/get-pictures**", async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
items: [
|
||||
{
|
||||
source: { key: "hudy", name: "Hudy", type: "supplier" },
|
||||
searchUrl: "https://www.hudy.cz/vyhledavani?q=8058428191284",
|
||||
candidates: [
|
||||
{
|
||||
rawTitle: "Aequilibrium Trek Woman GTX",
|
||||
url: "https://www.hudy.cz/product",
|
||||
pictures: [
|
||||
"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='80' height='64'%3E%3Crect width='80' height='64' fill='red'/%3E%3C/svg%3E",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem("catalog-maker:selected-manufacturer", "13");
|
||||
});
|
||||
await page.goto("/");
|
||||
await page.waitForFunction(() => document.querySelector("#manufacturerSelect")?.value === "13");
|
||||
await page.locator("#getPicturesButton").click();
|
||||
|
||||
await expect(page.locator("#sourceModal")).toBeVisible();
|
||||
await expect(page.locator(".picture-candidate-title")).toHaveText("Hudy picture #1");
|
||||
await expect(page.locator(".picture-candidate-dimensions")).toContainText("Size:");
|
||||
await expect(page.locator(".picture-candidate-target")).toContainText("Target: color Carbon/Malibu Blue");
|
||||
await expect(page.locator(".picture-candidate-save")).toHaveText("Save to DB");
|
||||
});
|
||||
|
||||
test("uses one shared height for the top panel headers", async ({ page }) => {
|
||||
await page.goto("/");
|
||||
const heights = await page.locator("#sidebarDrawer .sidebar-drawer-header, .secondary-panel .column-header, .main-workspace .column-header").evaluateAll((elements) => elements.map((element) => Math.round(element.getBoundingClientRect().height)));
|
||||
|
||||
Reference in New Issue
Block a user