From d3d1147a3abaa44c0278e77fd71baaba836a4e6d Mon Sep 17 00:00:00 2001 From: rajch_ales Date: Thu, 6 Aug 2026 00:33:44 +0200 Subject: [PATCH] Prefer large Idealo gallery pictures --- agent.md | 1 + src/services/catalog-products.ts | 110 ++++++++++++++++++++++++++++- test/idealo-picture-parser.test.ts | 33 +++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 test/idealo-picture-parser.test.ts diff --git a/agent.md b/agent.md index 07b5e68..d347430 100644 --- a/agent.md +++ b/agent.md @@ -193,6 +193,7 @@ There are three selectable data sources: - Review UI: show picture size, source, target assignment, duplicate state, per-row save and batch save actions. - Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images. - Idealo uses the browser adapter and is intended mainly as a picture source. +- Idealo picture parsing must read product gallery images from Splide gallery markup (`splide__track`, `splide__list`, `splide__slide`) and choose the largest available URL from `srcset`, `data-srcset`, `data-large`, `data-original`, `data-src` or `src`. Do not use search-result thumbnails or small preview URLs as production picture candidates when gallery images exist. - Image preview is square. - Settings is at the bottom of the left panel; database status is directly above it. diff --git a/src/services/catalog-products.ts b/src/services/catalog-products.ts index 3605004..1395ee5 100644 --- a/src/services/catalog-products.ts +++ b/src/services/catalog-products.ts @@ -310,7 +310,11 @@ async function findSourceCandidates(source, product, { includePictures = false } attempt, product, ); - if (includePictures) candidate.pictures = extractPictureUrls(result.pageSource, result.url || attempt.url); + if (includePictures) { + candidate.pictures = source.key === "idealo" + ? extractIdealoGalleryPictureUrls(result.pageSource, result.url || attempt.url) + : extractPictureUrls(result.pageSource, result.url || attempt.url); + } return { source, query: attempt.query, @@ -381,6 +385,8 @@ async function findSourceCandidates(source, product, { includePictures = false } for (const candidate of candidates) { if (source.key === "hudy" && candidate.url) { candidate.pictures = await extractPicturesFromPage(candidate.url, source.browserEngine); + } else if (source.key === "idealo") { + candidate.pictures = extractIdealoGalleryPictureUrls(html, response.url || attempt.url); } else { candidate.pictures = extractPictureUrls(html, response.url || attempt.url); } @@ -742,6 +748,108 @@ function extractPictureUrls(html, pageUrl) { return urls.slice(0, 12); } +export function extractIdealoGalleryPictureUrls(html, pageUrl) { + const source = String(html ?? ""); + const galleryMatches = [ + ...source.matchAll(/]*class=["'][^"']*splide__slide[^"']*["'][^>]*>[\s\S]*?<\/li>/gi), + ]; + const orderedGalleryUrls = []; + + for (const match of galleryMatches) { + const best = pickBestImageUrlFromMarkup(match[0], pageUrl); + if (best && !orderedGalleryUrls.includes(best)) orderedGalleryUrls.push(best); + } + + if (orderedGalleryUrls.length) return orderedGalleryUrls.slice(0, 12); + + const trackMatch = source.match(/]*class=["'][^"']*splide__track[^"']*["'][^>]*>[\s\S]*?<\/div>/i); + if (trackMatch) { + const best = pickBestImageUrlFromMarkup(trackMatch[0], pageUrl); + if (best) return [best]; + } + + return extractPictureUrls(source, pageUrl); +} + +function pickBestImageUrlFromMarkup(markup, pageUrl) { + const candidates = []; + const add = (value, descriptorScore = 0) => { + const normalized = normalizeImageCandidateUrl(value, pageUrl); + if (!normalized) return; + candidates.push({ + url: normalized, + score: scoreImageCandidate(normalized, descriptorScore), + }); + }; + + for (const attribute of ["srcset", "data-srcset"]) { + for (const match of markup.matchAll(new RegExp(`${attribute}=["']([^"']+)`, "gi"))) { + for (const candidate of parseSrcsetCandidates(match[1], pageUrl)) { + add(candidate.url, candidate.score); + } + } + } + + for (const attribute of ["data-large", "data-original", "data-full", "data-zoom-image", "data-src", "src"]) { + for (const match of markup.matchAll(new RegExp(`${attribute}=["']([^"']+)`, "gi"))) { + add(match[1]); + } + } + + candidates.sort((a, b) => b.score - a.score); + return candidates[0]?.url || ""; +} + +function parseSrcsetCandidates(value, pageUrl) { + return String(value ?? "") + .split(",") + .map((part) => { + const [urlPart, descriptor = ""] = part.trim().split(/\s+/); + const descriptorScore = /(\d+)w/i.test(descriptor) + ? Number(descriptor.match(/(\d+)w/i)?.[1] || 0) + : /(\d+(?:\.\d+)?)x/i.test(descriptor) + ? Number(descriptor.match(/(\d+(?:\.\d+)?)x/i)?.[1] || 0) * 400 + : 0; + return { + url: normalizeImageCandidateUrl(urlPart, pageUrl), + score: descriptorScore, + }; + }) + .filter((candidate) => candidate.url); +} + +function normalizeImageCandidateUrl(value, pageUrl) { + const decoded = decodeHtmlEntities(String(value ?? "").trim()); + if (!decoded || decoded.startsWith("data:") || /\.svg(?:[?#]|$)/i.test(decoded)) return ""; + + try { + const url = new URL(decoded.startsWith("//") ? `https:${decoded}` : decoded, pageUrl).href; + if (!/^https?:/i.test(url)) return ""; + if (/(?:logo|icon|sprite|favicon|placeholder)/i.test(url)) return ""; + return url; + } catch { + return ""; + } +} + +function scoreImageCandidate(url, descriptorScore = 0) { + const dimensionScore = Math.max( + ...[...String(url).matchAll(/(?:^|[^0-9])(\d{2,5})[x_-](\d{2,5})(?:[^0-9]|$)/g)].map( + (match) => Number(match[1]) * Number(match[2]), + ), + ...[...String(url).matchAll(/w(\d{2,5})h(\d{2,5})/gi)].map( + (match) => Number(match[1]) * Number(match[2]), + ), + 0, + ); + const largePathBonus = /(?:original|large|zoom|full|1000|1200|1500|2000)/i.test(url) ? 100000 : 0; + const thumbnailPenalty = /(?:thumb|thumbnail|small|preview|\/s\d+_|_s\d+|150x150|80x80)/i.test(url) + ? -100000 + : 0; + + return dimensionScore + descriptorScore + largePathBonus + thumbnailPenalty; +} + async function extractPicturesFromPage(url, browserEngine = "chromium") { try { const response = await scrapePage(url, { engine: browserEngine, waitAfterLoadMs: 750 }); diff --git a/test/idealo-picture-parser.test.ts b/test/idealo-picture-parser.test.ts new file mode 100644 index 0000000..cda5116 --- /dev/null +++ b/test/idealo-picture-parser.test.ts @@ -0,0 +1,33 @@ +// @ts-nocheck +import test from "node:test"; +import assert from "node:assert/strict"; +import { extractIdealoGalleryPictureUrls } from "../src/services/catalog-products.ts"; + +test("idealo gallery parser prefers large splide images over thumbnails", () => { + const html = ` +
+
    +
  • + +
    • +
  • + + + + +
    • +
+
`; + + const urls = extractIdealoGalleryPictureUrls(html, "https://www.idealo.de/product.html"); + + assert.deepEqual(urls, [ + "https://img.idealo.com/folder/Product/200123/4/200123456/s1_product_1000x1000.jpg", + "https://img.idealo.com/folder/Product/200123/4/200123456/s2_product_1200x1200.jpg", + ]); +}); +