From 6f2d99875472adca5a2f7931f557c45d014dd191 Mon Sep 17 00:00:00 2001
From: rajch_ales <rajch_ales@ma-al.com>
Date: Thu, 6 Aug 2026 00:39:50 +0200
Subject: [PATCH] Run source lookup headless and reject Idealo thumbnails

---
 agent.md                           |  3 ++-
 src/services/catalog-products.ts   | 11 +++--------
 test/idealo-picture-parser.test.ts | 12 ++++++++++++
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/agent.md b/agent.md
index d347430..ea084c7 100644
--- a/agent.md
+++ b/agent.md
@@ -73,6 +73,7 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
 - Do not implement new scraping with raw `fetch`, regex-only HTML parsing, Selenium or direct HTTP shortcuts.
 - Scrapers must use controlled browser navigation, selectors, explicit waits and bounded request counts.
 - The configured browser engine for a source is authoritative. If a source says `firefox`, launch Firefox or fail loudly; do not silently fall back to Chromium, Edge or another browser.
+- Automated source lookup, mapping checks and picture lookup must run headless/in the background by default. Open a visible browser only for explicit user `Open` actions or deliberate diagnostics after an error.
 - Visible source windows, URL health checks, source lookup and picture parsing must all use the same configured browser engine from that source's database setting.
 - Source candidate `Open` actions in the UI must call the backend source opener with `manufacturerId`, `sourceKey` and URL. Do not use plain `target="_blank"` source links for mapped supplier/manufacturer sources because that ignores the configured browser engine.
 - Keep scraping slow and observable, preserve source URL/status information, and handle blocked pages or missing selectors as explicit errors.
@@ -193,7 +194,7 @@ There are three selectable data sources:
 - Review UI: show picture size, source, target assignment, duplicate state, per-row save and batch save actions.
 - Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images.
 - Idealo uses the browser adapter and is intended mainly as a picture source.
-- Idealo picture parsing must read product gallery images from Splide gallery markup (`splide__track`, `splide__list`, `splide__slide`) and choose the largest available URL from `srcset`, `data-srcset`, `data-large`, `data-original`, `data-src` or `src`. Do not use search-result thumbnails or small preview URLs as production picture candidates when gallery images exist.
+- Idealo picture parsing must read product gallery images from Splide gallery markup (`splide__track`, `splide__list`, `splide__slide`) and choose the largest available URL from `srcset`, `data-srcset`, `data-large`, `data-original`, `data-src` or `src`. Do not use search-result thumbnails or small preview URLs as production picture candidates. If no Idealo gallery is found, return no Idealo pictures and show the quality/missing warning instead of saving thumbnails.
 - Image preview is square.
 - Settings is at the bottom of the left panel; database status is directly above it.
 
diff --git a/src/services/catalog-products.ts b/src/services/catalog-products.ts
index 1395ee5..60b5f4f 100644
--- a/src/services/catalog-products.ts
+++ b/src/services/catalog-products.ts
@@ -299,7 +299,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
         const result = await searchInBrowser({
           url: attempt.url,
           engine: source.browserEngine,
-          headless: source.browserHeadless,
+          headless: true,
         waitAfterLoadMs: source.waitAfterLoadMs,
       });
 
@@ -333,12 +333,6 @@ async function findSourceCandidates(source, product, { includePictures = false }
         candidates: buildManualSearchCandidates(searchAttempts, product),
       };
     } catch (error) {
-      try {
-        await openInBrowser(attempt.url, { engine: source.browserEngine });
-      } catch {
-        // The manual candidate below remains available even if the browser cannot start.
-      }
-
       return {
         source,
         query: attempt.query,
@@ -355,6 +349,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
     try {
       const response = await scrapePage(attempt.url, {
         engine: source.browserEngine,
+        headless: true,
         waitAfterLoadMs: source.waitAfterLoadMs,
       });
       const html = response.pageSource;
@@ -768,7 +763,7 @@ export function extractIdealoGalleryPictureUrls(html, pageUrl) {
     if (best) return [best];
   }
 
-  return extractPictureUrls(source, pageUrl);
+  return [];
 }
 
 function pickBestImageUrlFromMarkup(markup, pageUrl) {
diff --git a/test/idealo-picture-parser.test.ts b/test/idealo-picture-parser.test.ts
index cda5116..01d296b 100644
--- a/test/idealo-picture-parser.test.ts
+++ b/test/idealo-picture-parser.test.ts
@@ -31,3 +31,15 @@ test("idealo gallery parser prefers large splide images over thumbnails", () =>
   ]);
 });
 
+test("idealo gallery parser does not fall back to search thumbnails", () => {
+  const html = `
+    <main>
+      <article class="search-result">
+        <img src="//img.idealo.com/folder/Product/200123/4/200123456/result_thumb_150x150.jpg">
+      </article>
+    </main>`;
+
+  const urls = extractIdealoGalleryPictureUrls(html, "https://www.idealo.de/preisvergleich/MainSearchProductCategory.html?q=123");
+
+  assert.deepEqual(urls, []);
+});