Use configured browser for source windows
This commit is contained in:
@@ -70,6 +70,7 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
|
|||||||
- Do not implement new scraping with raw `fetch`, regex-only HTML parsing, Selenium or direct HTTP shortcuts.
|
- Do not implement new scraping with raw `fetch`, regex-only HTML parsing, Selenium or direct HTTP shortcuts.
|
||||||
- Scrapers must use controlled browser navigation, selectors, explicit waits and bounded request counts.
|
- Scrapers must use controlled browser navigation, selectors, explicit waits and bounded request counts.
|
||||||
- The configured browser engine for a source is authoritative. If a source says `firefox`, launch Firefox or fail loudly; do not silently fall back to Chromium, Edge or another browser.
|
- The configured browser engine for a source is authoritative. If a source says `firefox`, launch Firefox or fail loudly; do not silently fall back to Chromium, Edge or another browser.
|
||||||
|
- Visible source windows, URL health checks, source lookup and picture parsing must all use the same configured browser engine from that source's database setting.
|
||||||
- Keep scraping slow and observable, preserve source URL/status information, and handle blocked pages or missing selectors as explicit errors.
|
- Keep scraping slow and observable, preserve source URL/status information, and handle blocked pages or missing selectors as explicit errors.
|
||||||
- Supplier data remains a local draft until the user explicitly confirms an Apply / Save action.
|
- Supplier data remains a local draft until the user explicitly confirms an Apply / Save action.
|
||||||
|
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ export async function findProductSources(
|
|||||||
const candidate = item.candidates?.[0];
|
const candidate = item.candidates?.[0];
|
||||||
if (candidate?.url) {
|
if (candidate?.url) {
|
||||||
try {
|
try {
|
||||||
await openInBrowser(candidate.url);
|
await openInBrowser(candidate.url, { engine: item.source.browserEngine });
|
||||||
} catch {
|
} catch {
|
||||||
// Picture extraction can continue even if the visible browser cannot start.
|
// Picture extraction can continue even if the visible browser cannot start.
|
||||||
}
|
}
|
||||||
@@ -332,7 +332,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
|
|||||||
if (candidates.length) {
|
if (candidates.length) {
|
||||||
let verification = null;
|
let verification = null;
|
||||||
if (includePictures && source.key === "hudy" && product.color && candidates[0].url) {
|
if (includePictures && source.key === "hudy" && product.color && candidates[0].url) {
|
||||||
verification = await verifyHudyVariant(candidates[0].url, product);
|
verification = await verifyHudyVariant(candidates[0].url, product, source.browserEngine);
|
||||||
candidates[0].verification = verification;
|
candidates[0].verification = verification;
|
||||||
if (verification.matchedUrl) candidates[0].url = verification.matchedUrl;
|
if (verification.matchedUrl) candidates[0].url = verification.matchedUrl;
|
||||||
if (includePictures && verification.pictures?.length) {
|
if (includePictures && verification.pictures?.length) {
|
||||||
@@ -343,7 +343,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
|
|||||||
if (includePictures) {
|
if (includePictures) {
|
||||||
for (const candidate of candidates) {
|
for (const candidate of candidates) {
|
||||||
if (source.key === "hudy" && candidate.url) {
|
if (source.key === "hudy" && candidate.url) {
|
||||||
candidate.pictures = await extractPicturesFromPage(candidate.url);
|
candidate.pictures = await extractPicturesFromPage(candidate.url, source.browserEngine);
|
||||||
} else {
|
} else {
|
||||||
candidate.pictures = extractPictureUrls(html, response.url || attempt.url);
|
candidate.pictures = extractPictureUrls(html, response.url || attempt.url);
|
||||||
}
|
}
|
||||||
@@ -585,7 +585,7 @@ function findColorVariantUrl(html, pageUrl, color) {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
async function verifyHudyVariant(url, product) {
|
async function verifyHudyVariant(url, product, browserEngine = "chromium") {
|
||||||
try {
|
try {
|
||||||
const eanGroup = Array.isArray(product.eanGroup)
|
const eanGroup = Array.isArray(product.eanGroup)
|
||||||
? product.eanGroup
|
? product.eanGroup
|
||||||
@@ -603,7 +603,7 @@ async function verifyHudyVariant(url, product) {
|
|||||||
if (visited.has(page.url)) continue;
|
if (visited.has(page.url)) continue;
|
||||||
visited.add(page.url);
|
visited.add(page.url);
|
||||||
|
|
||||||
const response = await scrapePage(page.url, { waitAfterLoadMs: 750 });
|
const response = await scrapePage(page.url, { engine: browserEngine, waitAfterLoadMs: 750 });
|
||||||
const html = response.pageSource;
|
const html = response.pageSource;
|
||||||
lastStatus = response.status;
|
lastStatus = response.status;
|
||||||
const pageEans = extractPageEans(html);
|
const pageEans = extractPageEans(html);
|
||||||
@@ -705,9 +705,9 @@ function extractPictureUrls(html, pageUrl) {
|
|||||||
return urls.slice(0, 12);
|
return urls.slice(0, 12);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function extractPicturesFromPage(url) {
|
async function extractPicturesFromPage(url, browserEngine = "chromium") {
|
||||||
try {
|
try {
|
||||||
const response = await scrapePage(url, { waitAfterLoadMs: 750 });
|
const response = await scrapePage(url, { engine: browserEngine, waitAfterLoadMs: 750 });
|
||||||
return extractHudyGalleryPictureUrls(response.pageSource, response.url || url);
|
return extractHudyGalleryPictureUrls(response.pageSource, response.url || url);
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
|
|||||||
Reference in New Issue
Block a user