Use configured Playwright browser engines
This commit is contained in:
@@ -1,18 +1,18 @@
|
|||||||
// @ts-nocheck
|
// @ts-nocheck
|
||||||
import { chromium } from "playwright";
|
import { chromium, firefox } from "playwright";
|
||||||
|
|
||||||
const visibleBrowsers = new Set();
|
const visibleBrowsers = new Set();
|
||||||
|
|
||||||
export async function openInBrowser(url) {
|
export async function openInBrowser(url, { engine = "chromium" } = {}) {
|
||||||
const browser = await chromium.launch({ headless: false });
|
const browser = await launchBrowser(engine, false);
|
||||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||||
visibleBrowsers.add(browser);
|
visibleBrowsers.add(browser);
|
||||||
return { browser, page };
|
return { browser, page };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function searchInBrowser({ url, headless = true, waitAfterLoadMs = 3000 }) {
|
export async function searchInBrowser({ url, engine = "chromium", headless = true, waitAfterLoadMs = 3000 }) {
|
||||||
const browser = await chromium.launch({ headless });
|
const browser = await launchBrowser(engine, headless);
|
||||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -44,7 +44,12 @@ export async function searchInBrowser({ url, headless = true, waitAfterLoadMs =
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function scrapePage(url, options = {}) {
|
export async function scrapePage(url, options = {}) {
|
||||||
return searchInBrowser({ url, headless: true, waitAfterLoadMs: 750, ...options });
|
return searchInBrowser({ url, engine: "chromium", headless: true, waitAfterLoadMs: 750, ...options });
|
||||||
|
}
|
||||||
|
|
||||||
|
function launchBrowser(engine, headless) {
|
||||||
|
const browserType = String(engine).toLowerCase() === "firefox" ? firefox : chromium;
|
||||||
|
return browserType.launch({ headless });
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function closeVisibleBrowsers() {
|
export async function closeVisibleBrowsers() {
|
||||||
|
|||||||
@@ -261,6 +261,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
|
|||||||
try {
|
try {
|
||||||
const result = await searchInBrowser({
|
const result = await searchInBrowser({
|
||||||
url: attempt.url,
|
url: attempt.url,
|
||||||
|
engine: source.browserEngine,
|
||||||
headless: source.browserHeadless,
|
headless: source.browserHeadless,
|
||||||
waitAfterLoadMs: source.waitAfterLoadMs,
|
waitAfterLoadMs: source.waitAfterLoadMs,
|
||||||
});
|
});
|
||||||
@@ -292,7 +293,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
|
|||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
try {
|
try {
|
||||||
await openInBrowser(attempt.url);
|
await openInBrowser(attempt.url, { engine: source.browserEngine });
|
||||||
} catch {
|
} catch {
|
||||||
// The manual candidate below remains available even if the browser cannot start.
|
// The manual candidate below remains available even if the browser cannot start.
|
||||||
}
|
}
|
||||||
@@ -312,6 +313,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
|
|||||||
lastSearchUrl = attempt.url;
|
lastSearchUrl = attempt.url;
|
||||||
try {
|
try {
|
||||||
const response = await scrapePage(attempt.url, {
|
const response = await scrapePage(attempt.url, {
|
||||||
|
engine: source.browserEngine,
|
||||||
waitAfterLoadMs: source.waitAfterLoadMs,
|
waitAfterLoadMs: source.waitAfterLoadMs,
|
||||||
});
|
});
|
||||||
const html = response.pageSource;
|
const html = response.pageSource;
|
||||||
@@ -858,16 +860,16 @@ async function checkMappingSourceUrls(items) {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
...item,
|
...item,
|
||||||
...(await checkUrl(item.url)),
|
...(await checkUrl(item.url, item.browserEngine)),
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkUrl(url) {
|
async function checkUrl(url, browserEngine = "chromium") {
|
||||||
try {
|
try {
|
||||||
const target = normalizeSourceUrl(url);
|
const target = normalizeSourceUrl(url);
|
||||||
const response = await scrapePage(target, { waitAfterLoadMs: 250 });
|
const response = await scrapePage(target, { engine: browserEngine, waitAfterLoadMs: 250 });
|
||||||
|
|
||||||
return {
|
return {
|
||||||
health: response.ok ? "ok" : "warning",
|
health: response.ok ? "ok" : "warning",
|
||||||
|
|||||||
Reference in New Issue
Block a user