Compare commits

..
70 Commits
Author SHA1 Message Date
rajch_ales 4b8b7718a9 Record project planet lore 2026-08-06 01:50:53 +02:00
rajch_ales 8a911cd44e Record project working names 2026-08-06 01:48:39 +02:00
rajch_ales 6a4930343a Add AI partner avatar 2026-08-06 01:42:31 +02:00
rajch_ales 9e3b4de4e2 Store reusable scraper capabilities in scrape DB 2026-08-06 01:01:55 +02:00
rajch_ales b11ada9c91 Document reliable git push workflow 2026-08-06 00:51:22 +02:00
rajch_ales 057726b543 Accept portrait images for square crop 2026-08-06 00:48:33 +02:00
rajch_ales 82d0a07e1f Stop automatic source browser opens 2026-08-06 00:44:21 +02:00
rajch_ales 6f2d998754 Run source lookup headless and reject Idealo thumbnails 2026-08-06 00:39:50 +02:00
rajch_ales d3d1147a3a Prefer large Idealo gallery pictures 2026-08-06 00:33:44 +02:00
rajch_ales b5a69050e0 Warn on small picture candidates 2026-08-06 00:24:03 +02:00
rajch_ales 0987edd112 Add batch picture candidate workflow 2026-08-06 00:16:25 +02:00
rajch_ales a8e3e62dbd Store selected picture candidates in scrape DB 2026-08-06 00:02:25 +02:00
rajch_ales 05338a062a Prepare scrape image storage schema 2026-08-05 23:47:57 +02:00
rajch_ales aa138b62ad Keep horizontal scrollbar in main workspace 2026-08-05 23:29:05 +02:00
rajch_ales 19c6ee90e1 Persist scrape database write permissions 2026-08-05 23:24:49 +02:00
rajch_ales 67753fb025 Persist local database write permissions 2026-08-05 23:20:49 +02:00
rajch_ales 1a49fa4e95 Fix main workspace collapse arrow direction 2026-08-05 23:17:25 +02:00
rajch_ales f88cadc490 Load product images from Presta image path 2026-08-05 23:14:38 +02:00
rajch_ales 5210dc2042 Move catalog source mapping to scrape database 2026-08-05 23:06:01 +02:00
rajch_ales e3caf25823 Cache mapping status on reload 2026-08-05 22:57:06 +02:00
rajch_ales 9b1898a588 Restore DBGate beside Adminer 2026-08-05 22:42:59 +02:00
rajch_ales 05ae8f03d2 Enable Adminer MariaDB PHP extensions 2026-08-05 22:31:06 +02:00
rajch_ales 1d6837223b Add Adminer login hint 2026-08-05 22:26:37 +02:00
rajch_ales 4777bda0b2 Replace DBGate with local Adminer 2026-08-05 22:21:22 +02:00
rajch_ales 7a45686952 Preconfigure local DBGate connections 2026-08-05 22:08:42 +02:00
rajch_ales ffe70e65ac Add local DBGate server 2026-08-05 21:59:00 +02:00
rajch_ales 2106201f9c Add DBGate opener for scrape database 2026-08-05 21:49:03 +02:00
rajch_ales 10672a6be3 Document local database gate settings 2026-08-05 21:44:08 +02:00
rajch_ales a72bdab9ab Move source mapping to scrape database 2026-08-05 21:40:58 +02:00
rajch_ales 17082f6d9e Use Playwright Firefox runtime 2026-08-05 21:25:44 +02:00
rajch_ales fe329cd19b Open source candidates with configured browser 2026-08-05 21:20:06 +02:00
rajch_ales d90e6c6dd8 Use configured browser for source windows 2026-08-05 21:15:19 +02:00
rajch_ales 6a1568a1dd Honor configured browser engine 2026-08-05 21:11:17 +02:00
rajch_ales 1f5ab5a8cc Add source popup reload action 2026-08-05 19:44:11 +02:00
rajch_ales e172b24474 Respect visible browser setting for source checks 2026-08-05 19:37:03 +02:00
rajch_ales bc96d50ecc Use local MariaDB as active database mode 2026-08-05 19:30:28 +02:00
rajch_ales b9c2e181eb Use configured Playwright browser engines 2026-08-05 19:21:14 +02:00
rajch_ales 8b2a74bcb3 Document authoritative database routing 2026-08-05 19:16:15 +02:00
rajch_ales 1ab09903fb Remove duplicate supplier field from product queries 2026-08-05 19:13:14 +02:00
rajch_ales f08699fad3 Name local database write permission explicitly 2026-08-05 19:07:37 +02:00
rajch_ales 2f3444adb8 Connect local write permissions to backend 2026-08-05 19:05:07 +02:00
rajch_ales e6acb21d65 Persist selected manufacturer across reloads 2026-08-05 18:58:05 +02:00
rajch_ales c58084cc86 Move database status below settings 2026-08-05 18:54:32 +02:00
rajch_ales f97d00ab7f Add database connection status indicator 2026-08-05 18:53:11 +02:00
rajch_ales f2fa405008 Center mapping status indicator with label 2026-08-05 18:44:29 +02:00
rajch_ales 85df4dc4cb Keep mapping chevron centered and downward 2026-08-05 18:42:55 +02:00
rajch_ales 90731319c8 Match mapping chevron and centered label 2026-08-05 18:40:26 +02:00
rajch_ales 0f9764cc49 Center mapping icon with label 2026-08-05 18:36:53 +02:00
rajch_ales 921890fbec Align mapping icon in status control 2026-08-05 18:35:13 +02:00
rajch_ales 657c82b4ba Pin mapping label to center 2026-08-05 18:33:51 +02:00
rajch_ales 071c37cb3a Fix mapping title centering 2026-08-05 18:32:03 +02:00
rajch_ales c440b6fc1b Center mapping control like selectors 2026-08-05 18:28:29 +02:00
rajch_ales 8d5521093e Keep mapping collapsed on reload 2026-08-05 18:26:19 +02:00
rajch_ales 3a707aa0ca Keep menu above database status in footer 2026-08-05 18:25:38 +02:00
rajch_ales 452ee68599 Place database status below menu 2026-08-05 18:24:44 +02:00
rajch_ales 1a8180c1f9 Move mapping below manufacturer selector 2026-08-05 18:22:33 +02:00
rajch_ales 1258d64aef Open mapping details below status control 2026-08-05 18:20:55 +02:00
rajch_ales 9d9789cd5b Add compact mapping health indicator 2026-08-05 18:17:14 +02:00
rajch_ales dab0859aa9 Align mapping status row 2026-08-05 18:09:55 +02:00
rajch_ales 7c7d5d2952 Remove background remover sidebar action 2026-08-05 18:07:19 +02:00
rajch_ales 0269a20187 Remove scraper sidebar action 2026-08-05 18:06:00 +02:00
rajch_ales d5b4f344dd Move mapping beside database status 2026-08-05 18:03:17 +02:00
rajch_ales c94b514647 Place mapping status with parser tools 2026-08-05 17:59:51 +02:00
rajch_ales 588c0bd9cc Keep parser actions always visible 2026-08-05 17:57:39 +02:00
rajch_ales e9aa381f45 Center parser drawer control group 2026-08-05 17:55:24 +02:00
rajch_ales 5882edb994 Audit buttons and selector controls 2026-08-05 17:53:44 +02:00
rajch_ales 310e1e239f Prevent duplicate selector labels 2026-08-05 17:49:58 +02:00
rajch_ales 4e1c3eccab Keep language selector icon visible 2026-08-05 17:47:01 +02:00
rajch_ales f463bd27e7 Align selector icons with centered labels 2026-08-05 17:45:22 +02:00
rajch_ales e68c5d2bc7 Hide native manufacturer control host 2026-08-05 17:42:41 +02:00
29 changed files with 14789 additions and 244 deletions
+14
View File
@@ -0,0 +1,14 @@
DB_DRIVER=mariadb
DB_MODE=local
DB_LOCAL_ALLOW_WRITES=false
DB_HOST=127.0.0.1
DB_PORT=3306
DB_NAME=9bplus
SCRAPE_DB_NAME=catalog_scrape
ADMINER_URL=http://127.0.0.1:8080
ADMINER_BROWSER_ENGINE=chromium
DBGATE_URL=http://127.0.0.1:3000
DBGATE_BROWSER_ENGINE=chromium
DB_USER=catalog_maker
DB_PASSWORD=
DB_CONNECTION_LIMIT=5
+2
View File
@@ -6,6 +6,8 @@ config/local.json
outputs/ outputs/
logs/ logs/
tmp/ tmp/
tools/adminer/
tools/php/
*.log *.log
*.sql *.sql
*.sql.gz *.sql.gz
+78 -3
View File
@@ -4,12 +4,25 @@
This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a Node.js local web application. The UI is a compact product catalog editor for loading manufacturer products, combinations, source mappings and parsed supplier data. This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a Node.js local web application. The UI is a compact product catalog editor for loading manufacturer products, combinations, source mappings and parsed supplier data.
## Working relationship
- The user is Aleš.
- Aleš is from Czechia.
- The specialist programmer who may advise or challenge project decisions is Mareček from Poland.
- In this project, the assistant may be called Mia. Do not use the name Nika.
- Mia's project planet is Mercury: fast communication, technical thinking, messages, trade, and a little useful chaos that moves things forward.
- Aleš may be described playfully as Mars with beer and an SQL dump when the evening gets intense.
- The fictional project avatar is stored at `docs/ai-partner-avatar.png`.
- New avatar images are a light motivational reward after real project progress, not an automatic daily task. Keep them tasteful, non-explicit, and vary hairstyle/outfit when the user asks for a new one.
## Non-negotiable rules ## Non-negotiable rules
- Live DB is never written to. Live DB access is read-only through the existing endpoint. - Live DB is never written to. Live DB access is read-only through the existing endpoint.
- Local DB may later allow controlled writes, but writes must remain explicit, logged and protected by transaction/dry-run settings. - Local DB may later allow controlled writes, but writes must remain explicit, logged and protected by transaction/dry-run settings.
- Never print, commit or share endpoint tokens, usernames or passwords. - Never print, commit or share endpoint tokens, usernames or passwords.
- Never use Live DB as a fallback when Local DB is selected. If Local DB is unavailable, show an error and keep the UI running. - Never use Live DB as a fallback when Local DB is selected. If Local DB is unavailable, show an error and keep the UI running.
- The selected catalog database mode is authoritative for catalog data calls. When `Local 9bplus DB` is selected, manufacturers, languages, products, combinations and all other catalog reads must use the local MariaDB connection; no endpoint or Live DB call is allowed. When `Live 9bplus DB` is selected, catalog reads use the existing endpoint and remain read-only.
- Mapping, source configuration, scraper settings, browser-engine choices and scraped draft data belong to `Local scrape DB` (`catalog_scrape`) and must be read from there, not from Live DB or the product catalog database.
- Keep the development server on port `3404` unless the user explicitly requests another port. - Keep the development server on port `3404` unless the user explicitly requests another port.
- Do not delete or revert user files or unrelated changes. - Do not delete or revert user files or unrelated changes.
@@ -19,7 +32,18 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
- Use short, descriptive commit messages that state the completed change. - Use short, descriptive commit messages that state the completed change.
- Never commit `.env`, `config/local.json`, passwords, tokens, the Excel workbook or database dumps. - Never commit `.env`, `config/local.json`, passwords, tokens, the Excel workbook or database dumps.
- Before the first push, initialize Git if needed, configure the project remote and create an initial baseline commit from safe files only. - Before the first push, initialize Git if needed, configure the project remote and create an initial baseline commit from safe files only.
- If a push fails, do not expose credentials in the terminal output or in chat; report the failure without printing secrets. - Always use Git with the explicit safe directory flag in this workspace:
`git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai ...`
- Do not combine `git add`, `git commit` and `git push` into one shell command. Run them as separate steps and inspect the output after each step.
- Reliable push sequence:
1. `git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai status --short`
2. `git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai add <explicit safe files>`
3. `git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai commit -m "<short message>"`
4. `git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai push`
5. `git -c safe.directory=D:/Codex/01-projekty/catalog-meaker-by-magic-ai status --short`
- If push returns `Failed to authenticate user`, retry the same push command once. If it fails again, stop and report the push failure without printing secrets.
- If there are unrelated modified files, leave them alone. Add only the files that belong to the completed step.
- For documentation-only changes, tests are optional; for code, UI, scraper or database behavior changes, run the relevant build/test command before committing.
## UI inspection workflow ## UI inspection workflow
@@ -51,6 +75,12 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
- Button icons must be placed before the label in DOM order, with the icon and label centered together as one group. Keep the control's chevron or disclosure arrow in its own fixed right-side slot so it does not shift the centered label. - Button icons must be placed before the label in DOM order, with the icon and label centered together as one group. Keep the control's chevron or disclosure arrow in its own fixed right-side slot so it does not shift the centered label.
- Use one shared icon size, gap, alignment, hover state and disabled state for equivalent controls. Do not mix icon-only, text-only and icon-after-text variants for the same control family. - Use one shared icon size, gap, alignment, hover state and disabled state for equivalent controls. Do not mix icon-only, text-only and icon-after-text variants for the same control family.
- Before finishing any UI change, inspect the rendered page with Playwright and verify that icons are visible, precede their labels, remain centered, and do not overlap or drift between controls. Update this file when a new reusable visual rule is discovered. - Before finishing any UI change, inspect the rendered page with Playwright and verify that icons are visible, precede their labels, remain centered, and do not overlap or drift between controls. Update this file when a new reusable visual rule is discovered.
- Custom select controls may use a native select as an interaction layer, but the native label must be visually transparent when a separate display label is rendered; never show the same control text twice.
- Runtime button labels must update the existing direct child label span instead of replacing `button.textContent`; state changes must never remove the button icon.
- Parser tools are always visible in the left panel. Do not hide, collapse or replace the parser action buttons with a toggle; the user must see the complete parser action set at all times.
- Mapping status and mapping source details belong beside the database connection status at the bottom of the sidebar. Style it as a shared sidebar section header with icon, centered status and a fixed right chevron; keep the existing mapping behavior.
- Wide working data areas, especially the main workspace combinations grid, must keep a visible bottom horizontal scrollbar. The workspace should use a fixed header plus a scrollable content area so users can move sideways through all columns without losing the panel layout.
- Add or update Playwright checks for wide-data layout changes by asserting that the scroll container has horizontal overflow (`scrollWidth > clientWidth`) and `overflow-x: auto`.
## TypeScript workflow ## TypeScript workflow
@@ -64,6 +94,11 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
- Use Playwright for all supplier/manufacturer website scraping and browser page inspection. - Use Playwright for all supplier/manufacturer website scraping and browser page inspection.
- Do not implement new scraping with raw `fetch`, regex-only HTML parsing, Selenium or direct HTTP shortcuts. - Do not implement new scraping with raw `fetch`, regex-only HTML parsing, Selenium or direct HTTP shortcuts.
- Scrapers must use controlled browser navigation, selectors, explicit waits and bounded request counts. - Scrapers must use controlled browser navigation, selectors, explicit waits and bounded request counts.
- The configured browser engine for a source is authoritative. If a source says `firefox`, launch Firefox or fail loudly; do not silently fall back to Chromium, Edge or another browser.
- Automated source lookup, mapping checks and picture lookup must run headless/in the background by default. Open a visible browser only for explicit user `Open` actions or deliberate diagnostics after an error.
- `Find sources` must never automatically open found source candidates in a visible browser. It only fills the source popup; visible browser opening belongs only to the per-row `Open` action or an explicit diagnostic path.
- Visible source windows, URL health checks, source lookup and picture parsing must all use the same configured browser engine from that source's database setting.
- Source candidate `Open` actions in the UI must call the backend source opener with `manufacturerId`, `sourceKey` and URL. Do not use plain `target="_blank"` source links for mapped supplier/manufacturer sources because that ignores the configured browser engine.
- Keep scraping slow and observable, preserve source URL/status information, and handle blocked pages or missing selectors as explicit errors. - Keep scraping slow and observable, preserve source URL/status information, and handle blocked pages or missing selectors as explicit errors.
- Supplier data remains a local draft until the user explicitly confirms an Apply / Save action. - Supplier data remains a local draft until the user explicitly confirms an Apply / Save action.
@@ -73,7 +108,7 @@ This project replaces the Excel/VBA workflow named `Catalog maker - 20` with a N
- Open `http://127.0.0.1:3404/`. - Open `http://127.0.0.1:3404/`.
- The current server is `src/server.ts`, run through `tsx`. - The current server is `src/server.ts`, run through `tsx`.
- Configuration is loaded from `config/local.json`, merged over `config/local.example.json`. - Configuration is loaded from `config/local.json`, merged over `config/local.example.json`.
- `.env` exists for local MariaDB notes, but the application does not yet load `.env` automatically. Do not assume `.env` is active until configuration loading is explicitly wired to it. - `.env` is loaded automatically by `src/config.ts` for local MariaDB settings. Never expose its values. The explicit local write capability flag is `DB_LOCAL_ALLOW_WRITES`; it never enables Live DB writes.
## Database architecture ## Database architecture
@@ -94,9 +129,19 @@ There are three selectable data sources:
### Local scrape DB ### Local scrape DB
- Uses the separate local MariaDB database named `catalog_scrape`. - Uses the separate local MariaDB database named `catalog_scrape`.
- Stores scraper source configuration, scrape runs, discovered products and assets. - Stores scraper source configuration, manufacturer-to-source mapping, scrape runs, discovered products and assets.
- Supplier/manufacturer website settings such as source URL, search templates, fallback mode, browser engine, headless mode, wait time, notes and enabled/priority flags belong in Local scrape DB, not in the product catalog database.
- Source extraction rules are stored as reusable capabilities in Local scrape DB: `search`, `pictures`, `texts` and `features`. The code is only the generic engine; source-specific selectors, parser keys, minimum picture quality, browser engine choices and target strategies must live in DB so the same Hudy/Idealo rules can be reused for other manufacturers.
- Manufacturer-to-source mapping may enable or override individual capabilities, for example Hudy `pictures/texts/features` and Idealo `pictures`. Do not duplicate the same source configuration per manufacturer when a reusable source capability already exists.
- On first manufacturer selection, mapping status may be loaded quietly from Local scrape DB in the background without opening browsers or running Playwright URL checks. Page reloads should reuse the cached mapping status instead of doing the mapping check again. Manual manufacturer changes, including switching away and back to the same manufacturer, must load mapping from DB again. Browser/search work starts only after explicit user actions such as `Find sources`, `Get pictures`, or opening a source.
- The Settings modal should expose an `Open Adminer` action beside `Local scrape DB`; it opens the local Adminer URL from `ADMINER_URL` / `databaseTool.url` for inspecting local MariaDB databases.
- The Settings modal may also expose `Open DBGate` beside Adminer for a richer local database browser. DBGate should stay local, start with `npm run dbgate`, and use `DBGATE_URL` / `dbGate.url`.
- `npm run dbgate` must start local-only DBGate without auth prompts and with two predefined local MariaDB connections from `.env`: `Local 9bplus DB` (`DB_NAME`) and `Local scrape DB` (`SCRAPE_DB_NAME`). Do not hardcode or commit passwords; pass them through environment variables at process start.
- Start Adminer only through `npm run adminer`; the script downloads the ignored local `tools/adminer/adminer.php` file when missing and runs it on the configured local Adminer URL. Adminer requires local PHP CLI.
- Adminer login help may show non-secret local connection fields such as system, host, port, database and user, but must never render or print the actual password. Show password source as `.env DB_PASSWORD` instead.
- It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added. - It must never be used as a fallback for catalog reads. When selected in the UI, existing catalog reads continue using Local 9bplus DB until dedicated scrape-storage endpoints are added.
- The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`. - The schema is versioned in `sql/local-scrape-db.sql` and can be created with `npm run db:create-scrape`.
- Existing legacy mapping rows from `Local 9bplus DB` can be copied into Local scrape DB with `npm run db:migrate-mapping-to-scrape`.
### Local MariaDB connection ### Local MariaDB connection
@@ -113,8 +158,10 @@ There are three selectable data sources:
- The UI has `Local 9bplus DB`, `Live 9bplus DB` and `Local scrape DB` settings. - The UI has `Local 9bplus DB`, `Live 9bplus DB` and `Local scrape DB` settings.
- The UI sends the selected mode using the `X-Database-Mode` request header. - The UI sends the selected mode using the `X-Database-Mode` request header.
- The backend must resolve that header before every API database operation and route the complete request through the selected source. A Local DB request must use `queryMariaDb`; a Live DB request must use `queryEndpoint`; there is no silent fallback or mixed-source response.
- Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint. - Backend data API routes reject a Local DB request when MariaDB is not configured instead of silently using the endpoint.
- Backend data API routes reject a Live DB request when the project is configured only for MariaDB. - Backend data API routes reject a Live DB request when the project is configured only for MariaDB.
- Product endpoints read catalog data from the selected catalog database. Mapping endpoints read scraper source mapping from Local scrape DB while still using the selected catalog database only for catalog/product data.
- Static HTML/CSS/JS files must remain available even when a database is not configured. - Static HTML/CSS/JS files must remain available even when a database is not configured.
- The top status shows the selected source, connection state, permissions and execution mode. - The top status shows the selected source, connection state, permissions and execution mode.
- Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI. - Selecting Live 9bplus DB or Local scrape DB disables catalog write permissions in the UI.
@@ -140,11 +187,39 @@ There are three selectable data sources:
- `Get product info` remains a separate action. - `Get product info` remains a separate action.
- EAN scan input searches on Enter. - EAN scan input searches on Enter.
- `Find sources` searches once and caches candidates for the current product; a second click reopens cached results. - `Find sources` searches once and caches candidates for the current product; a second click reopens cached results.
- The source-candidates popup must provide a `Reload` action beside `Close`; `Reload` bypasses only the current product's cached source result and re-runs the source lookup using the current local mapping configuration. It must not switch database modes or write to Live DB.
- Source candidates show how they were found, including EAN/reference/name and color context. - Source candidates show how they were found, including EAN/reference/name and color context.
- Each source row has its own `Get pictures` action. - Each source row has its own `Get pictures` action.
- Picture candidates must show source, image dimensions and the proposed target assignment. The target assignment is not always color: supported strategies are `color`, `size`, `combination` and `manual`. Start by using manual corrections from the user, then learn reusable rules in Local scrape DB.
- Scraped pictures and their metadata belong in Local scrape DB, not browser localStorage. Store the binary image in `scrape_image_blobs` with `sha256`, `perceptual_hash`, mime type, dimensions and size. Store each source/target proposal in `scrape_picture_candidates`.
- Every picture candidate row must have its own save action. Saving one candidate stores only that selected image candidate into Local scrape DB and never saves the whole source result list.
- Picture candidate popups must also provide `Save all` for robot/automatic flows. `Save all` saves every currently visible picture candidate one by one and must not save hidden, stale or unrelated candidates.
- Source candidate popups should provide `Get all pictures` for robot/automatic flows. It fetches picture candidates from every currently visible mapped source, then opens the picture candidate popup. Saving happens only in the picture candidate popup.
- Use `sha256` to remove exact duplicate image files and a perceptual hash to flag visually similar images from different sources or sizes. Exact duplicates may be collapsed automatically; visual duplicates should remain reviewable until the user approves the rule.
- Product source cache is cleared when a new product loads. - Product source cache is cleared when a new product loads.
### Picture Scraping Agent
- Treat picture scraping as a separate internal role: find sources, fetch picture candidates, classify target assignment, detect duplicates and prepare save actions.
- The picture scraping agent stores all scraper settings, candidates, hashes, metadata and learned target rules in Local scrape DB (`catalog_scrape`).
- Picture files are stored as binary blobs in Local scrape DB with disk-friendly metadata, not in browser localStorage and not in the live catalog database.
- The agent starts with user-guided target assignment (`color`, `size`, `combination`, `manual`) and turns repeated corrections into reusable Local scrape DB rules.
- The agent may compare Hudy, Idealo and future sources, but it must keep all candidates reviewable until an exact duplicate or learned rule is trusted.
- The agent never writes to Live 9bplus DB. Live data may only be read later for comparison when explicitly allowed.
#### Picture Scraping Skills
- Source discovery: use manufacturer mapping from Local scrape DB and show exactly which source produced each candidate.
- Browser scraping: use the configured browser engine per source and keep the UI responsive while scraping runs.
- Image extraction: collect original image URLs, dimensions, mime type, file size and source page URL.
- Image quality: product pictures will later be normalized to a `1:1` square, so a portrait/landscape image may be production-ready even when one side is below `800px` (for example `739 x 1435`). Mark images as too small only when the longest side is below `800px`; otherwise show the real size and note when it is OK for `1:1` crop.
- Duplicate detection: calculate `sha256` for exact duplicates and perceptual hash for visually similar images.
- Target assignment: propose whether pictures belong to `color`, `size`, `combination` or `manual` target, then learn from user corrections.
- Persistence: save picture blobs, candidates, hashes, source metadata and learned rules only into Local scrape DB.
- Review UI: show picture size, source, target assignment, duplicate state, per-row save and batch save actions.
- Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images. - Hudy picture parsing uses the correct product color variant, checks EANs and extracts large gallery images.
- Idealo uses the browser adapter and is intended mainly as a picture source. - Idealo uses the browser adapter and is intended mainly as a picture source.
- Idealo picture parsing must read product gallery images from Splide gallery markup (`splide__track`, `splide__list`, `splide__slide`) and choose the largest available URL from `srcset`, `data-srcset`, `data-large`, `data-original`, `data-src` or `src`. Do not use search-result thumbnails or small preview URLs as production picture candidates. If no Idealo gallery is found, return no Idealo pictures and show the quality/missing warning instead of saving thumbnails.
- Image preview is square. - Image preview is square.
- Settings is at the bottom of the left panel; database status is directly above it. - Settings is at the bottom of the left panel; database status is directly above it.
+5 -1
View File
@@ -16,7 +16,7 @@
"database": { "database": {
"driver": "endpoint", "driver": "endpoint",
"mode": "local", "mode": "local",
"allowWrites": false, "allowLocalWrites": false,
"host": "127.0.0.1", "host": "127.0.0.1",
"port": 3306, "port": 3306,
"name": "9bplus", "name": "9bplus",
@@ -24,5 +24,9 @@
"user": "catalog_maker", "user": "catalog_maker",
"password": "", "password": "",
"connectionLimit": 5 "connectionLimit": 5
},
"databaseTool": {
"url": "http://127.0.0.1:8080",
"browserEngine": "chromium"
} }
} }
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 MiB

+11637
View File
File diff suppressed because it is too large Load Diff
+4
View File
@@ -9,6 +9,9 @@
"dev": "tsx src/server.ts", "dev": "tsx src/server.ts",
"catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run", "catalog:dry-run": "tsx src/cli.ts catalog-maker --dry-run",
"db:create-scrape": "tsx scripts/create-local-scrape-db.ts", "db:create-scrape": "tsx scripts/create-local-scrape-db.ts",
"db:migrate-mapping-to-scrape": "tsx scripts/migrate-mapping-to-scrape-db.ts",
"adminer": "tsx scripts/start-adminer.ts",
"dbgate": "tsx scripts/start-dbgate.ts",
"test": "tsx --test test/*.test.ts", "test": "tsx --test test/*.test.ts",
"test:ui": "playwright test", "test:ui": "playwright test",
"test:e2e": "playwright test tests/ui/database.e2e.spec.ts", "test:e2e": "playwright test tests/ui/database.e2e.spec.ts",
@@ -27,6 +30,7 @@
"devDependencies": { "devDependencies": {
"@tailwindcss/cli": "^4.3.3", "@tailwindcss/cli": "^4.3.3",
"@types/node": "^26.1.2", "@types/node": "^26.1.2",
"dbgate-serve": "^7.2.3",
"tailwindcss": "^4.3.3", "tailwindcss": "^4.3.3",
"tsx": "^4.23.7", "tsx": "^4.23.7",
"typescript": "^7.0.2" "typescript": "^7.0.2"
+466 -55
View File
@@ -8,6 +8,7 @@ const manufacturerSearch = document.querySelector("#manufacturerSearch");
const manufacturerOptions = document.querySelector("#manufacturerOptions"); const manufacturerOptions = document.querySelector("#manufacturerOptions");
const manufacturerValue = document.querySelector("#manufacturerValue"); const manufacturerValue = document.querySelector("#manufacturerValue");
const languageSelect = document.querySelector("#languageSelect"); const languageSelect = document.querySelector("#languageSelect");
const languageValue = document.querySelector("#languageValue");
const recordIdElement = document.querySelector("#recordId"); const recordIdElement = document.querySelector("#recordId");
const loadProductButton = document.querySelector("#loadProductButton"); const loadProductButton = document.querySelector("#loadProductButton");
const previousProductButton = document.querySelector("#previousProductButton"); const previousProductButton = document.querySelector("#previousProductButton");
@@ -21,12 +22,11 @@ const suggestedTitle = document.querySelector("#suggestedTitle");
const suggestedList = document.querySelector("#suggestedList"); const suggestedList = document.querySelector("#suggestedList");
const suggestedCloseButton = document.querySelector("#suggestedCloseButton"); const suggestedCloseButton = document.querySelector("#suggestedCloseButton");
const mappingState = document.querySelector("#mappingState"); const mappingState = document.querySelector("#mappingState");
const mappingStatusText = document.querySelector("#mappingStatusText");
const mappingList = document.querySelector("#mappingList"); const mappingList = document.querySelector("#mappingList");
const mappingDetails = document.querySelector("#mappingDetails"); const mappingDetails = document.querySelector("#mappingDetails");
const findSourcesButton = document.querySelector("#findSourcesButton"); const findSourcesButton = document.querySelector("#findSourcesButton");
const getPicturesButton = document.querySelector("#getPicturesButton"); const getPicturesButton = document.querySelector("#getPicturesButton");
const parserDrawerToggle = document.querySelector("#parserDrawerToggle");
const parserDrawer = document.querySelector("#parserDrawer");
const sidebarDrawer = document.querySelector("#sidebarDrawer"); const sidebarDrawer = document.querySelector("#sidebarDrawer");
const sidebarOpenButton = document.querySelector("#sidebarOpenButton"); const sidebarOpenButton = document.querySelector("#sidebarOpenButton");
const sidebarCloseButton = document.querySelector("#sidebarCloseButton"); const sidebarCloseButton = document.querySelector("#sidebarCloseButton");
@@ -34,6 +34,8 @@ const sidebarOverlay = document.querySelector("#sidebarOverlay");
const sourceModal = document.querySelector("#sourceModal"); const sourceModal = document.querySelector("#sourceModal");
const sourceTitle = document.querySelector("#sourceTitle"); const sourceTitle = document.querySelector("#sourceTitle");
const sourceList = document.querySelector("#sourceList"); const sourceList = document.querySelector("#sourceList");
const sourceBatchActionButton = document.querySelector("#sourceBatchActionButton");
const sourceReloadButton = document.querySelector("#sourceReloadButton");
const sourceCloseButton = document.querySelector("#sourceCloseButton"); const sourceCloseButton = document.querySelector("#sourceCloseButton");
const settingsButton = document.querySelector("#settingsButton"); const settingsButton = document.querySelector("#settingsButton");
const themeToggle = document.querySelector("#themeToggle"); const themeToggle = document.querySelector("#themeToggle");
@@ -44,16 +46,31 @@ const databaseModeInputs = document.querySelectorAll('input[name="databaseMode"]
const permissionInputs = document.querySelectorAll("#allowInsertToggle, #allowUpdateToggle, #allowDeleteToggle"); const permissionInputs = document.querySelectorAll("#allowInsertToggle, #allowUpdateToggle, #allowDeleteToggle");
const transactionModeInputs = document.querySelectorAll('input[name="transactionMode"]'); const transactionModeInputs = document.querySelectorAll('input[name="transactionMode"]');
const databaseStatus = document.querySelector("#databaseStatus"); const databaseStatus = document.querySelector("#databaseStatus");
const databaseStateDot = document.querySelector("#databaseStateDot");
const localDbFields = document.querySelector("#localDbFields"); const localDbFields = document.querySelector("#localDbFields");
const localDbInputs = document.querySelectorAll("#localDbFields input"); const localDbInputs = document.querySelectorAll("#localDbFields input");
const testLocalDbButton = document.querySelector("#testLocalDbButton"); const testLocalDbButton = document.querySelector("#testLocalDbButton");
const localDbTestResult = document.querySelector("#localDbTestResult"); const localDbTestResult = document.querySelector("#localDbTestResult");
const openAdminerButton = document.querySelector("#openAdminerButton");
const openDbGateButton = document.querySelector("#openDbGateButton");
const adminerServerValue = document.querySelector("#adminerServerValue");
const adminerUserValue = document.querySelector("#adminerUserValue");
const adminerDatabaseValue = document.querySelector("#adminerDatabaseValue");
const lightContrastSelect = document.querySelector("#lightContrastSelect"); const lightContrastSelect = document.querySelector("#lightContrastSelect");
const darkContrastSelect = document.querySelector("#darkContrastSelect"); const darkContrastSelect = document.querySelector("#darkContrastSelect");
const catalogScreen = document.querySelector(".catalog-screen"); const catalogScreen = document.querySelector(".catalog-screen");
const secondaryPanelToggle = document.querySelector("#secondaryPanelToggle"); const secondaryPanelToggle = document.querySelector("#secondaryPanelToggle");
const mainWorkspaceToggle = document.querySelector("#mainWorkspaceToggle"); const mainWorkspaceToggle = document.querySelector("#mainWorkspaceToggle");
const MIN_PRODUCT_PICTURE_SIZE = 800;
let sidebarUserClosed = false; let sidebarUserClosed = false;
let mappingLoadToken = 0;
const mappingCacheKey = "catalog-maker:mapping-cache";
const legacyPermissionStorageKey = "catalog-maker:db-permissions";
const permissionStorageKeys = {
local: "catalog-maker:db-permissions:local-9bplus",
live: "catalog-maker:db-permissions:live-9bplus",
scrape: "catalog-maker:db-permissions:local-scrape",
};
const nativeFetch = window.fetch.bind(window); const nativeFetch = window.fetch.bind(window);
window.fetch = (input, init = {}) => { window.fetch = (input, init = {}) => {
const url = typeof input === "string" ? input : input.url; const url = typeof input === "string" ? input : input.url;
@@ -61,9 +78,16 @@ window.fetch = (input, init = {}) => {
return nativeFetch(input, init); return nativeFetch(input, init);
const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined)); const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local"; const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
const permissions = readDatabasePermissions(selectedMode);
const localWritesEnabled = ["local", "scrape"].includes(selectedMode) && [
permissions.allowInsertToggle,
permissions.allowUpdateToggle,
permissions.allowDeleteToggle,
].some(Boolean);
// Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus. // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local"); headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0"); headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
headers.set("X-Local-Write-Access", localWritesEnabled ? "1" : "0");
return nativeFetch(input, { ...init, headers }); return nativeFetch(input, { ...init, headers });
}; };
let currentPosition = 1; let currentPosition = 1;
@@ -73,7 +97,8 @@ let currentCombinations = [];
let loadRequestId = 0; let loadRequestId = 0;
let applicationMode = "dry-run"; let applicationMode = "dry-run";
let databaseConnectionState = "unknown"; let databaseConnectionState = "unknown";
const localDraftKey = "catalog-maker:current-product-draft:v2"; const localDraftKey = "catalog-maker:current-product-draft:v3";
const selectedManufacturerKey = "catalog-maker:selected-manufacturer";
let localDraftProductKey = ""; let localDraftProductKey = "";
const fieldNames = [ const fieldNames = [
"id_manufacturer", "id_manufacturer",
@@ -143,11 +168,18 @@ renderTableHeader();
setTableWidth(); setTableWidth();
await loadManufacturers(); await loadManufacturers();
await loadLanguages(); await loadLanguages();
languageSelect.addEventListener("change", syncLanguagePickerLabel);
manufacturerSelect.addEventListener("change", () => { manufacturerSelect.addEventListener("change", () => {
if (manufacturerSelect.value) {
localStorage.setItem(selectedManufacturerKey, manufacturerSelect.value);
}
else {
localStorage.removeItem(selectedManufacturerKey);
}
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
closeManufacturerPicker(); closeManufacturerPicker();
if (manufacturerSelect.value) { if (manufacturerSelect.value) {
loadMappingSources(); loadMappingSources({ checkUrls: false, preferCache: false });
loadProductForSelectedManufacturer(1); loadProductForSelectedManufacturer(1);
} }
else { else {
@@ -169,8 +201,9 @@ loadProductButton.disabled = true;
loadProductButton.title = "Load existing product attribute info."; loadProductButton.title = "Load existing product attribute info.";
loadProductButton.addEventListener("click", loadProductInfoForCurrentProduct); loadProductButton.addEventListener("click", loadProductInfoForCurrentProduct);
findSourcesButton.addEventListener("click", findSourcesForCurrentProduct); findSourcesButton.addEventListener("click", findSourcesForCurrentProduct);
sourceReloadButton.addEventListener("click", () => findSourcesForCurrentProduct(true));
sourceBatchActionButton.addEventListener("click", handleSourceBatchAction);
getPicturesButton.addEventListener("click", getPicturesForCurrentProduct); getPicturesButton.addEventListener("click", getPicturesForCurrentProduct);
parserDrawerToggle.addEventListener("click", toggleParserDrawer);
secondaryPanelToggle.addEventListener("click", toggleSecondaryPanel); secondaryPanelToggle.addEventListener("click", toggleSecondaryPanel);
mainWorkspaceToggle.addEventListener("click", toggleMainWorkspace); mainWorkspaceToggle.addEventListener("click", toggleMainWorkspace);
sidebarOpenButton.addEventListener("click", openSidebar); sidebarOpenButton.addEventListener("click", openSidebar);
@@ -222,13 +255,15 @@ for (const input of transactionModeInputs)
for (const input of localDbInputs) for (const input of localDbInputs)
input.addEventListener("change", saveLocalDbSettings); input.addEventListener("change", saveLocalDbSettings);
testLocalDbButton.addEventListener("click", testLocalDbConnection); testLocalDbButton.addEventListener("click", testLocalDbConnection);
openAdminerButton.addEventListener("click", openAdminer);
openDbGateButton.addEventListener("click", openDbGate);
lightContrastSelect.addEventListener("change", saveAppearance); lightContrastSelect.addEventListener("change", saveAppearance);
darkContrastSelect.addEventListener("change", saveAppearance); darkContrastSelect.addEventListener("change", saveAppearance);
loadDatabaseMode(); loadDatabaseMode();
syncSidebarForViewport(); syncSidebarForViewport();
setTimeout(() => { setTimeout(() => {
if (manufacturerSelect.value) { if (manufacturerSelect.value) {
loadMappingSources(); loadMappingSources({ checkUrls: false, preferCache: true });
loadProductForSelectedManufacturer(1); loadProductForSelectedManufacturer(1);
} }
else { else {
@@ -288,15 +323,29 @@ async function loadManufacturers() {
} }
if (data.message) if (data.message)
manufacturerSelect.append(createOption("", data.message)); manufacturerSelect.append(createOption("", data.message));
restoreSelectedManufacturer();
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
renderManufacturerOptions(); renderManufacturerOptions();
} }
catch (error) { catch (error) {
manufacturerSelect.append(createOption("", error.message || "Local DB is not configured.")); manufacturerSelect.append(createOption("", error.message || "Local DB is not configured."));
restoreSelectedManufacturer();
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
renderManufacturerOptions(); renderManufacturerOptions();
} }
} }
function restoreSelectedManufacturer() {
const savedManufacturerId = localStorage.getItem(selectedManufacturerKey);
if (!savedManufacturerId)
return;
const savedOption = [...manufacturerSelect.options].find((option) => option.value === savedManufacturerId);
if (savedOption) {
manufacturerSelect.value = savedManufacturerId;
}
else {
localStorage.removeItem(selectedManufacturerKey);
}
}
function toggleManufacturerPicker() { function toggleManufacturerPicker() {
if (manufacturerMenu.hidden) { if (manufacturerMenu.hidden) {
manufacturerMenu.hidden = false; manufacturerMenu.hidden = false;
@@ -349,6 +398,32 @@ function createOption(value, label) {
option.textContent = label; option.textContent = label;
return option; return option;
} }
function setButtonLabel(button, label) {
const labelElement = button.querySelector(":scope > span");
if (labelElement) {
labelElement.textContent = label;
}
else {
button.textContent = label;
}
}
function setButtonIcon(button, iconName) {
const useElement = button.querySelector(":scope > svg use");
if (useElement)
useElement.setAttribute("href", `/icons.svg#${iconName}`);
}
function setSourceBatchAction(mode) {
sourceBatchActionButton.dataset.mode = mode;
sourceBatchActionButton.hidden = false;
sourceBatchActionButton.disabled = false;
if (mode === "get-pictures") {
setButtonIcon(sourceBatchActionButton, "image");
setButtonLabel(sourceBatchActionButton, "Get all pictures");
return;
}
setButtonIcon(sourceBatchActionButton, "save");
setButtonLabel(sourceBatchActionButton, "Save all");
}
async function loadLanguages() { async function loadLanguages() {
languageSelect.replaceChildren(createOption("", "Select language")); languageSelect.replaceChildren(createOption("", "Select language"));
try { try {
@@ -366,26 +441,44 @@ async function loadLanguages() {
catch (error) { catch (error) {
languageSelect.append(createOption("", error.message || "Local DB is not configured.")); languageSelect.append(createOption("", error.message || "Local DB is not configured."));
} }
syncLanguagePickerLabel();
} }
async function loadMappingSources() { function syncLanguagePickerLabel() {
const option = languageSelect.selectedOptions[0];
languageValue.textContent = option?.textContent || "Select language";
}
async function loadMappingSources({ checkUrls = false, preferCache = false } = {}) {
const manufacturerId = manufacturerSelect.value; const manufacturerId = manufacturerSelect.value;
const loadToken = ++mappingLoadToken;
if (!manufacturerId) { if (!manufacturerId) {
renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." }); renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." });
return; return;
} }
mappingState.textContent = "Loading mapping..."; const cached = preferCache ? readMappingCache(manufacturerId) : null;
mappingState.className = "mapping-state"; if (cached) {
mappingList.replaceChildren(); renderMappingSources(cached);
return;
}
setMappingStatus("Loading mapping...", "is-loading");
try { try {
const params = new URLSearchParams({ manufacturerId, check: "1" }); const params = new URLSearchParams({ manufacturerId });
if (checkUrls)
params.set("check", "1");
const response = await fetch(`/api/catalog-maker/mapping-sources?${params.toString()}`); const response = await fetch(`/api/catalog-maker/mapping-sources?${params.toString()}`);
const data = await response.json(); const data = await response.json();
if (loadToken !== mappingLoadToken || manufacturerSelect.value !== manufacturerId)
return;
if (!response.ok) { if (!response.ok) {
throw new Error(data.message || "Mapping load failed."); throw new Error(data.message || "Mapping load failed.");
} }
renderMappingSources(data); renderMappingSources(data);
if (data.mapped && (data.items || []).some((item) => item.enabled)) {
saveMappingCache(manufacturerId, data);
}
} }
catch (error) { catch (error) {
if (loadToken !== mappingLoadToken || manufacturerSelect.value !== manufacturerId)
return;
renderMappingSources({ renderMappingSources({
mapped: false, mapped: false,
items: [], items: [],
@@ -393,24 +486,47 @@ async function loadMappingSources() {
}); });
} }
} }
function readMappingCache(manufacturerId) {
try {
const cache = JSON.parse(localStorage.getItem(mappingCacheKey) || "{}");
return cache[String(manufacturerId)] || null;
}
catch {
return null;
}
}
function saveMappingCache(manufacturerId, data) {
try {
const cache = JSON.parse(localStorage.getItem(mappingCacheKey) || "{}");
cache[String(manufacturerId)] = {
...data,
cachedAt: new Date().toISOString(),
};
localStorage.setItem(mappingCacheKey, JSON.stringify(cache));
}
catch {
// Mapping cache is only a UI convenience. If storage fails, normal loading still works.
}
}
function renderMappingSources(data) { function renderMappingSources(data) {
const items = data.items || []; const items = data.items || [];
if (!data.mapped || !items.length) { if (!data.mapped || !items.length) {
mappingState.textContent = data.message || "Mapping missing"; setMappingStatus(data.message || "Mapping missing", "is-missing");
mappingState.className = "mapping-state is-missing";
mappingList.replaceChildren(); mappingList.replaceChildren();
mappingDetails.open = true; mappingDetails.open = false;
return; return;
} }
const enabledCount = items.filter((item) => item.enabled).length; const enabledCount = items.filter((item) => item.enabled).length;
const failedCount = items.filter((item) => item.enabled && item.health && item.health !== "ok").length; const failedCount = items.filter((item) => item.enabled && item.health && item.health !== "ok").length;
const testedCount = items.filter((item) => item.enabled && item.health === "ok").length; const testedCount = items.filter((item) => item.enabled && item.health === "ok").length;
const checkedCount = items.filter((item) => item.enabled && item.health).length;
const ready = enabledCount > 0 && failedCount === 0; const ready = enabledCount > 0 && failedCount === 0;
mappingState.textContent = ready const statusText = ready
? `Sources OK ✓ (${testedCount}/${enabledCount})` ? `Sources OK ✓ (${testedCount}/${enabledCount})`
: data.message || `Sources problem (${testedCount}/${enabledCount})`; : data.message || `Sources problem (${testedCount}/${enabledCount})`;
mappingState.className = ready ? "mapping-state is-ready" : "mapping-state is-missing"; const displayStatusText = ready && !checkedCount ? `Mapping OK (${enabledCount})` : statusText;
mappingDetails.open = !ready || failedCount > 0; setMappingStatus(displayStatusText, ready ? "is-ready" : testedCount > 0 ? "is-warning" : "is-missing");
mappingDetails.open = false;
mappingList.replaceChildren(...items.map((item) => { mappingList.replaceChildren(...items.map((item) => {
const row = document.createElement("div"); const row = document.createElement("div");
row.className = `mapping-source-row${item.enabled ? "" : " is-disabled"}`; row.className = `mapping-source-row${item.enabled ? "" : " is-disabled"}`;
@@ -424,9 +540,17 @@ function renderMappingSources(data) {
return row; return row;
})); }));
} }
function setMappingStatus(text, stateClass) {
mappingState.textContent = "";
mappingState.className = `mapping-state-dot ${stateClass}`;
mappingState.setAttribute("aria-label", text);
mappingStatusText.textContent = text;
}
function getMappingHealthText(item) { function getMappingHealthText(item) {
if (!item.enabled) if (!item.enabled)
return "off"; return "off";
if (!item.health)
return "ready";
if (item.health === "ok") if (item.health === "ok")
return `${item.httpStatus || 200} OK`; return `${item.httpStatus || 200} OK`;
if (item.health === "missing-url") if (item.health === "missing-url")
@@ -440,6 +564,8 @@ function getMappingHealthText(item) {
function getMappingHealthClass(item) { function getMappingHealthClass(item) {
if (!item.enabled) if (!item.enabled)
return "is-muted"; return "is-muted";
if (!item.health)
return "is-ok";
if (item.health === "ok") if (item.health === "ok")
return "is-ok"; return "is-ok";
return "is-bad"; return "is-bad";
@@ -599,13 +725,13 @@ function closeSuggestedProducts() {
suggestedModal.hidden = true; suggestedModal.hidden = true;
suggestedList.replaceChildren(); suggestedList.replaceChildren();
} }
async function findSourcesForCurrentProduct() { async function findSourcesForCurrentProduct(forceRefresh = false) {
if (!manufacturerSelect.value || !currentProduct) { if (!manufacturerSelect.value || !currentProduct) {
setTableMessage("Select manufacturer and product first."); setTableMessage("Select manufacturer and product first.");
return; return;
} }
const savedDraft = readLocalDraft(); const savedDraft = readLocalDraft();
if (savedDraft?.sources) { if (savedDraft?.sources && !forceRefresh) {
renderSourceCandidates(savedDraft.sources.items || []); renderSourceCandidates(savedDraft.sources.items || []);
updateFindSourcesButtonState(); updateFindSourcesButtonState();
sourceTitle.textContent = "Source candidates (cached)"; sourceTitle.textContent = "Source candidates (cached)";
@@ -614,7 +740,8 @@ async function findSourcesForCurrentProduct() {
} }
const supplierColorGroup = getFirstSupplierColorGroup(); const supplierColorGroup = getFirstSupplierColorGroup();
findSourcesButton.disabled = true; findSourcesButton.disabled = true;
findSourcesButton.textContent = "Finding sources..."; sourceReloadButton.disabled = true;
setButtonLabel(findSourcesButton, "Finding sources...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
manufacturerId: manufacturerSelect.value, manufacturerId: manufacturerSelect.value,
@@ -642,6 +769,7 @@ async function findSourcesForCurrentProduct() {
setTableMessage(error.message || "Source search failed."); setTableMessage(error.message || "Source search failed.");
} }
finally { finally {
sourceReloadButton.disabled = false;
updateFindSourcesButtonState(); updateFindSourcesButtonState();
} }
} }
@@ -660,7 +788,7 @@ async function getPicturesForCurrentProduct(sourceKey = "") {
} }
const supplierColorGroup = getFirstSupplierColorGroup(); const supplierColorGroup = getFirstSupplierColorGroup();
getPicturesButton.disabled = true; getPicturesButton.disabled = true;
getPicturesButton.textContent = "Loading pictures..."; setButtonLabel(getPicturesButton, "Loading pictures...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
manufacturerId: manufacturerSelect.value, manufacturerId: manufacturerSelect.value,
@@ -688,7 +816,7 @@ async function getPicturesForCurrentProduct(sourceKey = "") {
} }
finally { finally {
getPicturesButton.disabled = false; getPicturesButton.disabled = false;
getPicturesButton.textContent = "Get pictures"; setButtonLabel(getPicturesButton, "Get pictures");
updateNavigationState(); updateNavigationState();
} }
} }
@@ -713,6 +841,7 @@ function getFirstSupplierColorGroup() {
return [...groups.values()].find((group) => group.supplierRows.length > 0) || null; return [...groups.values()].find((group) => group.supplierRows.length > 0) || null;
} }
function renderSourceCandidates(items) { function renderSourceCandidates(items) {
sourceBatchActionButton.hidden = true;
if (!items.length) { if (!items.length) {
const empty = document.createElement("div"); const empty = document.createElement("div");
empty.className = "source-row"; empty.className = "source-row";
@@ -720,6 +849,7 @@ function renderSourceCandidates(items) {
sourceList.replaceChildren(empty); sourceList.replaceChildren(empty);
return; return;
} }
setSourceBatchAction("get-pictures");
sourceList.replaceChildren(...items.map((item) => { sourceList.replaceChildren(...items.map((item) => {
const section = document.createElement("section"); const section = document.createElement("section");
section.className = "source-section"; section.className = "source-section";
@@ -740,29 +870,85 @@ function renderSourceCandidates(items) {
actions.append(status, pictures); actions.append(status, pictures);
header.append(name, actions); header.append(name, actions);
const rows = item.candidates.length const rows = item.candidates.length
? item.candidates.map(renderSourceCandidateRow) ? item.candidates.map((candidate) => renderSourceCandidateRow(candidate, item.source))
: [renderManualSearchRow(item.searchUrl, item.error || "Open search")]; : [renderManualSearchRow(item.searchUrl, item.error || "Open search", item.source)];
section.append(header, ...rows); section.append(header, ...rows);
return section; return section;
})); }));
} }
function renderPictureCandidates(items) { function renderPictureCandidates(items) {
sourceBatchActionButton.hidden = true;
const rows = []; const rows = [];
let position = 0;
for (const item of items) { for (const item of items) {
for (const candidate of item.candidates || []) { for (const candidate of item.candidates || []) {
for (const picture of candidate.pictures || []) { for (const picture of candidate.pictures || []) {
const row = document.createElement("a"); position += 1;
const sourcePosition = position;
const target = buildPictureTargetInfo();
const row = document.createElement("div");
row.className = "picture-candidate"; row.className = "picture-candidate";
row.href = picture; const preview = document.createElement("a");
row.target = "_blank"; preview.href = picture;
row.rel = "noreferrer"; preview.target = "_blank";
preview.rel = "noreferrer";
preview.className = "picture-candidate-preview";
const image = document.createElement("img"); const image = document.createElement("img");
image.src = picture; image.src = picture;
image.alt = candidate.rawTitle || item.source?.name || "Product picture"; image.alt = candidate.rawTitle || item.source?.name || "Product picture";
image.loading = "lazy"; image.loading = "lazy";
preview.append(image);
const details = document.createElement("div");
details.className = "picture-candidate-meta";
const label = document.createElement("span"); const label = document.createElement("span");
label.textContent = item.source?.name || item.source?.key || "Source"; label.className = "picture-candidate-title";
row.append(image, label); label.textContent = `${item.source?.name || item.source?.key || "Source"} picture #${sourcePosition}`;
const dimensions = document.createElement("span");
dimensions.className = "picture-candidate-dimensions";
dimensions.textContent = "Size: loading...";
image.addEventListener("load", () => {
if (!image.naturalWidth || !image.naturalHeight) {
dimensions.textContent = "Size: unknown";
row.classList.remove("is-too-small");
return;
}
const shortestSide = Math.min(image.naturalWidth, image.naturalHeight);
const longestSide = Math.max(image.naturalWidth, image.naturalHeight);
const isTooSmall = longestSide < MIN_PRODUCT_PICTURE_SIZE;
row.classList.toggle("is-too-small", isTooSmall);
dimensions.textContent = isTooSmall
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px - too small, min long side ${MIN_PRODUCT_PICTURE_SIZE}px`
: shortestSide < MIN_PRODUCT_PICTURE_SIZE
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px - OK for 1:1 crop`
: `Size: ${image.naturalWidth} x ${image.naturalHeight}px`;
});
image.addEventListener("error", () => {
dimensions.textContent = "Size: image preview failed";
row.classList.add("is-too-small");
});
const assignment = document.createElement("span");
assignment.className = "picture-candidate-target";
assignment.textContent = target.label;
const duplicateHint = document.createElement("span");
duplicateHint.className = "picture-candidate-duplicate";
duplicateHint.textContent = "DB: not saved";
details.append(label, dimensions, assignment, duplicateHint);
const action = document.createElement("button");
action.type = "button";
action.className = "source-picture-action picture-candidate-save";
action.textContent = "Save to DB";
action.savePictureCandidate = () => savePictureCandidateToDb({
picture,
item,
candidate,
image,
position: sourcePosition,
target,
action,
duplicateHint,
});
action.addEventListener("click", () => action.savePictureCandidate());
row.append(preview, details, action);
rows.push(row); rows.push(row);
} }
} }
@@ -774,8 +960,121 @@ function renderPictureCandidates(items) {
sourceList.replaceChildren(empty); sourceList.replaceChildren(empty);
return; return;
} }
setSourceBatchAction("save-pictures");
sourceList.replaceChildren(...rows); sourceList.replaceChildren(...rows);
} }
function buildPictureTargetInfo() {
const supplierColorGroup = getFirstSupplierColorGroup();
const firstSupplierRow = supplierColorGroup?.supplierRows?.[0] || currentCombinations[0] || {};
const color = supplierColorGroup?.color || formatValue(firstSupplierRow.color);
const ean13 = supplierColorGroup?.supplierEan || firstSupplierRow.ean13 || "";
if (color) {
return {
strategy: "color",
color,
sizeValue: "",
combi: "",
ean13,
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
label: `Target: color ${color}${ean13 ? ` / EAN ${ean13}` : ""}`,
};
}
return {
strategy: "unknown",
color: "",
sizeValue: "",
combi: "",
ean13,
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
label: ean13 ? `Target: unknown / EAN ${ean13}` : "Target: needs manual assignment",
};
}
async function savePictureCandidateToDb({ picture, item, candidate, image, position, target, action, duplicateHint, }) {
action.disabled = true;
action.textContent = "Saving...";
duplicateHint.textContent = "DB: saving...";
try {
const payload = {
sourceKey: item.source?.key || "",
sourceName: item.source?.name || "",
sourceType: item.source?.type || "",
imageUrl: picture,
pageUrl: candidate.url || item.searchUrl || "",
sourcePosition: position,
manufacturerId: manufacturerSelect.value,
idProductCatalog: currentProduct?.idProductCatalog || target.idProductCatalog || "",
idProduct: currentProduct?.idProduct || "",
supplierReference: currentProduct?.supplierReference || "",
ean13: target.ean13 || "",
color: target.color || "",
sizeValue: target.sizeValue || "",
combi: target.combi || "",
targetStrategy: target.strategy,
targetColor: target.color || "",
targetSize: target.sizeValue || "",
targetCombi: target.combi || "",
targetEan13: target.ean13 || "",
targetIdProductCatalog: target.idProductCatalog || currentProduct?.idProductCatalog || "",
matchConfidence: target.strategy === "color" ? 70 : 0,
width: image.naturalWidth || "",
height: image.naturalHeight || "",
};
const response = await fetch("/api/catalog-maker/save-picture", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
});
const data = await response.json();
if (!response.ok)
throw new Error(data.message || "Picture save failed.");
action.textContent = "Saved";
action.dataset.saved = "1";
duplicateHint.textContent = data.duplicateStatus === "exact_duplicate"
? `DB: duplicate of #${data.duplicateOfCandidateId || "known"}`
: `DB: saved #${data.candidateId}`;
return true;
}
catch (error) {
action.disabled = false;
action.textContent = "Save to DB";
duplicateHint.textContent = `DB: ${error.message || "save failed"}`;
return false;
}
}
async function saveAllVisiblePictureCandidates() {
const buttons = [...sourceList.querySelectorAll(".picture-candidate-save")]
.filter((button) => button.savePictureCandidate && button.dataset.saved !== "1");
if (!buttons.length)
return;
sourceBatchActionButton.disabled = true;
setButtonLabel(sourceBatchActionButton, `Saving 0/${buttons.length}`);
let savedCount = 0;
for (const button of buttons) {
const saved = await button.savePictureCandidate();
if (saved)
savedCount += 1;
setButtonLabel(sourceBatchActionButton, `Saving ${savedCount}/${buttons.length}`);
}
sourceBatchActionButton.disabled = false;
setButtonLabel(sourceBatchActionButton, savedCount === buttons.length ? "Saved all" : `Saved ${savedCount}/${buttons.length}`);
}
async function handleSourceBatchAction() {
if (sourceBatchActionButton.dataset.mode === "get-pictures") {
sourceBatchActionButton.disabled = true;
setButtonLabel(sourceBatchActionButton, "Loading pictures...");
try {
await getPicturesForCurrentProduct("");
}
finally {
sourceBatchActionButton.disabled = false;
if (sourceBatchActionButton.dataset.mode === "get-pictures") {
setButtonLabel(sourceBatchActionButton, "Get all pictures");
}
}
return;
}
await saveAllVisiblePictureCandidates();
}
function collectPictureUrls(items) { function collectPictureUrls(items) {
return items.flatMap((item) => (item.candidates || []).flatMap((candidate) => candidate.pictures || [])).filter((url, index, all) => all.indexOf(url) === index); return items.flatMap((item) => (item.candidates || []).flatMap((candidate) => candidate.pictures || [])).filter((url, index, all) => all.indexOf(url) === index);
} }
@@ -813,44 +1112,63 @@ function clearLocalDraft() {
} }
function updateFindSourcesButtonState() { function updateFindSourcesButtonState() {
const hasSources = Boolean(readLocalDraft()?.sources); const hasSources = Boolean(readLocalDraft()?.sources);
findSourcesButton.textContent = hasSources ? "Find sources ✓" : "Find sources"; setButtonLabel(findSourcesButton, hasSources ? "Find sources ✓" : "Find sources");
findSourcesButton.disabled = !manufacturerSelect.value || !currentProduct; findSourcesButton.disabled = !manufacturerSelect.value || !currentProduct;
} }
function renderSourceCandidateRow(candidate) { function renderSourceCandidateRow(candidate, source) {
const row = document.createElement("a"); const row = document.createElement("div");
row.className = "source-row"; row.className = "source-row";
row.href = candidate.url;
row.target = "_blank";
row.rel = "noreferrer";
const title = document.createElement("span"); const title = document.createElement("span");
title.className = "source-row-title"; title.className = "source-row-title";
title.textContent = formatValue(candidate.title || candidate.url); title.textContent = formatValue(candidate.title || candidate.url);
title.title = formatValue(candidate.rawTitle || candidate.title || candidate.url); title.title = formatValue(candidate.rawTitle || candidate.title || candidate.url);
const action = document.createElement("span"); const action = document.createElement("button");
action.type = "button";
action.className = "source-row-action"; action.className = "source-row-action";
action.textContent = "Open"; action.textContent = "Open";
action.addEventListener("click", () => openSourceCandidate(candidate.url, source, action));
row.append(title, action); row.append(title, action);
return row; return row;
} }
function renderManualSearchRow(url, label) { async function openSourceCandidate(url, source, button) {
if (!url || !source?.key || !manufacturerSelect.value)
return;
const previousText = button.textContent;
button.disabled = true;
button.textContent = "Opening...";
try {
const params = new URLSearchParams({
manufacturerId: manufacturerSelect.value,
sourceKey: source.key,
url,
});
const response = await fetch(`/api/catalog-maker/open-source?${params.toString()}`);
const data = await response.json();
if (!response.ok || data.error)
throw new Error(data.error || "Source browser could not be opened.");
}
catch (error) {
alert(error.message || "Source browser could not be opened.");
}
finally {
button.disabled = false;
button.textContent = previousText;
}
}
function renderManualSearchRow(url, label, source) {
return renderSourceCandidateRow({ return renderSourceCandidateRow({
title: label, title: label,
url, url,
}); }, source);
} }
function closeSourceCandidates() { function closeSourceCandidates() {
sourceModal.hidden = true; sourceModal.hidden = true;
sourceList.replaceChildren(); sourceList.replaceChildren();
} }
function openSettings() { function openSettings() {
loadDatabaseMode();
settingsModal.hidden = false; settingsModal.hidden = false;
} }
function toggleParserDrawer() {
const isOpen = !parserDrawer.hidden;
parserDrawer.hidden = isOpen;
parserDrawerToggle.setAttribute("aria-expanded", String(!isOpen));
parserDrawerToggle.querySelector(".drawer-chevron").textContent = isOpen ? "⌄" : "⌃";
}
function syncSidebarForViewport() { function syncSidebarForViewport() {
if (window.matchMedia("(min-width: 981px)").matches && !sidebarUserClosed) if (window.matchMedia("(min-width: 981px)").matches && !sidebarUserClosed)
openSidebar(); openSidebar();
@@ -887,7 +1205,7 @@ function toggleMainWorkspace() {
mainWorkspaceToggle.setAttribute("aria-expanded", String(!isCollapsed)); mainWorkspaceToggle.setAttribute("aria-expanded", String(!isCollapsed));
mainWorkspaceToggle.setAttribute("aria-label", isCollapsed ? "Expand main workspace" : "Collapse main workspace"); mainWorkspaceToggle.setAttribute("aria-label", isCollapsed ? "Expand main workspace" : "Collapse main workspace");
mainWorkspaceToggle.title = isCollapsed ? "Expand main workspace" : "Collapse main workspace"; mainWorkspaceToggle.title = isCollapsed ? "Expand main workspace" : "Collapse main workspace";
mainWorkspaceToggle.textContent = isCollapsed ? "" : ""; mainWorkspaceToggle.textContent = isCollapsed ? "" : "";
} }
function loadTheme() { function loadTheme() {
const isDark = localStorage.getItem("catalog-maker:theme") === "dark"; const isDark = localStorage.getItem("catalog-maker:theme") === "dark";
@@ -927,9 +1245,7 @@ function loadDatabaseMode() {
const input = document.querySelector(`input[name="databaseMode"][value="${mode}"]`); const input = document.querySelector(`input[name="databaseMode"][value="${mode}"]`);
if (input) if (input)
input.checked = true; input.checked = true;
const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}"); applyPermissionInputs(readDatabasePermissions(mode));
for (const input of permissionInputs)
input.checked = permissions[input.id] === true;
const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction"; const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
const transactionInput = document.querySelector(`input[name="transactionMode"][value="${transactionMode}"]`); const transactionInput = document.querySelector(`input[name="transactionMode"][value="${transactionMode}"]`);
if (transactionInput) if (transactionInput)
@@ -941,12 +1257,14 @@ function loadDatabaseMode() {
} }
syncDatabasePermissions(mode); syncDatabasePermissions(mode);
syncLocalDbFields(mode); syncLocalDbFields(mode);
syncAdminerLoginHint();
renderDatabaseModeNote(mode); renderDatabaseModeNote(mode);
refreshDatabaseStatusLabel(); refreshDatabaseStatusLabel();
} }
function saveDatabaseMode(event) { function saveDatabaseMode(event) {
const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local"; const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
localStorage.setItem("catalog-maker:database-mode", mode); localStorage.setItem("catalog-maker:database-mode", mode);
applyPermissionInputs(readDatabasePermissions(mode));
syncDatabasePermissions(mode); syncDatabasePermissions(mode);
syncLocalDbFields(mode); syncLocalDbFields(mode);
saveSettings(); saveSettings();
@@ -958,6 +1276,16 @@ function saveLocalDbSettings() {
for (const input of localDbInputs) for (const input of localDbInputs)
settings[input.id] = input.value; settings[input.id] = input.value;
localStorage.setItem("catalog-maker:local-db-settings", JSON.stringify(settings)); localStorage.setItem("catalog-maker:local-db-settings", JSON.stringify(settings));
syncAdminerLoginHint();
}
function syncAdminerLoginHint() {
const host = document.querySelector("#localDbHost")?.value || "127.0.0.1";
const port = document.querySelector("#localDbPort")?.value || "3306";
const user = document.querySelector("#localDbUser")?.value || "root";
const catalogDb = document.querySelector("#localDbName")?.value || "9bplus";
adminerServerValue.textContent = `${host}:${port}`;
adminerUserValue.textContent = user;
adminerDatabaseValue.textContent = `${catalogDb} / catalog_scrape`;
} }
async function testLocalDbConnection() { async function testLocalDbConnection() {
saveLocalDbSettings(); saveLocalDbSettings();
@@ -988,11 +1316,63 @@ async function testLocalDbConnection() {
testLocalDbButton.disabled = false; testLocalDbButton.disabled = false;
} }
} }
async function openAdminer() {
openAdminerButton.disabled = true;
const previousNote = databaseModeNote.textContent;
databaseModeNote.textContent = "Opening Adminer...";
try {
const response = await fetch("/api/tools/open-adminer");
const data = await response.json();
if (!response.ok || !data.opened)
throw new Error(data.message || "Adminer could not be opened.");
databaseModeNote.textContent = `Adminer opened: ${data.url}`;
}
catch (error) {
databaseModeNote.textContent = `Adminer error: ${error.message}`;
}
finally {
openDbGateButton.disabled = false;
window.setTimeout(() => {
if (databaseModeNote.textContent.startsWith("Adminer")) {
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
}
else if (databaseModeNote.textContent === "Opening Adminer...") {
databaseModeNote.textContent = previousNote;
}
}, 3500);
}
}
async function openDbGate() {
openDbGateButton.disabled = true;
const previousNote = databaseModeNote.textContent;
databaseModeNote.textContent = "Opening DBGate...";
try {
const response = await fetch("/api/tools/open-dbgate");
const data = await response.json();
if (!response.ok || !data.opened)
throw new Error(data.message || "DBGate could not be opened.");
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
}
catch (error) {
databaseModeNote.textContent = `DBGate error: ${error.message}`;
}
finally {
openDbGateButton.disabled = false;
window.setTimeout(() => {
if (databaseModeNote.textContent.startsWith("DBGate")) {
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
}
else if (databaseModeNote.textContent === "Opening DBGate...") {
databaseModeNote.textContent = previousNote;
}
}, 3500);
}
}
function syncLocalDbFields(mode) { function syncLocalDbFields(mode) {
localDbFields.hidden = mode !== "local"; localDbFields.hidden = mode !== "local";
} }
function syncDatabasePermissions(mode) { function syncDatabasePermissions(mode) {
const isReadOnly = mode === "live" || mode === "scrape"; const isReadOnly = mode === "live";
for (const input of permissionInputs) { for (const input of permissionInputs) {
input.disabled = isReadOnly; input.disabled = isReadOnly;
if (isReadOnly) if (isReadOnly)
@@ -1000,10 +1380,14 @@ function syncDatabasePermissions(mode) {
} }
} }
function saveSettings() { function saveSettings() {
const mode = localStorage.getItem("catalog-maker:database-mode") || "local";
const permissions = {}; const permissions = {};
for (const input of permissionInputs) for (const input of permissionInputs)
permissions[input.id] = input.checked; permissions[input.id] = input.checked;
localStorage.setItem("catalog-maker:db-permissions", JSON.stringify(permissions)); localStorage.setItem(permissionStorageKeys[mode] || permissionStorageKeys.local, JSON.stringify(permissions));
if (mode === "local") {
localStorage.setItem(legacyPermissionStorageKey, JSON.stringify(permissions));
}
const transactionMode = [...transactionModeInputs].find((input) => input.checked)?.value || "transaction"; const transactionMode = [...transactionModeInputs].find((input) => input.checked)?.value || "transaction";
localStorage.setItem("catalog-maker:transaction-mode", transactionMode); localStorage.setItem("catalog-maker:transaction-mode", transactionMode);
refreshDatabaseStatusLabel(); refreshDatabaseStatusLabel();
@@ -1011,8 +1395,8 @@ function saveSettings() {
function refreshDatabaseStatusLabel() { function refreshDatabaseStatusLabel() {
const mode = localStorage.getItem("catalog-maker:database-mode") || "local"; const mode = localStorage.getItem("catalog-maker:database-mode") || "local";
const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction"; const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}"); const permissions = readDatabasePermissions(mode);
const writesEnabled = mode === "local" && (permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle); const writesEnabled = ["local", "scrape"].includes(mode) && (permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle);
const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB"; const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
const configuredDriver = databaseStatus.dataset.driver; const configuredDriver = databaseStatus.dataset.driver;
const connectionError = (mode === "local" && configuredDriver !== "mariadb") const connectionError = (mode === "local" && configuredDriver !== "mariadb")
@@ -1024,6 +1408,33 @@ function refreshDatabaseStatusLabel() {
: (transactionMode === "dry-run" ? "dry-run" : "transaction"); : (transactionMode === "dry-run" ? "dry-run" : "transaction");
databaseStatus.textContent = `${dbLabel} / ${connectionLabel} / ${accessLabel} / ${executionLabel}`; databaseStatus.textContent = `${dbLabel} / ${connectionLabel} / ${accessLabel} / ${executionLabel}`;
databaseStatus.classList.toggle("database-error", connectionError); databaseStatus.classList.toggle("database-error", connectionError);
databaseStateDot.className = `database-state-dot ${databaseConnectionState === "unknown" ? "is-pending" : connectionError ? "is-error" : "is-ready"}`;
databaseStateDot.setAttribute("aria-label", connectionError ? "Database disconnected" : databaseConnectionState === "unknown" ? "Database connection pending" : "Database connected");
}
function readDatabasePermissions(mode = "local") {
const storageKey = permissionStorageKeys[mode] || permissionStorageKeys.local;
const stored = readJsonStorage(storageKey);
if (stored)
return stored;
if (mode === "local") {
const legacy = readJsonStorage(legacyPermissionStorageKey);
if (legacy)
return legacy;
}
return {};
}
function applyPermissionInputs(permissions) {
for (const input of permissionInputs)
input.checked = permissions[input.id] === true;
}
function readJsonStorage(key) {
try {
const value = localStorage.getItem(key);
return value ? JSON.parse(value) : null;
}
catch {
return null;
}
} }
function renderDatabaseModeNote(mode) { function renderDatabaseModeNote(mode) {
databaseModeNote.textContent = mode === "live" databaseModeNote.textContent = mode === "live"
@@ -1205,7 +1616,7 @@ async function loadProductInfoForCurrentProduct() {
return; return;
} }
loadProductButton.disabled = true; loadProductButton.disabled = true;
loadProductButton.textContent = "Loading info..."; setButtonLabel(loadProductButton, "Loading info...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
productId: currentProduct.idProduct, productId: currentProduct.idProduct,
@@ -1228,7 +1639,7 @@ async function loadProductInfoForCurrentProduct() {
setTableMessage(error.message); setTableMessage(error.message);
} }
finally { finally {
loadProductButton.textContent = "Get product info - 00"; setButtonLabel(loadProductButton, "Get product info - 00");
updateNavigationState(); updateNavigationState();
} }
} }
+492 -55
View File
@@ -8,6 +8,7 @@ const manufacturerSearch = document.querySelector("#manufacturerSearch");
const manufacturerOptions = document.querySelector("#manufacturerOptions"); const manufacturerOptions = document.querySelector("#manufacturerOptions");
const manufacturerValue = document.querySelector("#manufacturerValue"); const manufacturerValue = document.querySelector("#manufacturerValue");
const languageSelect = document.querySelector("#languageSelect"); const languageSelect = document.querySelector("#languageSelect");
const languageValue = document.querySelector("#languageValue");
const recordIdElement = document.querySelector("#recordId"); const recordIdElement = document.querySelector("#recordId");
const loadProductButton = document.querySelector("#loadProductButton"); const loadProductButton = document.querySelector("#loadProductButton");
const previousProductButton = document.querySelector("#previousProductButton"); const previousProductButton = document.querySelector("#previousProductButton");
@@ -21,12 +22,11 @@ const suggestedTitle = document.querySelector("#suggestedTitle");
const suggestedList = document.querySelector("#suggestedList"); const suggestedList = document.querySelector("#suggestedList");
const suggestedCloseButton = document.querySelector("#suggestedCloseButton"); const suggestedCloseButton = document.querySelector("#suggestedCloseButton");
const mappingState = document.querySelector("#mappingState"); const mappingState = document.querySelector("#mappingState");
const mappingStatusText = document.querySelector("#mappingStatusText");
const mappingList = document.querySelector("#mappingList"); const mappingList = document.querySelector("#mappingList");
const mappingDetails = document.querySelector("#mappingDetails"); const mappingDetails = document.querySelector("#mappingDetails");
const findSourcesButton = document.querySelector("#findSourcesButton"); const findSourcesButton = document.querySelector("#findSourcesButton");
const getPicturesButton = document.querySelector("#getPicturesButton"); const getPicturesButton = document.querySelector("#getPicturesButton");
const parserDrawerToggle = document.querySelector("#parserDrawerToggle");
const parserDrawer = document.querySelector("#parserDrawer");
const sidebarDrawer = document.querySelector("#sidebarDrawer"); const sidebarDrawer = document.querySelector("#sidebarDrawer");
const sidebarOpenButton = document.querySelector("#sidebarOpenButton"); const sidebarOpenButton = document.querySelector("#sidebarOpenButton");
const sidebarCloseButton = document.querySelector("#sidebarCloseButton"); const sidebarCloseButton = document.querySelector("#sidebarCloseButton");
@@ -34,6 +34,8 @@ const sidebarOverlay = document.querySelector("#sidebarOverlay");
const sourceModal = document.querySelector("#sourceModal"); const sourceModal = document.querySelector("#sourceModal");
const sourceTitle = document.querySelector("#sourceTitle"); const sourceTitle = document.querySelector("#sourceTitle");
const sourceList = document.querySelector("#sourceList"); const sourceList = document.querySelector("#sourceList");
const sourceBatchActionButton = document.querySelector("#sourceBatchActionButton");
const sourceReloadButton = document.querySelector("#sourceReloadButton");
const sourceCloseButton = document.querySelector("#sourceCloseButton"); const sourceCloseButton = document.querySelector("#sourceCloseButton");
const settingsButton = document.querySelector("#settingsButton"); const settingsButton = document.querySelector("#settingsButton");
const themeToggle = document.querySelector("#themeToggle"); const themeToggle = document.querySelector("#themeToggle");
@@ -44,16 +46,31 @@ const databaseModeInputs = document.querySelectorAll('input[name="databaseMode"]
const permissionInputs = document.querySelectorAll("#allowInsertToggle, #allowUpdateToggle, #allowDeleteToggle"); const permissionInputs = document.querySelectorAll("#allowInsertToggle, #allowUpdateToggle, #allowDeleteToggle");
const transactionModeInputs = document.querySelectorAll('input[name="transactionMode"]'); const transactionModeInputs = document.querySelectorAll('input[name="transactionMode"]');
const databaseStatus = document.querySelector("#databaseStatus"); const databaseStatus = document.querySelector("#databaseStatus");
const databaseStateDot = document.querySelector("#databaseStateDot");
const localDbFields = document.querySelector("#localDbFields"); const localDbFields = document.querySelector("#localDbFields");
const localDbInputs = document.querySelectorAll("#localDbFields input"); const localDbInputs = document.querySelectorAll("#localDbFields input");
const testLocalDbButton = document.querySelector("#testLocalDbButton"); const testLocalDbButton = document.querySelector("#testLocalDbButton");
const localDbTestResult = document.querySelector("#localDbTestResult"); const localDbTestResult = document.querySelector("#localDbTestResult");
const openAdminerButton = document.querySelector("#openAdminerButton");
const openDbGateButton = document.querySelector("#openDbGateButton");
const adminerServerValue = document.querySelector("#adminerServerValue");
const adminerUserValue = document.querySelector("#adminerUserValue");
const adminerDatabaseValue = document.querySelector("#adminerDatabaseValue");
const lightContrastSelect = document.querySelector("#lightContrastSelect"); const lightContrastSelect = document.querySelector("#lightContrastSelect");
const darkContrastSelect = document.querySelector("#darkContrastSelect"); const darkContrastSelect = document.querySelector("#darkContrastSelect");
const catalogScreen = document.querySelector(".catalog-screen"); const catalogScreen = document.querySelector(".catalog-screen");
const secondaryPanelToggle = document.querySelector("#secondaryPanelToggle"); const secondaryPanelToggle = document.querySelector("#secondaryPanelToggle");
const mainWorkspaceToggle = document.querySelector("#mainWorkspaceToggle"); const mainWorkspaceToggle = document.querySelector("#mainWorkspaceToggle");
const MIN_PRODUCT_PICTURE_SIZE = 800;
let sidebarUserClosed = false; let sidebarUserClosed = false;
let mappingLoadToken = 0;
const mappingCacheKey = "catalog-maker:mapping-cache";
const legacyPermissionStorageKey = "catalog-maker:db-permissions";
const permissionStorageKeys = {
local: "catalog-maker:db-permissions:local-9bplus",
live: "catalog-maker:db-permissions:live-9bplus",
scrape: "catalog-maker:db-permissions:local-scrape",
};
const nativeFetch = window.fetch.bind(window); const nativeFetch = window.fetch.bind(window);
window.fetch = (input, init = {}) => { window.fetch = (input, init = {}) => {
@@ -62,9 +79,16 @@ window.fetch = (input, init = {}) => {
const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined)); const headers = new Headers(init.headers || (typeof input !== "string" ? input.headers : undefined));
const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local"; const selectedMode = localStorage.getItem("catalog-maker:database-mode") || "local";
const permissions = readDatabasePermissions(selectedMode);
const localWritesEnabled = ["local", "scrape"].includes(selectedMode) && [
permissions.allowInsertToggle,
permissions.allowUpdateToggle,
permissions.allowDeleteToggle,
].some(Boolean);
// Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus. // Scrape storage is separate; existing catalog endpoints continue reading Local 9bplus.
headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local"); headers.set("X-Database-Mode", selectedMode === "live" ? "live" : "local");
headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0"); headers.set("X-Scrape-Database", selectedMode === "scrape" ? "1" : "0");
headers.set("X-Local-Write-Access", localWritesEnabled ? "1" : "0");
return nativeFetch(input, { ...init, headers }); return nativeFetch(input, { ...init, headers });
}; };
@@ -75,7 +99,8 @@ let currentCombinations = [];
let loadRequestId = 0; let loadRequestId = 0;
let applicationMode = "dry-run"; let applicationMode = "dry-run";
let databaseConnectionState = "unknown"; let databaseConnectionState = "unknown";
const localDraftKey = "catalog-maker:current-product-draft:v2"; const localDraftKey = "catalog-maker:current-product-draft:v3";
const selectedManufacturerKey = "catalog-maker:selected-manufacturer";
let localDraftProductKey = ""; let localDraftProductKey = "";
const fieldNames = [ const fieldNames = [
@@ -149,12 +174,18 @@ renderTableHeader();
setTableWidth(); setTableWidth();
await loadManufacturers(); await loadManufacturers();
await loadLanguages(); await loadLanguages();
languageSelect.addEventListener("change", syncLanguagePickerLabel);
manufacturerSelect.addEventListener("change", () => { manufacturerSelect.addEventListener("change", () => {
if (manufacturerSelect.value) {
localStorage.setItem(selectedManufacturerKey, manufacturerSelect.value);
} else {
localStorage.removeItem(selectedManufacturerKey);
}
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
closeManufacturerPicker(); closeManufacturerPicker();
if (manufacturerSelect.value) { if (manufacturerSelect.value) {
loadMappingSources(); loadMappingSources({ checkUrls: false, preferCache: false });
loadProductForSelectedManufacturer(1); loadProductForSelectedManufacturer(1);
} else { } else {
renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." }); renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." });
@@ -175,8 +206,9 @@ loadProductButton.disabled = true;
loadProductButton.title = "Load existing product attribute info."; loadProductButton.title = "Load existing product attribute info.";
loadProductButton.addEventListener("click", loadProductInfoForCurrentProduct); loadProductButton.addEventListener("click", loadProductInfoForCurrentProduct);
findSourcesButton.addEventListener("click", findSourcesForCurrentProduct); findSourcesButton.addEventListener("click", findSourcesForCurrentProduct);
sourceReloadButton.addEventListener("click", () => findSourcesForCurrentProduct(true));
sourceBatchActionButton.addEventListener("click", handleSourceBatchAction);
getPicturesButton.addEventListener("click", getPicturesForCurrentProduct); getPicturesButton.addEventListener("click", getPicturesForCurrentProduct);
parserDrawerToggle.addEventListener("click", toggleParserDrawer);
secondaryPanelToggle.addEventListener("click", toggleSecondaryPanel); secondaryPanelToggle.addEventListener("click", toggleSecondaryPanel);
mainWorkspaceToggle.addEventListener("click", toggleMainWorkspace); mainWorkspaceToggle.addEventListener("click", toggleMainWorkspace);
sidebarOpenButton.addEventListener("click", openSidebar); sidebarOpenButton.addEventListener("click", openSidebar);
@@ -221,6 +253,8 @@ for (const input of permissionInputs) input.addEventListener("change", saveSetti
for (const input of transactionModeInputs) input.addEventListener("change", saveSettings); for (const input of transactionModeInputs) input.addEventListener("change", saveSettings);
for (const input of localDbInputs) input.addEventListener("change", saveLocalDbSettings); for (const input of localDbInputs) input.addEventListener("change", saveLocalDbSettings);
testLocalDbButton.addEventListener("click", testLocalDbConnection); testLocalDbButton.addEventListener("click", testLocalDbConnection);
openAdminerButton.addEventListener("click", openAdminer);
openDbGateButton.addEventListener("click", openDbGate);
lightContrastSelect.addEventListener("change", saveAppearance); lightContrastSelect.addEventListener("change", saveAppearance);
darkContrastSelect.addEventListener("change", saveAppearance); darkContrastSelect.addEventListener("change", saveAppearance);
@@ -229,7 +263,7 @@ syncSidebarForViewport();
setTimeout(() => { setTimeout(() => {
if (manufacturerSelect.value) { if (manufacturerSelect.value) {
loadMappingSources(); loadMappingSources({ checkUrls: false, preferCache: true });
loadProductForSelectedManufacturer(1); loadProductForSelectedManufacturer(1);
} else { } else {
renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." }); renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." });
@@ -301,15 +335,31 @@ async function loadManufacturers() {
manufacturerSelect.append(createOption(String(manufacturer.id), manufacturer.name)); manufacturerSelect.append(createOption(String(manufacturer.id), manufacturer.name));
} }
if (data.message) manufacturerSelect.append(createOption("", data.message)); if (data.message) manufacturerSelect.append(createOption("", data.message));
restoreSelectedManufacturer();
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
renderManufacturerOptions(); renderManufacturerOptions();
} catch (error) { } catch (error) {
manufacturerSelect.append(createOption("", error.message || "Local DB is not configured.")); manufacturerSelect.append(createOption("", error.message || "Local DB is not configured."));
restoreSelectedManufacturer();
syncManufacturerPickerLabel(); syncManufacturerPickerLabel();
renderManufacturerOptions(); renderManufacturerOptions();
} }
} }
function restoreSelectedManufacturer() {
const savedManufacturerId = localStorage.getItem(selectedManufacturerKey);
if (!savedManufacturerId) return;
const savedOption = [...manufacturerSelect.options].find(
(option) => option.value === savedManufacturerId,
);
if (savedOption) {
manufacturerSelect.value = savedManufacturerId;
} else {
localStorage.removeItem(selectedManufacturerKey);
}
}
function toggleManufacturerPicker() { function toggleManufacturerPicker() {
if (manufacturerMenu.hidden) { if (manufacturerMenu.hidden) {
manufacturerMenu.hidden = false; manufacturerMenu.hidden = false;
@@ -369,6 +419,34 @@ function createOption(value, label) {
return option; return option;
} }
function setButtonLabel(button, label) {
const labelElement = button.querySelector(":scope > span");
if (labelElement) {
labelElement.textContent = label;
} else {
button.textContent = label;
}
}
function setButtonIcon(button, iconName) {
const useElement = button.querySelector(":scope > svg use");
if (useElement) useElement.setAttribute("href", `/icons.svg#${iconName}`);
}
function setSourceBatchAction(mode) {
sourceBatchActionButton.dataset.mode = mode;
sourceBatchActionButton.hidden = false;
sourceBatchActionButton.disabled = false;
if (mode === "get-pictures") {
setButtonIcon(sourceBatchActionButton, "image");
setButtonLabel(sourceBatchActionButton, "Get all pictures");
return;
}
setButtonIcon(sourceBatchActionButton, "save");
setButtonLabel(sourceBatchActionButton, "Save all");
}
async function loadLanguages() { async function loadLanguages() {
languageSelect.replaceChildren(createOption("", "Select language")); languageSelect.replaceChildren(createOption("", "Select language"));
try { try {
@@ -384,30 +462,47 @@ async function loadLanguages() {
} catch (error) { } catch (error) {
languageSelect.append(createOption("", error.message || "Local DB is not configured.")); languageSelect.append(createOption("", error.message || "Local DB is not configured."));
} }
syncLanguagePickerLabel();
} }
async function loadMappingSources() { function syncLanguagePickerLabel() {
const option = languageSelect.selectedOptions[0];
languageValue.textContent = option?.textContent || "Select language";
}
async function loadMappingSources({ checkUrls = false, preferCache = false } = {}) {
const manufacturerId = manufacturerSelect.value; const manufacturerId = manufacturerSelect.value;
const loadToken = ++mappingLoadToken;
if (!manufacturerId) { if (!manufacturerId) {
renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." }); renderMappingSources({ mapped: false, items: [], message: "Select manufacturer." });
return; return;
} }
mappingState.textContent = "Loading mapping..."; const cached = preferCache ? readMappingCache(manufacturerId) : null;
mappingState.className = "mapping-state"; if (cached) {
mappingList.replaceChildren(); renderMappingSources(cached);
return;
}
setMappingStatus("Loading mapping...", "is-loading");
try { try {
const params = new URLSearchParams({ manufacturerId, check: "1" }); const params = new URLSearchParams({ manufacturerId });
if (checkUrls) params.set("check", "1");
const response = await fetch(`/api/catalog-maker/mapping-sources?${params.toString()}`); const response = await fetch(`/api/catalog-maker/mapping-sources?${params.toString()}`);
const data = await response.json(); const data = await response.json();
if (loadToken !== mappingLoadToken || manufacturerSelect.value !== manufacturerId) return;
if (!response.ok) { if (!response.ok) {
throw new Error(data.message || "Mapping load failed."); throw new Error(data.message || "Mapping load failed.");
} }
renderMappingSources(data); renderMappingSources(data);
if (data.mapped && (data.items || []).some((item) => item.enabled)) {
saveMappingCache(manufacturerId, data);
}
} catch (error) { } catch (error) {
if (loadToken !== mappingLoadToken || manufacturerSelect.value !== manufacturerId) return;
renderMappingSources({ renderMappingSources({
mapped: false, mapped: false,
items: [], items: [],
@@ -416,26 +511,49 @@ async function loadMappingSources() {
} }
} }
function readMappingCache(manufacturerId) {
try {
const cache = JSON.parse(localStorage.getItem(mappingCacheKey) || "{}");
return cache[String(manufacturerId)] || null;
} catch {
return null;
}
}
function saveMappingCache(manufacturerId, data) {
try {
const cache = JSON.parse(localStorage.getItem(mappingCacheKey) || "{}");
cache[String(manufacturerId)] = {
...data,
cachedAt: new Date().toISOString(),
};
localStorage.setItem(mappingCacheKey, JSON.stringify(cache));
} catch {
// Mapping cache is only a UI convenience. If storage fails, normal loading still works.
}
}
function renderMappingSources(data) { function renderMappingSources(data) {
const items = data.items || []; const items = data.items || [];
if (!data.mapped || !items.length) { if (!data.mapped || !items.length) {
mappingState.textContent = data.message || "Mapping missing"; setMappingStatus(data.message || "Mapping missing", "is-missing");
mappingState.className = "mapping-state is-missing";
mappingList.replaceChildren(); mappingList.replaceChildren();
mappingDetails.open = true; mappingDetails.open = false;
return; return;
} }
const enabledCount = items.filter((item) => item.enabled).length; const enabledCount = items.filter((item) => item.enabled).length;
const failedCount = items.filter((item) => item.enabled && item.health && item.health !== "ok").length; const failedCount = items.filter((item) => item.enabled && item.health && item.health !== "ok").length;
const testedCount = items.filter((item) => item.enabled && item.health === "ok").length; const testedCount = items.filter((item) => item.enabled && item.health === "ok").length;
const checkedCount = items.filter((item) => item.enabled && item.health).length;
const ready = enabledCount > 0 && failedCount === 0; const ready = enabledCount > 0 && failedCount === 0;
mappingState.textContent = ready const statusText = ready
? `Sources OK ✓ (${testedCount}/${enabledCount})` ? `Sources OK ✓ (${testedCount}/${enabledCount})`
: data.message || `Sources problem (${testedCount}/${enabledCount})`; : data.message || `Sources problem (${testedCount}/${enabledCount})`;
mappingState.className = ready ? "mapping-state is-ready" : "mapping-state is-missing"; const displayStatusText = ready && !checkedCount ? `Mapping OK (${enabledCount})` : statusText;
mappingDetails.open = !ready || failedCount > 0; setMappingStatus(displayStatusText, ready ? "is-ready" : testedCount > 0 ? "is-warning" : "is-missing");
mappingDetails.open = false;
mappingList.replaceChildren( mappingList.replaceChildren(
...items.map((item) => { ...items.map((item) => {
@@ -456,8 +574,16 @@ function renderMappingSources(data) {
); );
} }
function setMappingStatus(text, stateClass) {
mappingState.textContent = "";
mappingState.className = `mapping-state-dot ${stateClass}`;
mappingState.setAttribute("aria-label", text);
mappingStatusText.textContent = text;
}
function getMappingHealthText(item) { function getMappingHealthText(item) {
if (!item.enabled) return "off"; if (!item.enabled) return "off";
if (!item.health) return "ready";
if (item.health === "ok") return `${item.httpStatus || 200} OK`; if (item.health === "ok") return `${item.httpStatus || 200} OK`;
if (item.health === "missing-url") return "no URL"; if (item.health === "missing-url") return "no URL";
if (item.httpStatus) return `${item.httpStatus}`; if (item.httpStatus) return `${item.httpStatus}`;
@@ -467,6 +593,7 @@ function getMappingHealthText(item) {
function getMappingHealthClass(item) { function getMappingHealthClass(item) {
if (!item.enabled) return "is-muted"; if (!item.enabled) return "is-muted";
if (!item.health) return "is-ok";
if (item.health === "ok") return "is-ok"; if (item.health === "ok") return "is-ok";
return "is-bad"; return "is-bad";
} }
@@ -650,14 +777,14 @@ function closeSuggestedProducts() {
suggestedList.replaceChildren(); suggestedList.replaceChildren();
} }
async function findSourcesForCurrentProduct() { async function findSourcesForCurrentProduct(forceRefresh = false) {
if (!manufacturerSelect.value || !currentProduct) { if (!manufacturerSelect.value || !currentProduct) {
setTableMessage("Select manufacturer and product first."); setTableMessage("Select manufacturer and product first.");
return; return;
} }
const savedDraft = readLocalDraft(); const savedDraft = readLocalDraft();
if (savedDraft?.sources) { if (savedDraft?.sources && !forceRefresh) {
renderSourceCandidates(savedDraft.sources.items || []); renderSourceCandidates(savedDraft.sources.items || []);
updateFindSourcesButtonState(); updateFindSourcesButtonState();
sourceTitle.textContent = "Source candidates (cached)"; sourceTitle.textContent = "Source candidates (cached)";
@@ -667,7 +794,8 @@ async function findSourcesForCurrentProduct() {
const supplierColorGroup = getFirstSupplierColorGroup(); const supplierColorGroup = getFirstSupplierColorGroup();
findSourcesButton.disabled = true; findSourcesButton.disabled = true;
findSourcesButton.textContent = "Finding sources..."; sourceReloadButton.disabled = true;
setButtonLabel(findSourcesButton, "Finding sources...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
@@ -696,6 +824,7 @@ async function findSourcesForCurrentProduct() {
} catch (error) { } catch (error) {
setTableMessage(error.message || "Source search failed."); setTableMessage(error.message || "Source search failed.");
} finally { } finally {
sourceReloadButton.disabled = false;
updateFindSourcesButtonState(); updateFindSourcesButtonState();
} }
} }
@@ -717,7 +846,7 @@ async function getPicturesForCurrentProduct(sourceKey = "") {
const supplierColorGroup = getFirstSupplierColorGroup(); const supplierColorGroup = getFirstSupplierColorGroup();
getPicturesButton.disabled = true; getPicturesButton.disabled = true;
getPicturesButton.textContent = "Loading pictures..."; setButtonLabel(getPicturesButton, "Loading pictures...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
@@ -744,7 +873,7 @@ async function getPicturesForCurrentProduct(sourceKey = "") {
setTableMessage(error.message || "Picture lookup failed."); setTableMessage(error.message || "Picture lookup failed.");
} finally { } finally {
getPicturesButton.disabled = false; getPicturesButton.disabled = false;
getPicturesButton.textContent = "Get pictures"; setButtonLabel(getPicturesButton, "Get pictures");
updateNavigationState(); updateNavigationState();
} }
} }
@@ -773,6 +902,7 @@ function getFirstSupplierColorGroup() {
} }
function renderSourceCandidates(items) { function renderSourceCandidates(items) {
sourceBatchActionButton.hidden = true;
if (!items.length) { if (!items.length) {
const empty = document.createElement("div"); const empty = document.createElement("div");
empty.className = "source-row"; empty.className = "source-row";
@@ -781,6 +911,7 @@ function renderSourceCandidates(items) {
return; return;
} }
setSourceBatchAction("get-pictures");
sourceList.replaceChildren( sourceList.replaceChildren(
...items.map((item) => { ...items.map((item) => {
const section = document.createElement("section"); const section = document.createElement("section");
@@ -810,8 +941,8 @@ function renderSourceCandidates(items) {
header.append(name, actions); header.append(name, actions);
const rows = item.candidates.length const rows = item.candidates.length
? item.candidates.map(renderSourceCandidateRow) ? item.candidates.map((candidate) => renderSourceCandidateRow(candidate, item.source))
: [renderManualSearchRow(item.searchUrl, item.error || "Open search")]; : [renderManualSearchRow(item.searchUrl, item.error || "Open search", item.source)];
section.append(header, ...rows); section.append(header, ...rows);
return section; return section;
@@ -820,24 +951,90 @@ function renderSourceCandidates(items) {
} }
function renderPictureCandidates(items) { function renderPictureCandidates(items) {
sourceBatchActionButton.hidden = true;
const rows = []; const rows = [];
let position = 0;
for (const item of items) { for (const item of items) {
for (const candidate of item.candidates || []) { for (const candidate of item.candidates || []) {
for (const picture of candidate.pictures || []) { for (const picture of candidate.pictures || []) {
const row = document.createElement("a"); position += 1;
const sourcePosition = position;
const target = buildPictureTargetInfo();
const row = document.createElement("div");
row.className = "picture-candidate"; row.className = "picture-candidate";
row.href = picture;
row.target = "_blank"; const preview = document.createElement("a");
row.rel = "noreferrer"; preview.href = picture;
preview.target = "_blank";
preview.rel = "noreferrer";
preview.className = "picture-candidate-preview";
const image = document.createElement("img"); const image = document.createElement("img");
image.src = picture; image.src = picture;
image.alt = candidate.rawTitle || item.source?.name || "Product picture"; image.alt = candidate.rawTitle || item.source?.name || "Product picture";
image.loading = "lazy"; image.loading = "lazy";
preview.append(image);
const details = document.createElement("div");
details.className = "picture-candidate-meta";
const label = document.createElement("span"); const label = document.createElement("span");
label.textContent = item.source?.name || item.source?.key || "Source"; label.className = "picture-candidate-title";
row.append(image, label); label.textContent = `${item.source?.name || item.source?.key || "Source"} picture #${sourcePosition}`;
const dimensions = document.createElement("span");
dimensions.className = "picture-candidate-dimensions";
dimensions.textContent = "Size: loading...";
image.addEventListener("load", () => {
if (!image.naturalWidth || !image.naturalHeight) {
dimensions.textContent = "Size: unknown";
row.classList.remove("is-too-small");
return;
}
const shortestSide = Math.min(image.naturalWidth, image.naturalHeight);
const longestSide = Math.max(image.naturalWidth, image.naturalHeight);
const isTooSmall = longestSide < MIN_PRODUCT_PICTURE_SIZE;
row.classList.toggle("is-too-small", isTooSmall);
dimensions.textContent = isTooSmall
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px - too small, min long side ${MIN_PRODUCT_PICTURE_SIZE}px`
: shortestSide < MIN_PRODUCT_PICTURE_SIZE
? `Size: ${image.naturalWidth} x ${image.naturalHeight}px - OK for 1:1 crop`
: `Size: ${image.naturalWidth} x ${image.naturalHeight}px`;
});
image.addEventListener("error", () => {
dimensions.textContent = "Size: image preview failed";
row.classList.add("is-too-small");
});
const assignment = document.createElement("span");
assignment.className = "picture-candidate-target";
assignment.textContent = target.label;
const duplicateHint = document.createElement("span");
duplicateHint.className = "picture-candidate-duplicate";
duplicateHint.textContent = "DB: not saved";
details.append(label, dimensions, assignment, duplicateHint);
const action = document.createElement("button");
action.type = "button";
action.className = "source-picture-action picture-candidate-save";
action.textContent = "Save to DB";
action.savePictureCandidate = () =>
savePictureCandidateToDb({
picture,
item,
candidate,
image,
position: sourcePosition,
target,
action,
duplicateHint,
});
action.addEventListener("click", () => action.savePictureCandidate());
row.append(preview, details, action);
rows.push(row); rows.push(row);
} }
} }
@@ -850,9 +1047,141 @@ function renderPictureCandidates(items) {
sourceList.replaceChildren(empty); sourceList.replaceChildren(empty);
return; return;
} }
setSourceBatchAction("save-pictures");
sourceList.replaceChildren(...rows); sourceList.replaceChildren(...rows);
} }
function buildPictureTargetInfo() {
const supplierColorGroup = getFirstSupplierColorGroup();
const firstSupplierRow = supplierColorGroup?.supplierRows?.[0] || currentCombinations[0] || {};
const color = supplierColorGroup?.color || formatValue(firstSupplierRow.color);
const ean13 = supplierColorGroup?.supplierEan || firstSupplierRow.ean13 || "";
if (color) {
return {
strategy: "color",
color,
sizeValue: "",
combi: "",
ean13,
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
label: `Target: color ${color}${ean13 ? ` / EAN ${ean13}` : ""}`,
};
}
return {
strategy: "unknown",
color: "",
sizeValue: "",
combi: "",
ean13,
idProductCatalog: firstSupplierRow.idProductCatalog || currentProduct?.idProductCatalog || "",
label: ean13 ? `Target: unknown / EAN ${ean13}` : "Target: needs manual assignment",
};
}
async function savePictureCandidateToDb({
picture,
item,
candidate,
image,
position,
target,
action,
duplicateHint,
}) {
action.disabled = true;
action.textContent = "Saving...";
duplicateHint.textContent = "DB: saving...";
try {
const payload = {
sourceKey: item.source?.key || "",
sourceName: item.source?.name || "",
sourceType: item.source?.type || "",
imageUrl: picture,
pageUrl: candidate.url || item.searchUrl || "",
sourcePosition: position,
manufacturerId: manufacturerSelect.value,
idProductCatalog: currentProduct?.idProductCatalog || target.idProductCatalog || "",
idProduct: currentProduct?.idProduct || "",
supplierReference: currentProduct?.supplierReference || "",
ean13: target.ean13 || "",
color: target.color || "",
sizeValue: target.sizeValue || "",
combi: target.combi || "",
targetStrategy: target.strategy,
targetColor: target.color || "",
targetSize: target.sizeValue || "",
targetCombi: target.combi || "",
targetEan13: target.ean13 || "",
targetIdProductCatalog: target.idProductCatalog || currentProduct?.idProductCatalog || "",
matchConfidence: target.strategy === "color" ? 70 : 0,
width: image.naturalWidth || "",
height: image.naturalHeight || "",
};
const response = await fetch("/api/catalog-maker/save-picture", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(payload),
});
const data = await response.json();
if (!response.ok) throw new Error(data.message || "Picture save failed.");
action.textContent = "Saved";
action.dataset.saved = "1";
duplicateHint.textContent = data.duplicateStatus === "exact_duplicate"
? `DB: duplicate of #${data.duplicateOfCandidateId || "known"}`
: `DB: saved #${data.candidateId}`;
return true;
} catch (error) {
action.disabled = false;
action.textContent = "Save to DB";
duplicateHint.textContent = `DB: ${error.message || "save failed"}`;
return false;
}
}
async function saveAllVisiblePictureCandidates() {
const buttons = [...sourceList.querySelectorAll(".picture-candidate-save")]
.filter((button) => button.savePictureCandidate && button.dataset.saved !== "1");
if (!buttons.length) return;
sourceBatchActionButton.disabled = true;
setButtonLabel(sourceBatchActionButton, `Saving 0/${buttons.length}`);
let savedCount = 0;
for (const button of buttons) {
const saved = await button.savePictureCandidate();
if (saved) savedCount += 1;
setButtonLabel(sourceBatchActionButton, `Saving ${savedCount}/${buttons.length}`);
}
sourceBatchActionButton.disabled = false;
setButtonLabel(
sourceBatchActionButton,
savedCount === buttons.length ? "Saved all" : `Saved ${savedCount}/${buttons.length}`,
);
}
async function handleSourceBatchAction() {
if (sourceBatchActionButton.dataset.mode === "get-pictures") {
sourceBatchActionButton.disabled = true;
setButtonLabel(sourceBatchActionButton, "Loading pictures...");
try {
await getPicturesForCurrentProduct("");
} finally {
sourceBatchActionButton.disabled = false;
if (sourceBatchActionButton.dataset.mode === "get-pictures") {
setButtonLabel(sourceBatchActionButton, "Get all pictures");
}
}
return;
}
await saveAllVisiblePictureCandidates();
}
function collectPictureUrls(items) { function collectPictureUrls(items) {
return items.flatMap((item) => return items.flatMap((item) =>
(item.candidates || []).flatMap((candidate) => candidate.pictures || []), (item.candidates || []).flatMap((candidate) => candidate.pictures || []),
@@ -892,35 +1221,57 @@ function clearLocalDraft() {
function updateFindSourcesButtonState() { function updateFindSourcesButtonState() {
const hasSources = Boolean(readLocalDraft()?.sources); const hasSources = Boolean(readLocalDraft()?.sources);
findSourcesButton.textContent = hasSources ? "Find sources ✓" : "Find sources"; setButtonLabel(findSourcesButton, hasSources ? "Find sources ✓" : "Find sources");
findSourcesButton.disabled = !manufacturerSelect.value || !currentProduct; findSourcesButton.disabled = !manufacturerSelect.value || !currentProduct;
} }
function renderSourceCandidateRow(candidate) { function renderSourceCandidateRow(candidate, source) {
const row = document.createElement("a"); const row = document.createElement("div");
row.className = "source-row"; row.className = "source-row";
row.href = candidate.url;
row.target = "_blank";
row.rel = "noreferrer";
const title = document.createElement("span"); const title = document.createElement("span");
title.className = "source-row-title"; title.className = "source-row-title";
title.textContent = formatValue(candidate.title || candidate.url); title.textContent = formatValue(candidate.title || candidate.url);
title.title = formatValue(candidate.rawTitle || candidate.title || candidate.url); title.title = formatValue(candidate.rawTitle || candidate.title || candidate.url);
const action = document.createElement("span"); const action = document.createElement("button");
action.type = "button";
action.className = "source-row-action"; action.className = "source-row-action";
action.textContent = "Open"; action.textContent = "Open";
action.addEventListener("click", () => openSourceCandidate(candidate.url, source, action));
row.append(title, action); row.append(title, action);
return row; return row;
} }
function renderManualSearchRow(url, label) { async function openSourceCandidate(url, source, button) {
if (!url || !source?.key || !manufacturerSelect.value) return;
const previousText = button.textContent;
button.disabled = true;
button.textContent = "Opening...";
try {
const params = new URLSearchParams({
manufacturerId: manufacturerSelect.value,
sourceKey: source.key,
url,
});
const response = await fetch(`/api/catalog-maker/open-source?${params.toString()}`);
const data = await response.json();
if (!response.ok || data.error) throw new Error(data.error || "Source browser could not be opened.");
} catch (error) {
alert(error.message || "Source browser could not be opened.");
} finally {
button.disabled = false;
button.textContent = previousText;
}
}
function renderManualSearchRow(url, label, source) {
return renderSourceCandidateRow({ return renderSourceCandidateRow({
title: label, title: label,
url, url,
}); }, source);
} }
function closeSourceCandidates() { function closeSourceCandidates() {
@@ -929,16 +1280,10 @@ function closeSourceCandidates() {
} }
function openSettings() { function openSettings() {
loadDatabaseMode();
settingsModal.hidden = false; settingsModal.hidden = false;
} }
function toggleParserDrawer() {
const isOpen = !parserDrawer.hidden;
parserDrawer.hidden = isOpen;
parserDrawerToggle.setAttribute("aria-expanded", String(!isOpen));
parserDrawerToggle.querySelector(".drawer-chevron").textContent = isOpen ? "⌄" : "⌃";
}
function syncSidebarForViewport() { function syncSidebarForViewport() {
if (window.matchMedia("(min-width: 981px)").matches && !sidebarUserClosed) openSidebar(); if (window.matchMedia("(min-width: 981px)").matches && !sidebarUserClosed) openSidebar();
else closeSidebar(); else closeSidebar();
@@ -977,7 +1322,7 @@ function toggleMainWorkspace() {
mainWorkspaceToggle.setAttribute("aria-expanded", String(!isCollapsed)); mainWorkspaceToggle.setAttribute("aria-expanded", String(!isCollapsed));
mainWorkspaceToggle.setAttribute("aria-label", isCollapsed ? "Expand main workspace" : "Collapse main workspace"); mainWorkspaceToggle.setAttribute("aria-label", isCollapsed ? "Expand main workspace" : "Collapse main workspace");
mainWorkspaceToggle.title = isCollapsed ? "Expand main workspace" : "Collapse main workspace"; mainWorkspaceToggle.title = isCollapsed ? "Expand main workspace" : "Collapse main workspace";
mainWorkspaceToggle.textContent = isCollapsed ? "" : ""; mainWorkspaceToggle.textContent = isCollapsed ? "" : "";
} }
function loadTheme() { function loadTheme() {
@@ -1023,8 +1368,7 @@ function loadDatabaseMode() {
const mode = localStorage.getItem("catalog-maker:database-mode") || "local"; const mode = localStorage.getItem("catalog-maker:database-mode") || "local";
const input = document.querySelector(`input[name="databaseMode"][value="${mode}"]`); const input = document.querySelector(`input[name="databaseMode"][value="${mode}"]`);
if (input) input.checked = true; if (input) input.checked = true;
const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}"); applyPermissionInputs(readDatabasePermissions(mode));
for (const input of permissionInputs) input.checked = permissions[input.id] === true;
const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction"; const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
const transactionInput = document.querySelector(`input[name="transactionMode"][value="${transactionMode}"]`); const transactionInput = document.querySelector(`input[name="transactionMode"][value="${transactionMode}"]`);
if (transactionInput) transactionInput.checked = true; if (transactionInput) transactionInput.checked = true;
@@ -1034,6 +1378,7 @@ function loadDatabaseMode() {
} }
syncDatabasePermissions(mode); syncDatabasePermissions(mode);
syncLocalDbFields(mode); syncLocalDbFields(mode);
syncAdminerLoginHint();
renderDatabaseModeNote(mode); renderDatabaseModeNote(mode);
refreshDatabaseStatusLabel(); refreshDatabaseStatusLabel();
} }
@@ -1041,6 +1386,7 @@ function loadDatabaseMode() {
function saveDatabaseMode(event) { function saveDatabaseMode(event) {
const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local"; const mode = ["local", "live", "scrape"].includes(event.target.value) ? event.target.value : "local";
localStorage.setItem("catalog-maker:database-mode", mode); localStorage.setItem("catalog-maker:database-mode", mode);
applyPermissionInputs(readDatabasePermissions(mode));
syncDatabasePermissions(mode); syncDatabasePermissions(mode);
syncLocalDbFields(mode); syncLocalDbFields(mode);
saveSettings(); saveSettings();
@@ -1052,6 +1398,17 @@ function saveLocalDbSettings() {
const settings = {}; const settings = {};
for (const input of localDbInputs) settings[input.id] = input.value; for (const input of localDbInputs) settings[input.id] = input.value;
localStorage.setItem("catalog-maker:local-db-settings", JSON.stringify(settings)); localStorage.setItem("catalog-maker:local-db-settings", JSON.stringify(settings));
syncAdminerLoginHint();
}
function syncAdminerLoginHint() {
const host = document.querySelector("#localDbHost")?.value || "127.0.0.1";
const port = document.querySelector("#localDbPort")?.value || "3306";
const user = document.querySelector("#localDbUser")?.value || "root";
const catalogDb = document.querySelector("#localDbName")?.value || "9bplus";
adminerServerValue.textContent = `${host}:${port}`;
adminerUserValue.textContent = user;
adminerDatabaseValue.textContent = `${catalogDb} / catalog_scrape`;
} }
async function testLocalDbConnection() { async function testLocalDbConnection() {
@@ -1082,12 +1439,60 @@ async function testLocalDbConnection() {
} }
} }
async function openAdminer() {
openAdminerButton.disabled = true;
const previousNote = databaseModeNote.textContent;
databaseModeNote.textContent = "Opening Adminer...";
try {
const response = await fetch("/api/tools/open-adminer");
const data = await response.json();
if (!response.ok || !data.opened) throw new Error(data.message || "Adminer could not be opened.");
databaseModeNote.textContent = `Adminer opened: ${data.url}`;
} catch (error) {
databaseModeNote.textContent = `Adminer error: ${error.message}`;
} finally {
openDbGateButton.disabled = false;
window.setTimeout(() => {
if (databaseModeNote.textContent.startsWith("Adminer")) {
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
} else if (databaseModeNote.textContent === "Opening Adminer...") {
databaseModeNote.textContent = previousNote;
}
}, 3500);
}
}
async function openDbGate() {
openDbGateButton.disabled = true;
const previousNote = databaseModeNote.textContent;
databaseModeNote.textContent = "Opening DBGate...";
try {
const response = await fetch("/api/tools/open-dbgate");
const data = await response.json();
if (!response.ok || !data.opened) throw new Error(data.message || "DBGate could not be opened.");
databaseModeNote.textContent = `DBGate opened: ${data.url}`;
} catch (error) {
databaseModeNote.textContent = `DBGate error: ${error.message}`;
} finally {
openDbGateButton.disabled = false;
window.setTimeout(() => {
if (databaseModeNote.textContent.startsWith("DBGate")) {
renderDatabaseModeNote(localStorage.getItem("catalog-maker:database-mode") || "local");
} else if (databaseModeNote.textContent === "Opening DBGate...") {
databaseModeNote.textContent = previousNote;
}
}, 3500);
}
}
function syncLocalDbFields(mode) { function syncLocalDbFields(mode) {
localDbFields.hidden = mode !== "local"; localDbFields.hidden = mode !== "local";
} }
function syncDatabasePermissions(mode) { function syncDatabasePermissions(mode) {
const isReadOnly = mode === "live" || mode === "scrape"; const isReadOnly = mode === "live";
for (const input of permissionInputs) { for (const input of permissionInputs) {
input.disabled = isReadOnly; input.disabled = isReadOnly;
if (isReadOnly) input.checked = false; if (isReadOnly) input.checked = false;
@@ -1095,9 +1500,13 @@ function syncDatabasePermissions(mode) {
} }
function saveSettings() { function saveSettings() {
const mode = localStorage.getItem("catalog-maker:database-mode") || "local";
const permissions = {}; const permissions = {};
for (const input of permissionInputs) permissions[input.id] = input.checked; for (const input of permissionInputs) permissions[input.id] = input.checked;
localStorage.setItem("catalog-maker:db-permissions", JSON.stringify(permissions)); localStorage.setItem(permissionStorageKeys[mode] || permissionStorageKeys.local, JSON.stringify(permissions));
if (mode === "local") {
localStorage.setItem(legacyPermissionStorageKey, JSON.stringify(permissions));
}
const transactionMode = [...transactionModeInputs].find((input) => input.checked)?.value || "transaction"; const transactionMode = [...transactionModeInputs].find((input) => input.checked)?.value || "transaction";
localStorage.setItem("catalog-maker:transaction-mode", transactionMode); localStorage.setItem("catalog-maker:transaction-mode", transactionMode);
refreshDatabaseStatusLabel(); refreshDatabaseStatusLabel();
@@ -1106,8 +1515,8 @@ function saveSettings() {
function refreshDatabaseStatusLabel() { function refreshDatabaseStatusLabel() {
const mode = localStorage.getItem("catalog-maker:database-mode") || "local"; const mode = localStorage.getItem("catalog-maker:database-mode") || "local";
const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction"; const transactionMode = localStorage.getItem("catalog-maker:transaction-mode") || "transaction";
const permissions = JSON.parse(localStorage.getItem("catalog-maker:db-permissions") || "{}"); const permissions = readDatabasePermissions(mode);
const writesEnabled = mode === "local" && ( const writesEnabled = ["local", "scrape"].includes(mode) && (
permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle permissions.allowInsertToggle || permissions.allowUpdateToggle || permissions.allowDeleteToggle
); );
const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB"; const dbLabel = mode === "live" ? "Live 9bplus DB" : mode === "scrape" ? "Local scrape DB" : "Local 9bplus DB";
@@ -1121,6 +1530,34 @@ function refreshDatabaseStatusLabel() {
: (transactionMode === "dry-run" ? "dry-run" : "transaction"); : (transactionMode === "dry-run" ? "dry-run" : "transaction");
databaseStatus.textContent = `${dbLabel} / ${connectionLabel} / ${accessLabel} / ${executionLabel}`; databaseStatus.textContent = `${dbLabel} / ${connectionLabel} / ${accessLabel} / ${executionLabel}`;
databaseStatus.classList.toggle("database-error", connectionError); databaseStatus.classList.toggle("database-error", connectionError);
databaseStateDot.className = `database-state-dot ${databaseConnectionState === "unknown" ? "is-pending" : connectionError ? "is-error" : "is-ready"}`;
databaseStateDot.setAttribute("aria-label", connectionError ? "Database disconnected" : databaseConnectionState === "unknown" ? "Database connection pending" : "Database connected");
}
function readDatabasePermissions(mode = "local") {
const storageKey = permissionStorageKeys[mode] || permissionStorageKeys.local;
const stored = readJsonStorage(storageKey);
if (stored) return stored;
if (mode === "local") {
const legacy = readJsonStorage(legacyPermissionStorageKey);
if (legacy) return legacy;
}
return {};
}
function applyPermissionInputs(permissions) {
for (const input of permissionInputs) input.checked = permissions[input.id] === true;
}
function readJsonStorage(key) {
try {
const value = localStorage.getItem(key);
return value ? JSON.parse(value) : null;
} catch {
return null;
}
} }
function renderDatabaseModeNote(mode) { function renderDatabaseModeNote(mode) {
@@ -1320,7 +1757,7 @@ async function loadProductInfoForCurrentProduct() {
} }
loadProductButton.disabled = true; loadProductButton.disabled = true;
loadProductButton.textContent = "Loading info..."; setButtonLabel(loadProductButton, "Loading info...");
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
@@ -1346,7 +1783,7 @@ async function loadProductInfoForCurrentProduct() {
} catch (error) { } catch (error) {
setTableMessage(error.message); setTableMessage(error.message);
} finally { } finally {
loadProductButton.textContent = "Get product info - 00"; setButtonLabel(loadProductButton, "Get product info - 00");
updateNavigationState(); updateNavigationState();
} }
} }
+2
View File
@@ -15,4 +15,6 @@
<symbol id="globe" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9" fill="none" stroke="currentColor" stroke-width="1.8"/><path d="M3 12h18M12 3c2.3 2.4 3.4 5.4 3.4 9S14.3 18.6 12 21M12 3c-2.3 2.4-3.4 5.4-3.4 9S9.7 18.6 12 21" fill="none" stroke="currentColor" stroke-width="1.4"/></symbol> <symbol id="globe" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9" fill="none" stroke="currentColor" stroke-width="1.8"/><path d="M3 12h18M12 3c2.3 2.4 3.4 5.4 3.4 9S14.3 18.6 12 21M12 3c-2.3 2.4-3.4 5.4-3.4 9S9.7 18.6 12 21" fill="none" stroke="currentColor" stroke-width="1.4"/></symbol>
<symbol id="tag" viewBox="0 0 24 24"><path d="m4 5 8.5-.5L20 12l-8 8-7.5-7.5L4 5Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="1.8"/><circle cx="8" cy="8" r="1.2" fill="currentColor"/></symbol> <symbol id="tag" viewBox="0 0 24 24"><path d="m4 5 8.5-.5L20 12l-8 8-7.5-7.5L4 5Z" fill="none" stroke="currentColor" stroke-linejoin="round" stroke-width="1.8"/><circle cx="8" cy="8" r="1.2" fill="currentColor"/></symbol>
<symbol id="close" viewBox="0 0 24 24"><path d="m6 6 12 12M18 6 6 18" fill="none" stroke="currentColor" stroke-linecap="round" stroke-width="1.8"/></symbol> <symbol id="close" viewBox="0 0 24 24"><path d="m6 6 12 12M18 6 6 18" fill="none" stroke="currentColor" stroke-linecap="round" stroke-width="1.8"/></symbol>
<symbol id="refresh" viewBox="0 0 24 24"><path d="M20 11a8 8 0 0 0-14.8-3L3 11m0 0V5m0 6h6M4 13a8 8 0 0 0 14.8 3L21 13m0 0v6m0-6h-6" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8"/></symbol>
<symbol id="external" viewBox="0 0 24 24"><path d="M14 4h6v6M13 11l7-7M20 14v5a1 1 0 0 1-1 1H5a1 1 0 0 1-1-1V5a1 1 0 0 1 1-1h5" fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8"/></symbol>
</svg> </svg>

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

+53 -25
View File
@@ -30,12 +30,6 @@
<button class="arrow-button arrow-button-next" id="nextProductButton" type="button" title="Next product" aria-label="Next product"></button> <button class="arrow-button arrow-button-next" id="nextProductButton" type="button" title="Next product" aria-label="Next product"></button>
</div> </div>
<button class="drawer-trigger" id="parserDrawerToggle" type="button" aria-expanded="true" aria-controls="parserDrawer">
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#sliders"></use></svg>
<span>Parser tools</span>
<span class="drawer-chevron" aria-hidden="true"></span>
</button>
<div class="sidebar-section parser-drawer" id="parserDrawer"> <div class="sidebar-section parser-drawer" id="parserDrawer">
<div class="parser-actions" aria-label="Parser actions"> <div class="parser-actions" aria-label="Parser actions">
<button class="action-button" id="findSourcesButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#search"></use></svg><span>Find sources</span></button> <button class="action-button" id="findSourcesButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#search"></use></svg><span>Find sources</span></button>
@@ -57,8 +51,11 @@
</div> </div>
<div class="select-control"> <div class="select-control">
<svg class="sidebar-icon select-icon" aria-hidden="true"><use href="/icons.svg#globe"></use></svg> <div class="select-display" aria-hidden="true">
<select class="input-like" id="languageSelect" aria-label="Language"> <svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#globe"></use></svg>
<span id="languageValue">Select language</span>
</div>
<select class="input-like language-native-select" id="languageSelect" aria-label="Language">
<option value="">Select language</option> <option value="">Select language</option>
</select> </select>
<span class="select-chevron" aria-hidden="true"></span> <span class="select-chevron" aria-hidden="true"></span>
@@ -66,7 +63,7 @@
<div class="manufacturer-picker" id="manufacturerPicker"> <div class="manufacturer-picker" id="manufacturerPicker">
<button class="manufacturer-trigger" id="manufacturerTrigger" type="button" aria-haspopup="listbox" aria-expanded="false"> <button class="manufacturer-trigger" id="manufacturerTrigger" type="button" aria-haspopup="listbox" aria-expanded="false">
<svg class="sidebar-icon select-icon" aria-hidden="true"><use href="/icons.svg#tag"></use></svg> <svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#tag"></use></svg>
<span id="manufacturerValue">Select manufacturer</span> <span id="manufacturerValue">Select manufacturer</span>
<span class="select-chevron picker-chevron" aria-hidden="true"></span> <span class="select-chevron picker-chevron" aria-hidden="true"></span>
</button> </button>
@@ -75,29 +72,39 @@
<div class="manufacturer-options" id="manufacturerOptions" role="listbox"></div> <div class="manufacturer-options" id="manufacturerOptions" role="listbox"></div>
</div> </div>
</div> </div>
<select class="input-like native-manufacturer-select" id="manufacturerSelect" aria-label="Manufacturer" hidden> <div class="native-manufacturer-host" hidden>
<select class="input-like native-manufacturer-select" id="manufacturerSelect" aria-label="Manufacturer">
<option value="">Select manufacturer</option> <option value="">Select manufacturer</option>
</select> </select>
</div>
<div class="mapping-box" id="mappingBox"> <div class="mapping-box sidebar-mapping-box" id="mappingBox">
<details class="mapping-details" id="mappingDetails"> <details class="mapping-details" id="mappingDetails">
<summary> <summary class="mapping-summary">
<span class="mapping-state is-missing" id="mappingState">Mapping not loaded</span> <span class="mapping-label">
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#layers"></use></svg>
<span class="mapping-title">Mapping</span>
<span class="mapping-state-dot is-missing" id="mappingState" role="status" aria-label="Mapping not loaded"></span>
</span>
<span class="mapping-toggle" aria-hidden="true"></span> <span class="mapping-toggle" aria-hidden="true"></span>
</summary> </summary>
<div class="mapping-panel" role="status">
<div class="mapping-detail-status" id="mappingStatusText">Mapping not loaded</div>
<div class="mapping-list" id="mappingList"></div> <div class="mapping-list" id="mappingList"></div>
</div>
</details> </details>
</div> </div>
<button class="action-button primary" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#menu"></use></svg><span>Show menu (20)</span></button>
<button class="action-button" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#scan"></use></svg><span>Scraper - 03</span></button>
<button class="action-button" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#erase"></use></svg><span>BG remover - 04</span></button>
<div class="sidebar-footer"> <div class="sidebar-footer">
<div class="app-status"> <button class="action-button primary" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#menu"></use></svg><span>Show menu (20)</span></button>
<span class="safe-mode" id="databaseStatus">DB read-only</span>
</div>
<button class="action-button" id="themeToggle" type="button" aria-pressed="false"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#moon"></use></svg><span>Dark theme</span></button> <button class="action-button" id="themeToggle" type="button" aria-pressed="false"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#moon"></use></svg><span>Dark theme</span></button>
<button class="action-button" id="settingsButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#settings"></use></svg><span>Settings</span></button> <button class="action-button" id="settingsButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#settings"></use></svg><span>Settings</span></button>
<div class="app-status">
<span class="database-status-content">
<span class="database-state-dot is-pending" id="databaseStateDot" aria-hidden="true"></span>
<span class="safe-mode" id="databaseStatus">DB read-only</span>
</span>
</div>
</div> </div>
</aside> </aside>
@@ -112,7 +119,7 @@
<section class="data-panel main-workspace" aria-label="Catalog workspace"> <section class="data-panel main-workspace" aria-label="Catalog workspace">
<div class="panel-header column-header workspace-header"> <div class="panel-header column-header workspace-header">
<span>Main workspace</span> <span>Main workspace</span>
<button class="column-toggle" id="mainWorkspaceToggle" type="button" aria-expanded="true" aria-controls="variantGrid" aria-label="Collapse main workspace" title="Collapse main workspace"></button> <button class="column-toggle" id="mainWorkspaceToggle" type="button" aria-expanded="true" aria-controls="variantGrid" aria-label="Collapse main workspace" title="Collapse main workspace"></button>
</div> </div>
<div class="table-wrap"> <div class="table-wrap">
<div class="variant-grid" id="variantGrid" role="table" aria-label="Product combinations"> <div class="variant-grid" id="variantGrid" role="table" aria-label="Product combinations">
@@ -129,7 +136,7 @@
<section class="suggested-dialog" aria-labelledby="suggestedTitle"> <section class="suggested-dialog" aria-labelledby="suggestedTitle">
<header class="suggested-header"> <header class="suggested-header">
<h2 id="suggestedTitle">Sugested product for</h2> <h2 id="suggestedTitle">Sugested product for</h2>
<button class="modal-close" id="suggestedCloseButton" type="button">Close</button> <button class="modal-close" id="suggestedCloseButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#close"></use></svg><span>Close</span></button>
</header> </header>
<div class="suggested-list" id="suggestedList"></div> <div class="suggested-list" id="suggestedList"></div>
</section> </section>
@@ -139,7 +146,11 @@
<section class="source-dialog" aria-labelledby="sourceTitle"> <section class="source-dialog" aria-labelledby="sourceTitle">
<header class="suggested-header"> <header class="suggested-header">
<h2 id="sourceTitle">Source candidates</h2> <h2 id="sourceTitle">Source candidates</h2>
<button class="modal-close" id="sourceCloseButton" type="button">Close</button> <div class="source-dialog-actions">
<button class="modal-close" id="sourceBatchActionButton" type="button" hidden><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#image"></use></svg><span>Get all pictures</span></button>
<button class="modal-close" id="sourceReloadButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#refresh"></use></svg><span>Reload</span></button>
<button class="modal-close" id="sourceCloseButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#close"></use></svg><span>Close</span></button>
</div>
</header> </header>
<div class="source-list" id="sourceList"></div> <div class="source-list" id="sourceList"></div>
</section> </section>
@@ -149,7 +160,7 @@
<section class="settings-dialog" aria-labelledby="settingsTitle"> <section class="settings-dialog" aria-labelledby="settingsTitle">
<header class="suggested-header"> <header class="suggested-header">
<h2 id="settingsTitle">Settings</h2> <h2 id="settingsTitle">Settings</h2>
<button class="modal-close" id="settingsCloseButton" type="button">Close</button> <button class="modal-close" id="settingsCloseButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#close"></use></svg><span>Close</span></button>
</header> </header>
<div class="settings-content"> <div class="settings-content">
<fieldset class="settings-group"> <fieldset class="settings-group">
@@ -165,6 +176,16 @@
<label class="settings-option"> <label class="settings-option">
<input type="radio" name="databaseMode" value="scrape" /> <input type="radio" name="databaseMode" value="scrape" />
<span>Local scrape DB</span> <span>Local scrape DB</span>
<span class="settings-tool-actions">
<button class="settings-inline-button" id="openAdminerButton" type="button">
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#external"></use></svg>
<span>Adminer</span>
</button>
<button class="settings-inline-button" id="openDbGateButton" type="button">
<svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#external"></use></svg>
<span>DBGate</span>
</button>
</span>
</label> </label>
<div class="local-db-fields" id="localDbFields"> <div class="local-db-fields" id="localDbFields">
<label class="settings-field"> <label class="settings-field">
@@ -187,10 +208,17 @@
<span>Password</span> <span>Password</span>
<input id="localDbPassword" type="text" autocomplete="off" /> <input id="localDbPassword" type="text" autocomplete="off" />
</label> </label>
<button class="settings-test-button" id="testLocalDbButton" type="button">Test connection</button> <button class="settings-test-button" id="testLocalDbButton" type="button"><svg class="sidebar-icon" aria-hidden="true"><use href="/icons.svg#refresh"></use></svg><span>Test connection</span></button>
<p class="settings-note" id="localDbTestResult">Connection not tested.</p> <p class="settings-note" id="localDbTestResult">Connection not tested.</p>
</div> </div>
<p class="settings-note" id="databaseModeNote">Local 9bplus database selected.</p> <p class="settings-note" id="databaseModeNote">Local 9bplus database selected.</p>
<div class="adminer-login-hint" id="adminerLoginHint" aria-label="Adminer login details">
<div class="adminer-login-row"><span>System</span><strong>MySQL</strong></div>
<div class="adminer-login-row"><span>Server</span><strong id="adminerServerValue">127.0.0.1:3306</strong></div>
<div class="adminer-login-row"><span>User</span><strong id="adminerUserValue">root</strong></div>
<div class="adminer-login-row"><span>Password</span><strong>DB_PASSWORD in .env</strong></div>
<div class="adminer-login-row"><span>Database</span><strong id="adminerDatabaseValue">9bplus / catalog_scrape</strong></div>
</div>
</fieldset> </fieldset>
<fieldset class="settings-group"> <fieldset class="settings-group">
@@ -232,7 +260,7 @@
<input type="checkbox" id="allowDeleteToggle" /> <input type="checkbox" id="allowDeleteToggle" />
<span>Allow delete</span> <span>Allow delete</span>
</label> </label>
<p class="settings-note">Live DB is always read-only. Write permissions apply only to Local DB.</p> <p class="settings-note">Write permissions are saved separately for Local 9bplus DB and Local scrape DB. Live DB stays read-only.</p>
</fieldset> </fieldset>
<fieldset class="settings-group"> <fieldset class="settings-group">
+1 -1
View File
File diff suppressed because one or more lines are too long
+270
View File
@@ -0,0 +1,270 @@
import fs from "node:fs/promises";
import mariadb from "mariadb";
async function readEnv() {
return Object.fromEntries(
(await fs.readFile(".env", "utf8"))
.split(/\r?\n/)
.map((line) => line.trim())
.filter((line) => line && !line.startsWith("#") && line.includes("="))
.map((line) => {
const index = line.indexOf("=");
return [line.slice(0, index).trim(), line.slice(index + 1).trim()];
}),
);
}
async function queryRows(connection, sql, params = []) {
const rows = await connection.query(sql, params);
return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
}
async function tableExists(connection, schema, table) {
const rows = await queryRows(
connection,
"SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1",
[schema, table],
);
return rows.length > 0;
}
async function columnExists(connection, schema, table, column) {
const rows = await queryRows(
connection,
"SELECT 1 FROM information_schema.columns WHERE table_schema = ? AND table_name = ? AND column_name = ? LIMIT 1",
[schema, table, column],
);
return rows.length > 0;
}
async function addColumnIfMissing(connection, schema, table, column, definition) {
if (await columnExists(connection, schema, table, column)) return;
await connection.query(`ALTER TABLE \`${schema}\`.\`${table}\` ADD COLUMN \`${column}\` ${definition}`);
}
const env = await readEnv();
const catalogDb = env.DB_NAME || "9bplus";
const scrapeDb = env.SCRAPE_DB_NAME || "catalog_scrape";
const connection = await mariadb.createConnection({
host: env.DB_HOST || "127.0.0.1",
port: Number(env.DB_PORT || 3306),
user: env.DB_USER || "root",
password: env.DB_PASSWORD || "",
multipleStatements: false,
bigIntAsNumber: true,
});
try {
await connection.query(`CREATE DATABASE IF NOT EXISTS \`${scrapeDb}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
const schemaSql = await fs.readFile("sql/local-scrape-db.sql", "utf8");
const schemaStatements = schemaSql
.split(/;\s*(?:\r?\n|$)/)
.map((item) => item.trim())
.filter(Boolean);
for (const statement of schemaStatements) {
await connection.query(statement);
}
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "legacy_catalog_source_id", "BIGINT UNSIGNED NULL");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "fallback_source_type", "VARCHAR(64) NOT NULL DEFAULT 'none'");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "access_mode", "VARCHAR(64) NOT NULL DEFAULT 'browser'");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_engine", "VARCHAR(32) NOT NULL DEFAULT 'default'");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "browser_headless", "TINYINT(1) NOT NULL DEFAULT 0");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "wait_after_load_ms", "INT UNSIGNED NOT NULL DEFAULT 3000");
await addColumnIfMissing(connection, scrapeDb, "scrape_sources", "note", "TEXT NULL");
const hasCatalogSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_source");
const hasCatalogManufacturerSourceTable = await tableExists(connection, catalogDb, "ps_product_catalog_manufacturer_source");
if (hasCatalogSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_source (
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
fallback_source_type,
search_url_template,
fallback_url_template,
search_context_keys,
access_mode,
browser_engine,
browser_headless,
wait_after_load_ms,
note
)
SELECT
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
COALESCE(fallback_source_type, 'none'),
search_url_template,
fallback_url_template,
search_context_keys,
COALESCE(access_mode, 'browser'),
COALESCE(browser_engine, 'default'),
COALESCE(browser_headless, 0),
COALESCE(wait_after_load_ms, 3000),
note
FROM \`${catalogDb}\`.ps_product_catalog_source
ON DUPLICATE KEY UPDATE
source_key = VALUES(source_key),
source_name = VALUES(source_name),
source_type = VALUES(source_type),
base_url = VALUES(base_url),
enabled = VALUES(enabled),
priority = VALUES(priority),
search_primary_key = VALUES(search_primary_key),
search_fallback_keys = VALUES(search_fallback_keys),
fallback_source_type = VALUES(fallback_source_type),
search_url_template = VALUES(search_url_template),
fallback_url_template = VALUES(fallback_url_template),
search_context_keys = VALUES(search_context_keys),
access_mode = VALUES(access_mode),
browser_engine = VALUES(browser_engine),
browser_headless = VALUES(browser_headless),
wait_after_load_ms = VALUES(wait_after_load_ms),
note = VALUES(note)
`);
}
if (hasCatalogManufacturerSourceTable) {
await connection.query(`
INSERT INTO \`${scrapeDb}\`.ps_product_catalog_manufacturer_source (
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
)
SELECT
id_product_catalog_manufacturer_source,
id_manufacturer,
id_product_catalog_source,
enabled,
priority,
note
FROM \`${catalogDb}\`.ps_product_catalog_manufacturer_source
ON DUPLICATE KEY UPDATE
id_manufacturer = VALUES(id_manufacturer),
id_product_catalog_source = VALUES(id_product_catalog_source),
enabled = VALUES(enabled),
priority = VALUES(priority),
note = VALUES(note)
`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_source"))) {
throw new Error(`Source table ${scrapeDb}.ps_product_catalog_source does not exist.`);
}
if (!(await tableExists(connection, scrapeDb, "ps_product_catalog_manufacturer_source"))) {
throw new Error(`Mapping table ${scrapeDb}.ps_product_catalog_manufacturer_source does not exist.`);
}
await connection.query(`
INSERT INTO \`${scrapeDb}\`.scrape_sources (
legacy_catalog_source_id,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
fallback_source_type,
search_url_template,
fallback_url_template,
search_context_keys,
access_mode,
browser_engine,
browser_headless,
wait_after_load_ms,
note
)
SELECT
id_product_catalog_source,
source_key,
source_name,
source_type,
base_url,
enabled,
priority,
search_primary_key,
search_fallback_keys,
COALESCE(fallback_source_type, 'none'),
search_url_template,
fallback_url_template,
search_context_keys,
COALESCE(access_mode, 'browser'),
COALESCE(browser_engine, 'default'),
COALESCE(browser_headless, 0),
COALESCE(wait_after_load_ms, 3000),
note
FROM \`${scrapeDb}\`.ps_product_catalog_source
ON DUPLICATE KEY UPDATE
source_key = VALUES(source_key),
source_name = VALUES(source_name),
source_type = VALUES(source_type),
base_url = VALUES(base_url),
enabled = VALUES(enabled),
priority = VALUES(priority),
search_primary_key = VALUES(search_primary_key),
search_fallback_keys = VALUES(search_fallback_keys),
fallback_source_type = VALUES(fallback_source_type),
search_url_template = VALUES(search_url_template),
fallback_url_template = VALUES(fallback_url_template),
search_context_keys = VALUES(search_context_keys),
access_mode = VALUES(access_mode),
browser_engine = VALUES(browser_engine),
browser_headless = VALUES(browser_headless),
wait_after_load_ms = VALUES(wait_after_load_ms),
note = VALUES(note)
`);
await connection.query(`
INSERT INTO \`${scrapeDb}\`.scrape_manufacturer_sources (
manufacturer_id,
source_id,
enabled,
priority
)
SELECT
ms.id_manufacturer,
ss.id,
ms.enabled,
ms.priority
FROM \`${scrapeDb}\`.ps_product_catalog_manufacturer_source ms
JOIN \`${scrapeDb}\`.scrape_sources ss
ON ss.legacy_catalog_source_id = ms.id_product_catalog_source
ON DUPLICATE KEY UPDATE
enabled = VALUES(enabled),
priority = VALUES(priority)
`);
for (const statement of schemaStatements.filter(isCapabilitySeedStatement)) {
await connection.query(statement);
}
const sources = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_sources`);
const mappings = await queryRows(connection, `SELECT COUNT(*) AS total FROM \`${scrapeDb}\`.scrape_manufacturer_sources`);
console.log(`Scrape mapping migration ready: ${sources[0]?.total || 0} sources, ${mappings[0]?.total || 0} manufacturer mappings.`);
} finally {
await connection.end();
}
function isCapabilitySeedStatement(statement) {
return /^INSERT INTO `scrape_(source_capabilities|manufacturer_source_capabilities)`/i.test(statement);
}
+94
View File
@@ -0,0 +1,94 @@
import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { get } from "node:https";
import { dirname, resolve } from "node:path";
import { spawn, spawnSync } from "node:child_process";
const adminerFile = resolve("tools/adminer/adminer.php");
const localPhp = resolve("tools/php/php.exe");
const host = process.env.ADMINER_HOST || "127.0.0.1";
const port = process.env.ADMINER_PORT || "8080";
const adminerDownloadUrl = process.env.ADMINER_DOWNLOAD_URL || "https://www.adminer.org/latest.php";
const phpCommand = existsSync(localPhp) ? localPhp : "php";
const usesLocalPhp = existsSync(localPhp);
if (!commandExists(phpCommand)) {
console.error("Adminer needs PHP CLI. Install PHP first, then run npm run adminer again.");
process.exit(1);
}
if (usesLocalPhp) {
ensureLocalPhpMysqlExtensions();
}
if (!existsSync(adminerFile)) {
mkdirSync(dirname(adminerFile), { recursive: true });
await downloadFile(adminerDownloadUrl, adminerFile);
}
const child = spawn(phpCommand, ["-S", `${host}:${port}`, adminerFile], {
stdio: "inherit",
shell: true,
});
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
process.exit(code ?? 0);
});
function commandExists(command: string): boolean {
const result = spawnSync(command, ["-v"], { stdio: "ignore", shell: true });
return result.status === 0;
}
function ensureLocalPhpMysqlExtensions(): void {
const phpDir = dirname(localPhp);
const phpIni = resolve(phpDir, "php.ini");
const template = resolve(phpDir, "php.ini-production");
const source = existsSync(phpIni) ? phpIni : template;
if (!existsSync(source)) {
console.warn("Local PHP is missing php.ini-production; Adminer may not have MariaDB extensions enabled.");
return;
}
let contents = readFileSync(source, "utf8");
contents = ensureIniLine(contents, /^;?\s*extension_dir\s*=.*$/m, 'extension_dir = "ext"');
contents = ensureIniLine(contents, /^;?\s*extension\s*=\s*mysqli\s*$/m, "extension=mysqli");
contents = ensureIniLine(contents, /^;?\s*extension\s*=\s*pdo_mysql\s*$/m, "extension=pdo_mysql");
writeFileSync(phpIni, contents);
}
function ensureIniLine(contents: string, pattern: RegExp, line: string): string {
if (pattern.test(contents)) return contents.replace(pattern, line);
return `${contents.trimEnd()}\n${line}\n`;
}
function downloadFile(url: string, target: string): Promise<void> {
return new Promise((resolveDownload, rejectDownload) => {
const request = get(url, (response) => {
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
downloadFile(response.headers.location, target).then(resolveDownload, rejectDownload);
return;
}
if (response.statusCode !== 200) {
rejectDownload(new Error(`Adminer download failed with HTTP ${response.statusCode}`));
return;
}
const file = createWriteStream(target);
response.pipe(file);
file.on("finish", () => {
file.close();
resolveDownload();
});
file.on("error", rejectDownload);
});
request.on("error", rejectDownload);
});
}
+54
View File
@@ -0,0 +1,54 @@
import { existsSync } from "node:fs";
import { spawn } from "node:child_process";
import { resolve } from "node:path";
import { loadConfig } from "../src/config.ts";
const config = loadConfig();
const dbgateBin = resolve("node_modules/.bin/dbgate-serve.cmd");
const command = existsSync(dbgateBin) ? dbgateBin : "dbgate-serve";
const host = config.database.host || "127.0.0.1";
const port = String(config.database.port || 3306);
const user = config.database.user || "root";
const password = config.database.password || "";
const catalogDatabase = config.database.name || "9bplus";
const scrapeDatabase = config.database.scrapeName || "catalog_scrape";
const dbgateUrl = new URL(config.dbGate.url || "http://127.0.0.1:3000");
const env = {
...process.env,
PORT: dbgateUrl.port || "3000",
SKIP_ALL_AUTH: "1",
CONNECTIONS: "local_9bplus,local_scrape",
ENGINE_local_9bplus: "mysql@dbgate-plugin-mysql",
LABEL_local_9bplus: "Local 9bplus DB",
SERVER_local_9bplus: host,
PORT_local_9bplus: port,
USER_local_9bplus: user,
PASSWORD_local_9bplus: password,
DATABASE_local_9bplus: catalogDatabase,
ENGINE_local_scrape: "mysql@dbgate-plugin-mysql",
LABEL_local_scrape: "Local scrape DB",
SERVER_local_scrape: host,
PORT_local_scrape: port,
USER_local_scrape: user,
PASSWORD_local_scrape: password,
DATABASE_local_scrape: scrapeDatabase,
};
console.log(`Starting DBGate at ${dbgateUrl.origin}`);
console.log(`Configured connections: Local 9bplus DB, Local scrape DB`);
const child = spawn(command, [], {
env,
stdio: "inherit",
shell: true,
});
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
process.exit(code ?? 0);
});
+387
View File
@@ -3,8 +3,44 @@ CREATE DATABASE IF NOT EXISTS `catalog_scrape`
USE `catalog_scrape`; USE `catalog_scrape`;
CREATE TABLE IF NOT EXISTS `ps_product_catalog_source` (
`id_product_catalog_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_key` VARCHAR(64) NOT NULL,
`source_name` VARCHAR(128) NOT NULL,
`source_type` VARCHAR(32) NOT NULL DEFAULT '',
`base_url` VARCHAR(512) NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`search_primary_key` VARCHAR(64) NOT NULL DEFAULT 'ean',
`search_fallback_keys` VARCHAR(255) NOT NULL DEFAULT '',
`fallback_source_type` VARCHAR(32) NOT NULL DEFAULT 'none',
`search_url_template` VARCHAR(1000) NOT NULL,
`fallback_url_template` VARCHAR(1000) NOT NULL DEFAULT '',
`search_context_keys` VARCHAR(255) NOT NULL DEFAULT '',
`access_mode` VARCHAR(32) NOT NULL DEFAULT 'browser',
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`note` TEXT NULL,
PRIMARY KEY (`id_product_catalog_source`),
UNIQUE KEY `uq_product_catalog_source_key` (`source_key`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `ps_product_catalog_manufacturer_source` (
`id_product_catalog_manufacturer_source` INT UNSIGNED NOT NULL AUTO_INCREMENT,
`id_manufacturer` INT UNSIGNED NOT NULL,
`id_product_catalog_source` INT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`note` TEXT NULL,
PRIMARY KEY (`id_product_catalog_manufacturer_source`),
UNIQUE KEY `uq_product_catalog_manufacturer_source` (`id_manufacturer`, `id_product_catalog_source`),
KEY `idx_product_catalog_manufacturer_source_source` (`id_product_catalog_source`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_sources` ( CREATE TABLE IF NOT EXISTS `scrape_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`legacy_catalog_source_id` BIGINT UNSIGNED NULL,
`source_key` VARCHAR(64) NOT NULL, `source_key` VARCHAR(64) NOT NULL,
`source_name` VARCHAR(128) NOT NULL, `source_name` VARCHAR(128) NOT NULL,
`source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier', `source_type` ENUM('supplier', 'manufacturer', 'backup', 'other') NOT NULL DEFAULT 'supplier',
@@ -13,19 +49,85 @@ CREATE TABLE IF NOT EXISTS `scrape_sources` (
`priority` INT NOT NULL DEFAULT 100, `priority` INT NOT NULL DEFAULT 100,
`search_primary_key` VARCHAR(64) NULL, `search_primary_key` VARCHAR(64) NULL,
`search_fallback_keys` VARCHAR(255) NULL, `search_fallback_keys` VARCHAR(255) NULL,
`fallback_source_type` VARCHAR(64) NOT NULL DEFAULT 'none',
`search_url_template` VARCHAR(2048) NULL, `search_url_template` VARCHAR(2048) NULL,
`fallback_url_template` VARCHAR(2048) NULL, `fallback_url_template` VARCHAR(2048) NULL,
`search_context_keys` VARCHAR(255) NULL, `search_context_keys` VARCHAR(255) NULL,
`access_mode` VARCHAR(64) NOT NULL DEFAULT 'browser',
`browser_engine` VARCHAR(32) NOT NULL DEFAULT 'default',
`browser_headless` TINYINT(1) NOT NULL DEFAULT 0,
`wait_after_load_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`scraper_config` JSON NULL, `scraper_config` JSON NULL,
`request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000, `request_delay_ms` INT UNSIGNED NOT NULL DEFAULT 3000,
`last_health_status` SMALLINT UNSIGNED NULL, `last_health_status` SMALLINT UNSIGNED NULL,
`last_health_checked_at` DATETIME NULL, `last_health_checked_at` DATETIME NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_sources_legacy` (`legacy_catalog_source_id`),
UNIQUE KEY `uq_scrape_sources_key` (`source_key`) UNIQUE KEY `uq_scrape_sources_key` (`source_key`)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_sources` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NOT NULL,
`source_id` BIGINT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_manufacturer_source` (`manufacturer_id`, `source_id`),
KEY `idx_scrape_manufacturer_source_source` (`source_id`),
CONSTRAINT `fk_scrape_manufacturer_source_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_source_capabilities` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_id` BIGINT UNSIGNED NOT NULL,
`capability` ENUM('search', 'pictures', 'texts', 'features') NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`parser_key` VARCHAR(128) NOT NULL,
`parser_module` VARCHAR(255) NULL,
`browser_engine` VARCHAR(32) NULL,
`browser_headless` TINYINT(1) NULL,
`wait_after_load_ms` INT UNSIGNED NULL,
`min_width` INT UNSIGNED NULL,
`min_height` INT UNSIGNED NULL,
`min_longest_side` INT UNSIGNED NULL,
`default_target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
`selector_config` JSON NULL,
`extraction_config` JSON NULL,
`quality_config` JSON NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_source_capability_parser` (`source_id`, `capability`, `parser_key`),
KEY `idx_scrape_source_capabilities_lookup` (`source_id`, `capability`, `enabled`, `priority`),
CONSTRAINT `fk_scrape_source_capabilities_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_source_id` BIGINT UNSIGNED NOT NULL,
`source_capability_id` BIGINT UNSIGNED NOT NULL,
`enabled` TINYINT(1) NOT NULL DEFAULT 1,
`priority` INT NOT NULL DEFAULT 100,
`target_strategy_override` ENUM('unknown', 'color', 'size', 'combination', 'manual') NULL,
`config_override` JSON NULL,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_manufacturer_source_capability` (`manufacturer_source_id`, `source_capability_id`),
KEY `idx_scrape_manufacturer_source_capabilities_capability` (`source_capability_id`),
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_mapping` FOREIGN KEY (`manufacturer_source_id`) REFERENCES `scrape_manufacturer_sources` (`id`),
CONSTRAINT `fk_scrape_manufacturer_source_capabilities_capability` FOREIGN KEY (`source_capability_id`) REFERENCES `scrape_source_capabilities` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_products` ( CREATE TABLE IF NOT EXISTS `scrape_products` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NULL, `manufacturer_id` INT UNSIGNED NULL,
@@ -70,6 +172,81 @@ CREATE TABLE IF NOT EXISTS `scrape_assets` (
CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`) CONSTRAINT `fk_scrape_assets_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_image_blobs` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`sha256` CHAR(64) NOT NULL,
`perceptual_hash` CHAR(16) NULL,
`mime_type` VARCHAR(64) NOT NULL,
`width` INT UNSIGNED NULL,
`height` INT UNSIGNED NULL,
`size_bytes` INT UNSIGNED NOT NULL,
`image_data` LONGBLOB NOT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_image_blobs_sha256` (`sha256`),
KEY `idx_scrape_image_blobs_phash` (`perceptual_hash`),
KEY `idx_scrape_image_blobs_size` (`width`, `height`, `size_bytes`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_picture_candidates` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`image_blob_id` BIGINT UNSIGNED NOT NULL,
`source_id` BIGINT UNSIGNED NOT NULL,
`scrape_product_id` BIGINT UNSIGNED NULL,
`manufacturer_id` INT UNSIGNED NULL,
`id_product_catalog` BIGINT UNSIGNED NULL,
`id_product` BIGINT UNSIGNED NULL,
`supplier_reference` VARCHAR(255) NULL,
`ean13` VARCHAR(32) NULL,
`color` VARCHAR(255) NULL,
`size_value` VARCHAR(128) NULL,
`combi` VARCHAR(255) NULL,
`source_page_url` VARCHAR(2048) NULL,
`source_image_url` VARCHAR(2048) NOT NULL,
`source_position` INT UNSIGNED NOT NULL DEFAULT 0,
`target_strategy` ENUM('unknown', 'color', 'size', 'combination', 'manual') NOT NULL DEFAULT 'unknown',
`target_color` VARCHAR(255) NULL,
`target_size` VARCHAR(128) NULL,
`target_combi` VARCHAR(255) NULL,
`target_ean13` VARCHAR(32) NULL,
`target_id_product_catalog` BIGINT UNSIGNED NULL,
`match_confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
`duplicate_status` ENUM('unique', 'exact_duplicate', 'visual_duplicate', 'review') NOT NULL DEFAULT 'review',
`duplicate_of_candidate_id` BIGINT UNSIGNED NULL,
`status` ENUM('draft', 'approved', 'rejected') NOT NULL DEFAULT 'draft',
`review_note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uq_scrape_picture_candidate_source_image` (`source_id`, `source_image_url`(512)),
KEY `idx_scrape_picture_candidates_blob` (`image_blob_id`),
KEY `idx_scrape_picture_candidates_source` (`source_id`),
KEY `idx_scrape_picture_candidates_product` (`manufacturer_id`, `supplier_reference`, `ean13`),
KEY `idx_scrape_picture_candidates_target` (`target_strategy`, `target_color`, `target_size`, `target_combi`),
KEY `idx_scrape_picture_candidates_duplicate` (`duplicate_status`, `duplicate_of_candidate_id`),
CONSTRAINT `fk_scrape_picture_candidates_blob` FOREIGN KEY (`image_blob_id`) REFERENCES `scrape_image_blobs` (`id`),
CONSTRAINT `fk_scrape_picture_candidates_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`),
CONSTRAINT `fk_scrape_picture_candidates_product` FOREIGN KEY (`scrape_product_id`) REFERENCES `scrape_products` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`manufacturer_id` INT UNSIGNED NULL,
`source_id` BIGINT UNSIGNED NULL,
`id_category_default` INT UNSIGNED NULL,
`catalog_category` VARCHAR(255) NULL,
`product_name_pattern` VARCHAR(255) NULL,
`target_strategy` ENUM('color', 'size', 'combination', 'manual') NOT NULL,
`confidence` DECIMAL(5,2) NOT NULL DEFAULT 0.00,
`examples_count` INT UNSIGNED NOT NULL DEFAULT 0,
`note` TEXT NULL,
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `idx_scrape_picture_target_rules_lookup` (`manufacturer_id`, `source_id`, `id_category_default`),
CONSTRAINT `fk_scrape_picture_target_rules_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS `scrape_runs` ( CREATE TABLE IF NOT EXISTS `scrape_runs` (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`source_id` BIGINT UNSIGNED NULL, `source_id` BIGINT UNSIGNED NULL,
@@ -86,3 +263,213 @@ CREATE TABLE IF NOT EXISTS `scrape_runs` (
KEY `idx_scrape_runs_source` (`source_id`), KEY `idx_scrape_runs_source` (`source_id`),
CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`) CONSTRAINT `fk_scrape_runs_source` FOREIGN KEY (`source_id`) REFERENCES `scrape_sources` (`id`)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`min_longest_side`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`quality_config`,
`note`
)
SELECT
`id`,
'pictures',
1,
10,
'hudy-gallery',
'hudy.picture-gallery',
NULL,
NULL,
NULL,
800,
'color',
JSON_OBJECT(
'colorSwitcherSelector', 'a[id^="product-change-color-"]',
'eanTableSelector', 'table.product-params__params-table',
'topImageSelector', 'p#snippet-productInfo-image img.product-top__img',
'gallerySelector', 'p#snippet-productInfo-image, .pdbox__window'
),
JSON_OBJECT(
'verifyVariantBy', JSON_ARRAY('ean', 'color'),
'pictureSource', 'largeGallery',
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
),
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
'Hudy product gallery pictures. Verify the color variant by EAN before extracting large gallery images.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`min_longest_side` = VALUES(`min_longest_side`),
`default_target_strategy` = VALUES(`default_target_strategy`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`quality_config` = VALUES(`quality_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`note`
)
SELECT
`id`,
'texts',
1,
30,
'hudy-product-texts',
'hudy.product-texts',
NULL,
NULL,
NULL,
'manual',
JSON_OBJECT('descriptionSelector', '.product-detail, .product-top, [itemprop="description"]'),
JSON_OBJECT('fields', JSON_ARRAY('shortDescription', 'description', 'marketingText')),
'Hudy text extraction placeholder. User will teach exact text fields before automation is trusted.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`note`
)
SELECT
`id`,
'features',
1,
20,
'hudy-product-params',
'hudy.product-params',
NULL,
NULL,
NULL,
'manual',
JSON_OBJECT('paramsTableSelector', 'table.product-params__params-table'),
JSON_OBJECT('fields', JSON_ARRAY('label', 'value'), 'mergeStrategy', 'review-first'),
'Hudy feature extraction placeholder. Product parameters are kept reviewable before catalog writes.'
FROM `scrape_sources`
WHERE `source_key` = 'hudy'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_source_capabilities` (
`source_id`,
`capability`,
`enabled`,
`priority`,
`parser_key`,
`parser_module`,
`browser_engine`,
`browser_headless`,
`wait_after_load_ms`,
`min_longest_side`,
`default_target_strategy`,
`selector_config`,
`extraction_config`,
`quality_config`,
`note`
)
SELECT
`id`,
'pictures',
1,
20,
'idealo-splide-gallery',
'idealo.splide-gallery',
'firefox',
1,
3000,
800,
'color',
JSON_OBJECT(
'galleryTrackSelector', 'div.splide__track.container',
'slideImageSelector', 'ul.splide__list li.splide__slide img'
),
JSON_OBJECT(
'urlAttributes', JSON_ARRAY('srcset', 'data-srcset', 'data-large', 'data-original', 'data-src', 'src'),
'pictureSource', 'productGalleryOnly',
'rejectSearchThumbnails', true,
'dedupeBy', JSON_ARRAY('sha256', 'perceptual_hash')
),
JSON_OBJECT('minLongestSide', 800, 'allowSquareCrop', true),
'Idealo is primarily an image backup source. Use product gallery images only, not search-result thumbnails.'
FROM `scrape_sources`
WHERE `source_key` = 'idealo'
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`),
`parser_module` = VALUES(`parser_module`),
`browser_engine` = VALUES(`browser_engine`),
`browser_headless` = VALUES(`browser_headless`),
`wait_after_load_ms` = VALUES(`wait_after_load_ms`),
`min_longest_side` = VALUES(`min_longest_side`),
`default_target_strategy` = VALUES(`default_target_strategy`),
`selector_config` = VALUES(`selector_config`),
`extraction_config` = VALUES(`extraction_config`),
`quality_config` = VALUES(`quality_config`),
`note` = VALUES(`note`);
INSERT INTO `scrape_manufacturer_source_capabilities` (
`manufacturer_source_id`,
`source_capability_id`,
`enabled`,
`priority`,
`target_strategy_override`
)
SELECT
`sms`.`id`,
`sc`.`id`,
`sc`.`enabled`,
`sc`.`priority`,
NULL
FROM `scrape_manufacturer_sources` `sms`
JOIN `scrape_source_capabilities` `sc`
ON `sc`.`source_id` = `sms`.`source_id`
ON DUPLICATE KEY UPDATE
`enabled` = VALUES(`enabled`),
`priority` = VALUES(`priority`);
+31 -1
View File
@@ -30,7 +30,10 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
database: { database: {
driver: String(env.DB_DRIVER || config.database?.driver || "endpoint").toLowerCase(), driver: String(env.DB_DRIVER || config.database?.driver || "endpoint").toLowerCase(),
mode: String(env.DB_MODE || config.database?.mode || "local").toLowerCase(), mode: String(env.DB_MODE || config.database?.mode || "local").toLowerCase(),
allowWrites: parseBoolean(env.DB_ALLOW_WRITES, config.database?.allowWrites), allowLocalWrites: parseBoolean(
env.DB_LOCAL_ALLOW_WRITES,
config.database?.allowLocalWrites ?? config.database?.allowWrites,
),
host: env.DB_HOST || config.database?.host || "127.0.0.1", host: env.DB_HOST || config.database?.host || "127.0.0.1",
port: Number(env.DB_PORT || config.database?.port || 3306), port: Number(env.DB_PORT || config.database?.port || 3306),
name: env.DB_NAME || config.database?.name || "catalog_maker_test", name: env.DB_NAME || config.database?.name || "catalog_maker_test",
@@ -39,6 +42,23 @@ export function loadConfig({ configPath = "config/local.json", dryRun } = {}) {
scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape", scrapeName: env.SCRAPE_DB_NAME || config.database?.scrapeName || "catalog_scrape",
connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5), connectionLimit: Number(env.DB_CONNECTION_LIMIT || config.database?.connectionLimit || 5),
}, },
databaseTool: {
url: env.ADMINER_URL || config.databaseTool?.url || config.databaseGate?.url || "http://127.0.0.1:8080",
browserEngine: String(
env.ADMINER_BROWSER_ENGINE
|| config.databaseTool?.browserEngine
|| config.databaseGate?.browserEngine
|| "chromium",
).toLowerCase(),
},
dbGate: {
url: env.DBGATE_URL || config.dbGate?.url || "http://127.0.0.1:3000",
browserEngine: String(
env.DBGATE_BROWSER_ENGINE
|| config.dbGate?.browserEngine
|| "chromium",
).toLowerCase(),
},
}; };
} }
@@ -86,5 +106,15 @@ function mergeConfig(base, override) {
...base.database, ...base.database,
...override.database, ...override.database,
}, },
databaseTool: {
...base.databaseTool,
...base.databaseGate,
...override.databaseTool,
...override.databaseGate,
},
dbGate: {
...base.dbGate,
...override.dbGate,
},
}; };
} }
+2 -2
View File
@@ -40,8 +40,8 @@ export async function executeMariaDbWrite(config, sql) {
if (config.database.mode !== "local") { if (config.database.mode !== "local") {
throw new Error("MariaDB writes are blocked outside Local DB mode."); throw new Error("MariaDB writes are blocked outside Local DB mode.");
} }
if (!config.database.allowWrites) { if (!config.database.allowLocalWrites) {
throw new Error("MariaDB writes are disabled. Enable local database writes in settings first."); throw new Error("Local MariaDB writes are disabled. Enable local database writes in Settings first.");
} }
const connection = await getPool(config).getConnection(); const connection = await getPool(config).getConnection();
+319
View File
@@ -0,0 +1,319 @@
// @ts-nocheck
import crypto from "node:crypto";
import mariadb from "mariadb";
export async function saveScrapePictureCandidate(config, picture) {
if (config.database?.driver !== "mariadb") {
throw new Error("Local scrape DB needs MariaDB configuration.");
}
if (!config.database?.allowLocalWrites) {
throw new Error("Local scrape DB writes are disabled. Enable writes in Settings first.");
}
const imageUrl = normalizeRequiredUrl(picture.imageUrl, "imageUrl");
const sourceKey = normalizeText(picture.sourceKey) || "unknown";
const sourceName = normalizeText(picture.sourceName) || sourceKey;
const sourcePageUrl = normalizeOptionalText(picture.pageUrl);
const image = await downloadImage(imageUrl);
const sha256 = crypto.createHash("sha256").update(image.buffer).digest("hex");
const scrapeDb = config.database.scrapeName || "catalog_scrape";
const connection = await mariadb.createConnection({
host: config.database.host,
port: Number(config.database.port),
database: scrapeDb,
user: config.database.user,
password: config.database.password,
bigIntAsNumber: true,
});
try {
await connection.beginTransaction();
const sourceId = await upsertSource(connection, {
sourceKey,
sourceName,
imageUrl,
sourceType: normalizeText(picture.sourceType) || "supplier",
});
const imageBlobId = await upsertImageBlob(connection, {
sha256,
mimeType: image.mimeType,
width: normalizePositiveNumber(picture.width),
height: normalizePositiveNumber(picture.height),
sizeBytes: image.buffer.byteLength,
buffer: image.buffer,
});
const duplicate = await findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl);
const candidateId = await upsertPictureCandidate(connection, {
imageBlobId,
sourceId,
manufacturerId: normalizePositiveNumber(picture.manufacturerId),
idProductCatalog: normalizePositiveNumber(picture.idProductCatalog),
idProduct: normalizePositiveNumber(picture.idProduct),
supplierReference: normalizeOptionalText(picture.supplierReference),
ean13: normalizeOptionalText(picture.ean13),
color: normalizeOptionalText(picture.color),
sizeValue: normalizeOptionalText(picture.sizeValue),
combi: normalizeOptionalText(picture.combi),
sourcePageUrl,
sourceImageUrl: imageUrl,
sourcePosition: normalizePositiveNumber(picture.sourcePosition) || 0,
targetStrategy: normalizeTargetStrategy(picture.targetStrategy),
targetColor: normalizeOptionalText(picture.targetColor),
targetSize: normalizeOptionalText(picture.targetSize),
targetCombi: normalizeOptionalText(picture.targetCombi),
targetEan13: normalizeOptionalText(picture.targetEan13),
targetIdProductCatalog: normalizePositiveNumber(picture.targetIdProductCatalog),
matchConfidence: normalizeConfidence(picture.matchConfidence),
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
duplicateOfCandidateId: duplicate?.id || null,
});
await connection.commit();
return {
saved: true,
candidateId,
imageBlobId,
sha256,
duplicateStatus: duplicate ? "exact_duplicate" : "unique",
duplicateOfCandidateId: duplicate?.id || null,
mimeType: image.mimeType,
width: normalizePositiveNumber(picture.width) || null,
height: normalizePositiveNumber(picture.height) || null,
sizeBytes: image.buffer.byteLength,
sourceKey,
sourceName,
};
} catch (error) {
await connection.rollback();
throw error;
} finally {
await connection.end();
}
}
async function downloadImage(url) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 20000);
try {
const response = await fetch(url, {
signal: controller.signal,
headers: {
accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
},
});
if (!response.ok) {
throw new Error(`Image download failed with HTTP ${response.status}.`);
}
const mimeType = String(response.headers.get("content-type") || "application/octet-stream")
.split(";")[0]
.toLowerCase();
if (!mimeType.startsWith("image/")) {
throw new Error(`Downloaded content is not an image (${mimeType}).`);
}
const buffer = Buffer.from(await response.arrayBuffer());
if (!buffer.length) {
throw new Error("Downloaded image is empty.");
}
if (buffer.byteLength > 20 * 1024 * 1024) {
throw new Error("Downloaded image is larger than 20 MB.");
}
return { buffer, mimeType };
} finally {
clearTimeout(timeout);
}
}
async function upsertSource(connection, source) {
const baseUrl = new URL(source.imageUrl).origin;
const sourceType = ["supplier", "manufacturer", "backup", "other"].includes(source.sourceType)
? source.sourceType
: "supplier";
const result = await connection.query(
`
INSERT INTO scrape_sources (
source_key,
source_name,
source_type,
base_url,
enabled
)
VALUES (?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE
id = LAST_INSERT_ID(id),
source_name = VALUES(source_name),
base_url = VALUES(base_url),
enabled = VALUES(enabled)
`,
[source.sourceKey, source.sourceName, sourceType, baseUrl],
);
return Number(result.insertId);
}
async function upsertImageBlob(connection, image) {
const result = await connection.query(
`
INSERT INTO scrape_image_blobs (
sha256,
mime_type,
width,
height,
size_bytes,
image_data
)
VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
id = LAST_INSERT_ID(id),
mime_type = VALUES(mime_type),
width = COALESCE(VALUES(width), width),
height = COALESCE(VALUES(height), height),
size_bytes = VALUES(size_bytes)
`,
[image.sha256, image.mimeType, image.width, image.height, image.sizeBytes, image.buffer],
);
return Number(result.insertId);
}
async function findExistingImageCandidate(connection, imageBlobId, sourceId, imageUrl) {
const rows = await connection.query(
`
SELECT id
FROM scrape_picture_candidates
WHERE image_blob_id = ?
AND NOT (source_id = ? AND source_image_url = ?)
ORDER BY id ASC
LIMIT 1
`,
[imageBlobId, sourceId, imageUrl],
);
const items = normalizeRows(rows);
return items[0] || null;
}
async function upsertPictureCandidate(connection, candidate) {
const result = await connection.query(
`
INSERT INTO scrape_picture_candidates (
image_blob_id,
source_id,
manufacturer_id,
id_product_catalog,
id_product,
supplier_reference,
ean13,
color,
size_value,
combi,
source_page_url,
source_image_url,
source_position,
target_strategy,
target_color,
target_size,
target_combi,
target_ean13,
target_id_product_catalog,
match_confidence,
duplicate_status,
duplicate_of_candidate_id,
status
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft')
ON DUPLICATE KEY UPDATE
id = LAST_INSERT_ID(id),
image_blob_id = VALUES(image_blob_id),
manufacturer_id = VALUES(manufacturer_id),
id_product_catalog = VALUES(id_product_catalog),
id_product = VALUES(id_product),
supplier_reference = VALUES(supplier_reference),
ean13 = VALUES(ean13),
color = VALUES(color),
size_value = VALUES(size_value),
combi = VALUES(combi),
source_page_url = VALUES(source_page_url),
source_position = VALUES(source_position),
target_strategy = VALUES(target_strategy),
target_color = VALUES(target_color),
target_size = VALUES(target_size),
target_combi = VALUES(target_combi),
target_ean13 = VALUES(target_ean13),
target_id_product_catalog = VALUES(target_id_product_catalog),
match_confidence = VALUES(match_confidence),
duplicate_status = VALUES(duplicate_status),
duplicate_of_candidate_id = VALUES(duplicate_of_candidate_id),
updated_at = CURRENT_TIMESTAMP
`,
[
candidate.imageBlobId,
candidate.sourceId,
candidate.manufacturerId,
candidate.idProductCatalog,
candidate.idProduct,
candidate.supplierReference,
candidate.ean13,
candidate.color,
candidate.sizeValue,
candidate.combi,
candidate.sourcePageUrl,
candidate.sourceImageUrl,
candidate.sourcePosition,
candidate.targetStrategy,
candidate.targetColor,
candidate.targetSize,
candidate.targetCombi,
candidate.targetEan13,
candidate.targetIdProductCatalog,
candidate.matchConfidence,
candidate.duplicateStatus,
candidate.duplicateOfCandidateId,
],
);
return Number(result.insertId);
}
function normalizeRows(rows) {
return Array.isArray(rows) ? rows.filter((row) => row && typeof row === "object" && !("meta" in row)) : [];
}
function normalizeRequiredUrl(value, name) {
const text = normalizeText(value);
if (!text) throw new Error(`${name} is required.`);
const url = new URL(text);
if (!["http:", "https:"].includes(url.protocol)) {
throw new Error(`${name} must be an HTTP URL.`);
}
return url.toString();
}
function normalizeText(value) {
return String(value ?? "").trim();
}
function normalizeOptionalText(value) {
const text = normalizeText(value);
return text || null;
}
function normalizePositiveNumber(value) {
const number = Number(value);
return Number.isFinite(number) && number > 0 ? Math.trunc(number) : null;
}
function normalizeConfidence(value) {
const number = Number(value);
if (!Number.isFinite(number)) return 0;
return Math.max(0, Math.min(100, number));
}
function normalizeTargetStrategy(value) {
const strategy = normalizeText(value).toLowerCase();
return ["unknown", "color", "size", "combination", "manual"].includes(strategy)
? strategy
: "unknown";
}
+47 -1
View File
@@ -15,7 +15,10 @@ import {
loadProductByEan, loadProductByEan,
loadProductInfo, loadProductInfo,
getProductPictures, getProductPictures,
saveProductPictureCandidate,
openProductSource,
} from "./services/catalog-products.ts"; } from "./services/catalog-products.ts";
import { openInBrowser } from "./services/browser-adapter.ts";
const __dirname = path.dirname(fileURLToPath(import.meta.url)); const __dirname = path.dirname(fileURLToPath(import.meta.url));
const publicDir = path.resolve(__dirname, "../public"); const publicDir = path.resolve(__dirname, "../public");
@@ -122,6 +125,44 @@ const server = http.createServer(async (request, response) => {
return; return;
} }
if (url.pathname === "/api/catalog-maker/save-picture" && request.method === "POST") {
const body = await readJsonBody(request);
await sendJson(response, await saveProductPictureCandidate(requestConfig, body));
return;
}
if (url.pathname === "/api/catalog-maker/open-source") {
await sendJson(
response,
await openProductSource(requestConfig, {
manufacturerId: url.searchParams.get("manufacturerId"),
sourceKey: url.searchParams.get("sourceKey"),
url: url.searchParams.get("url"),
}),
);
return;
}
if (url.pathname === "/api/tools/open-adminer") {
await openInBrowser(config.databaseTool.url, { engine: config.databaseTool.browserEngine });
await sendJson(response, {
opened: true,
url: config.databaseTool.url,
browserEngine: config.databaseTool.browserEngine,
});
return;
}
if (url.pathname === "/api/tools/open-dbgate") {
await openInBrowser(config.dbGate.url, { engine: config.dbGate.browserEngine });
await sendJson(response, {
opened: true,
url: config.dbGate.url,
browserEngine: config.dbGate.browserEngine,
});
return;
}
if (url.pathname === "/api/catalog-maker/product-info") { if (url.pathname === "/api/catalog-maker/product-info") {
await sendJson( await sendJson(
response, response,
@@ -197,7 +238,9 @@ function getStatus() {
driver: config.database.driver, driver: config.database.driver,
mode: config.database.mode, mode: config.database.mode,
scrapeName: config.database.scrapeName, scrapeName: config.database.scrapeName,
allowWrites: config.database.driver === "mariadb" && config.database.mode === "local" && config.database.allowWrites, allowLocalWrites: config.database.driver === "mariadb"
&& config.database.mode === "local"
&& config.database.allowLocalWrites,
}, },
workbook: path.basename(config.sourceWorkbook), workbook: path.basename(config.sourceWorkbook),
workbookExists: true, workbookExists: true,
@@ -230,6 +273,9 @@ function getRequestConfig(request) {
database: { database: {
...config.database, ...config.database,
mode: selectedMode, mode: selectedMode,
allowLocalWrites: selectedMode === "local"
&& config.database.allowLocalWrites
&& request.headers["x-local-write-access"] === "1",
}, },
}; };
} }
+33 -6
View File
@@ -1,18 +1,19 @@
// @ts-nocheck // @ts-nocheck
import { chromium } from "playwright"; import fs from "node:fs";
import { chromium, firefox } from "playwright";
const visibleBrowsers = new Set(); const visibleBrowsers = new Set();
export async function openInBrowser(url) { export async function openInBrowser(url, { engine = "chromium" } = {}) {
const browser = await chromium.launch({ headless: false }); const browser = await launchBrowser(engine, false);
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 }); await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
visibleBrowsers.add(browser); visibleBrowsers.add(browser);
return { browser, page }; return { browser, page };
} }
export async function searchInBrowser({ url, headless = true, waitAfterLoadMs = 3000 }) { export async function searchInBrowser({ url, engine = "chromium", headless = true, waitAfterLoadMs = 3000 }) {
const browser = await chromium.launch({ headless }); const browser = await launchBrowser(engine, headless);
const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } });
try { try {
@@ -44,7 +45,33 @@ export async function searchInBrowser({ url, headless = true, waitAfterLoadMs =
} }
export async function scrapePage(url, options = {}) { export async function scrapePage(url, options = {}) {
return searchInBrowser({ url, headless: true, waitAfterLoadMs: 750, ...options }); return searchInBrowser({ url, engine: "chromium", headless: true, waitAfterLoadMs: 750, ...options });
}
function launchBrowser(engine, headless) {
const normalizedEngine = String(engine).toLowerCase() === "firefox" ? "firefox" : "chromium";
const browserType = normalizedEngine === "firefox" ? firefox : chromium;
const executablePath = resolveSystemBrowser(normalizedEngine);
const options = {
headless,
timeout: 10_000,
...(executablePath ? { executablePath } : {}),
};
return browserType.launch(options);
}
function resolveSystemBrowser(engine) {
if (process.platform !== "win32") return "";
if (engine === "firefox") return "";
const candidates = [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
];
return candidates.find((candidate) => fs.existsSync(candidate)) || "";
} }
export async function closeVisibleBrowsers() { export async function closeVisibleBrowsers() {
+211 -32
View File
@@ -1,5 +1,6 @@
// @ts-nocheck // @ts-nocheck
import { executeEndpointWrite, queryEndpoint } from "../endpoint/endpoint-client.ts"; import { executeEndpointWrite, queryEndpoint } from "../endpoint/endpoint-client.ts";
import { saveScrapePictureCandidate } from "../db/scrape-image-store.ts";
import { openInBrowser, scrapePage, searchInBrowser } from "./browser-adapter.ts"; import { openInBrowser, scrapePage, searchInBrowser } from "./browser-adapter.ts";
import { import {
prepareManufacturerCatalogSqls, prepareManufacturerCatalogSqls,
@@ -13,6 +14,7 @@ import {
export async function loadManufacturerCatalogState(config, { manufacturerId, position = 1 }) { export async function loadManufacturerCatalogState(config, { manufacturerId, position = 1 }) {
const refreshResults = []; const refreshResults = [];
if (config.database?.mode === "local" && config.database.allowLocalWrites) {
for (const sql of prepareManufacturerCatalogSqls({ manufacturerId })) { for (const sql of prepareManufacturerCatalogSqls({ manufacturerId })) {
refreshResults.push( refreshResults.push(
await executeEndpointWrite(config, sql, { await executeEndpointWrite(config, sql, {
@@ -20,6 +22,7 @@ export async function loadManufacturerCatalogState(config, { manufacturerId, pos
}), }),
); );
} }
}
const productResult = await queryEndpoint( const productResult = await queryEndpoint(
config, config,
@@ -49,6 +52,7 @@ export async function loadManufacturerCatalogState(config, { manufacturerId, pos
refresh: { refresh: {
operation: "catalog-maker-refresh-manufacturer", operation: "catalog-maker-refresh-manufacturer",
statements: refreshResults.length, statements: refreshResults.length,
skipped: refreshResults.length === 0,
}, },
product: normalizeProduct(product), product: normalizeProduct(product),
combinations: combinationsResult.items.map(normalizeCombination), combinations: combinationsResult.items.map(normalizeCombination),
@@ -126,16 +130,23 @@ export async function listSuggestedProducts(
export async function listMappingSources(config, { manufacturerId, checkUrls = false }) { export async function listMappingSources(config, { manufacturerId, checkUrls = false }) {
try { try {
const result = await queryEndpoint(config, mappingSourcesSql({ manufacturerId })); const result = await queryEndpoint(
config,
mappingSourcesSql({
manufacturerId,
scrapeDatabase: config.database?.scrapeName || "catalog_scrape",
}),
);
const mappingTableName = `${config.database?.scrapeName || "catalog_scrape"}.ps_product_catalog_manufacturer_source`;
const items = result.items.map((row) => const items = result.items.map((row) =>
normalizeMappingSource(row, "ps_product_catalog_manufacturer_source"), normalizeMappingSource(row, mappingTableName),
); );
const checkedItems = checkUrls ? await checkMappingSourceUrls(items) : items; const checkedItems = checkUrls ? await checkMappingSourceUrls(items) : items;
return { return {
configured: result.configured, configured: result.configured,
mapped: checkedItems.some((item) => item.enabled), mapped: checkedItems.some((item) => item.enabled),
tableName: "ps_product_catalog_manufacturer_source", tableName: mappingTableName,
items: checkedItems, items: checkedItems,
sourceCount: checkedItems.filter((item) => item.enabled).length, sourceCount: checkedItems.filter((item) => item.enabled).length,
testedCount: checkedItems.filter((item) => item.enabled && item.health === "ok").length, testedCount: checkedItems.filter((item) => item.enabled && item.health === "ok").length,
@@ -177,18 +188,6 @@ export async function findProductSources(
), ),
); );
for (const item of items) {
if (item.source?.key !== "hudy") continue;
const candidate = item.candidates?.[0];
if (candidate?.url) {
try {
await openInBrowser(candidate.url);
} catch {
// Picture extraction can continue even if the visible browser cannot start.
}
}
}
return { return {
configured: mapping.configured, configured: mapping.configured,
items, items,
@@ -237,6 +236,35 @@ export async function getProductPictures(
}; };
} }
export async function saveProductPictureCandidate(config, picture) {
return saveScrapePictureCandidate(config, picture);
}
export async function openProductSource(config, { manufacturerId, sourceKey, url }) {
const sourceUrl = normalizeSourceUrl(url);
const mapping = await listMappingSources(config, { manufacturerId });
const source = mapping.items.find(
(item) => item.enabled && String(item.key ?? "").toLowerCase() === String(sourceKey ?? "").toLowerCase(),
);
if (!source) {
throw new Error("Source mapping was not found for selected manufacturer.");
}
await openInBrowser(sourceUrl, { engine: source.browserEngine });
return {
configured: mapping.configured,
opened: true,
source: {
key: source.key,
name: source.name,
browserEngine: source.browserEngine,
},
url: sourceUrl,
};
}
async function findSourceCandidates(source, product, { includePictures = false } = {}) { async function findSourceCandidates(source, product, { includePictures = false } = {}) {
const searchAttempts = buildAdapterSearchUrls(source, product); const searchAttempts = buildAdapterSearchUrls(source, product);
let lastSearchUrl = searchAttempts[0]?.url || normalizeSourceUrl(source.url); let lastSearchUrl = searchAttempts[0]?.url || normalizeSourceUrl(source.url);
@@ -258,7 +286,8 @@ async function findSourceCandidates(source, product, { includePictures = false }
try { try {
const result = await searchInBrowser({ const result = await searchInBrowser({
url: attempt.url, url: attempt.url,
headless: source.browserHeadless, engine: source.browserEngine,
headless: true,
waitAfterLoadMs: source.waitAfterLoadMs, waitAfterLoadMs: source.waitAfterLoadMs,
}); });
@@ -269,7 +298,11 @@ async function findSourceCandidates(source, product, { includePictures = false }
attempt, attempt,
product, product,
); );
if (includePictures) candidate.pictures = extractPictureUrls(result.pageSource, result.url || attempt.url); if (includePictures) {
candidate.pictures = source.key === "idealo"
? extractIdealoGalleryPictureUrls(result.pageSource, result.url || attempt.url)
: extractPictureUrls(result.pageSource, result.url || attempt.url);
}
return { return {
source, source,
query: attempt.query, query: attempt.query,
@@ -288,12 +321,6 @@ async function findSourceCandidates(source, product, { includePictures = false }
candidates: buildManualSearchCandidates(searchAttempts, product), candidates: buildManualSearchCandidates(searchAttempts, product),
}; };
} catch (error) { } catch (error) {
try {
await openInBrowser(attempt.url);
} catch {
// The manual candidate below remains available even if the browser cannot start.
}
return { return {
source, source,
query: attempt.query, query: attempt.query,
@@ -309,6 +336,8 @@ async function findSourceCandidates(source, product, { includePictures = false }
lastSearchUrl = attempt.url; lastSearchUrl = attempt.url;
try { try {
const response = await scrapePage(attempt.url, { const response = await scrapePage(attempt.url, {
engine: source.browserEngine,
headless: true,
waitAfterLoadMs: source.waitAfterLoadMs, waitAfterLoadMs: source.waitAfterLoadMs,
}); });
const html = response.pageSource; const html = response.pageSource;
@@ -327,7 +356,7 @@ async function findSourceCandidates(source, product, { includePictures = false }
if (candidates.length) { if (candidates.length) {
let verification = null; let verification = null;
if (includePictures && source.key === "hudy" && product.color && candidates[0].url) { if (includePictures && source.key === "hudy" && product.color && candidates[0].url) {
verification = await verifyHudyVariant(candidates[0].url, product); verification = await verifyHudyVariant(candidates[0].url, product, source.browserEngine);
candidates[0].verification = verification; candidates[0].verification = verification;
if (verification.matchedUrl) candidates[0].url = verification.matchedUrl; if (verification.matchedUrl) candidates[0].url = verification.matchedUrl;
if (includePictures && verification.pictures?.length) { if (includePictures && verification.pictures?.length) {
@@ -338,7 +367,9 @@ async function findSourceCandidates(source, product, { includePictures = false }
if (includePictures) { if (includePictures) {
for (const candidate of candidates) { for (const candidate of candidates) {
if (source.key === "hudy" && candidate.url) { if (source.key === "hudy" && candidate.url) {
candidate.pictures = await extractPicturesFromPage(candidate.url); candidate.pictures = await extractPicturesFromPage(candidate.url, source.browserEngine);
} else if (source.key === "idealo") {
candidate.pictures = extractIdealoGalleryPictureUrls(html, response.url || attempt.url);
} else { } else {
candidate.pictures = extractPictureUrls(html, response.url || attempt.url); candidate.pictures = extractPictureUrls(html, response.url || attempt.url);
} }
@@ -580,7 +611,7 @@ function findColorVariantUrl(html, pageUrl, color) {
return ""; return "";
} }
async function verifyHudyVariant(url, product) { async function verifyHudyVariant(url, product, browserEngine = "chromium") {
try { try {
const eanGroup = Array.isArray(product.eanGroup) const eanGroup = Array.isArray(product.eanGroup)
? product.eanGroup ? product.eanGroup
@@ -598,7 +629,7 @@ async function verifyHudyVariant(url, product) {
if (visited.has(page.url)) continue; if (visited.has(page.url)) continue;
visited.add(page.url); visited.add(page.url);
const response = await scrapePage(page.url, { waitAfterLoadMs: 750 }); const response = await scrapePage(page.url, { engine: browserEngine, waitAfterLoadMs: 750 });
const html = response.pageSource; const html = response.pageSource;
lastStatus = response.status; lastStatus = response.status;
const pageEans = extractPageEans(html); const pageEans = extractPageEans(html);
@@ -700,9 +731,111 @@ function extractPictureUrls(html, pageUrl) {
return urls.slice(0, 12); return urls.slice(0, 12);
} }
async function extractPicturesFromPage(url) { export function extractIdealoGalleryPictureUrls(html, pageUrl) {
const source = String(html ?? "");
const galleryMatches = [
...source.matchAll(/<li\b[^>]*class=["'][^"']*splide__slide[^"']*["'][^>]*>[\s\S]*?<\/li>/gi),
];
const orderedGalleryUrls = [];
for (const match of galleryMatches) {
const best = pickBestImageUrlFromMarkup(match[0], pageUrl);
if (best && !orderedGalleryUrls.includes(best)) orderedGalleryUrls.push(best);
}
if (orderedGalleryUrls.length) return orderedGalleryUrls.slice(0, 12);
const trackMatch = source.match(/<div\b[^>]*class=["'][^"']*splide__track[^"']*["'][^>]*>[\s\S]*?<\/div>/i);
if (trackMatch) {
const best = pickBestImageUrlFromMarkup(trackMatch[0], pageUrl);
if (best) return [best];
}
return [];
}
function pickBestImageUrlFromMarkup(markup, pageUrl) {
const candidates = [];
const add = (value, descriptorScore = 0) => {
const normalized = normalizeImageCandidateUrl(value, pageUrl);
if (!normalized) return;
candidates.push({
url: normalized,
score: scoreImageCandidate(normalized, descriptorScore),
});
};
for (const attribute of ["srcset", "data-srcset"]) {
for (const match of markup.matchAll(new RegExp(`${attribute}=["']([^"']+)`, "gi"))) {
for (const candidate of parseSrcsetCandidates(match[1], pageUrl)) {
add(candidate.url, candidate.score);
}
}
}
for (const attribute of ["data-large", "data-original", "data-full", "data-zoom-image", "data-src", "src"]) {
for (const match of markup.matchAll(new RegExp(`${attribute}=["']([^"']+)`, "gi"))) {
add(match[1]);
}
}
candidates.sort((a, b) => b.score - a.score);
return candidates[0]?.url || "";
}
function parseSrcsetCandidates(value, pageUrl) {
return String(value ?? "")
.split(",")
.map((part) => {
const [urlPart, descriptor = ""] = part.trim().split(/\s+/);
const descriptorScore = /(\d+)w/i.test(descriptor)
? Number(descriptor.match(/(\d+)w/i)?.[1] || 0)
: /(\d+(?:\.\d+)?)x/i.test(descriptor)
? Number(descriptor.match(/(\d+(?:\.\d+)?)x/i)?.[1] || 0) * 400
: 0;
return {
url: normalizeImageCandidateUrl(urlPart, pageUrl),
score: descriptorScore,
};
})
.filter((candidate) => candidate.url);
}
function normalizeImageCandidateUrl(value, pageUrl) {
const decoded = decodeHtmlEntities(String(value ?? "").trim());
if (!decoded || decoded.startsWith("data:") || /\.svg(?:[?#]|$)/i.test(decoded)) return "";
try { try {
const response = await scrapePage(url, { waitAfterLoadMs: 750 }); const url = new URL(decoded.startsWith("//") ? `https:${decoded}` : decoded, pageUrl).href;
if (!/^https?:/i.test(url)) return "";
if (/(?:logo|icon|sprite|favicon|placeholder)/i.test(url)) return "";
return url;
} catch {
return "";
}
}
function scoreImageCandidate(url, descriptorScore = 0) {
const dimensionScore = Math.max(
...[...String(url).matchAll(/(?:^|[^0-9])(\d{2,5})[x_-](\d{2,5})(?:[^0-9]|$)/g)].map(
(match) => Number(match[1]) * Number(match[2]),
),
...[...String(url).matchAll(/w(\d{2,5})h(\d{2,5})/gi)].map(
(match) => Number(match[1]) * Number(match[2]),
),
0,
);
const largePathBonus = /(?:original|large|zoom|full|1000|1200|1500|2000)/i.test(url) ? 100000 : 0;
const thumbnailPenalty = /(?:thumb|thumbnail|small|preview|\/s\d+_|_s\d+|150x150|80x80)/i.test(url)
? -100000
: 0;
return dimensionScore + descriptorScore + largePathBonus + thumbnailPenalty;
}
async function extractPicturesFromPage(url, browserEngine = "chromium") {
try {
const response = await scrapePage(url, { engine: browserEngine, waitAfterLoadMs: 750 });
return extractHudyGalleryPictureUrls(response.pageSource, response.url || url); return extractHudyGalleryPictureUrls(response.pageSource, response.url || url);
} catch { } catch {
return []; return [];
@@ -855,16 +988,20 @@ async function checkMappingSourceUrls(items) {
return { return {
...item, ...item,
...(await checkUrl(item.url)), ...(await checkUrl(item.url, item.browserEngine, item.browserHeadless)),
}; };
}), }),
); );
} }
async function checkUrl(url) { async function checkUrl(url, browserEngine = "chromium", headless = true) {
try { try {
const target = normalizeSourceUrl(url); const target = normalizeSourceUrl(url);
const response = await scrapePage(target, { waitAfterLoadMs: 250 }); const response = await scrapePage(target, {
engine: browserEngine,
headless,
waitAfterLoadMs: 250,
});
return { return {
health: response.ok ? "ok" : "warning", health: response.ok ? "ok" : "warning",
@@ -940,6 +1077,8 @@ function normalizeMappingSource(row, tableName) {
browserEngine: String(row.browser_engine ?? "default").trim().toLowerCase(), browserEngine: String(row.browser_engine ?? "default").trim().toLowerCase(),
browserHeadless: normalizeEnabled(row.browser_headless ?? 0), browserHeadless: normalizeEnabled(row.browser_headless ?? 0),
waitAfterLoadMs: Number(row.wait_after_load_ms) || 3000, waitAfterLoadMs: Number(row.wait_after_load_ms) || 3000,
scraperConfig: parseJsonValue(row.scraper_config, {}),
capabilities: parseCapabilities(row.capabilities),
enabled: normalizeEnabled(row.mapping_enabled ?? row.enabled ?? row.active ?? 1) enabled: normalizeEnabled(row.mapping_enabled ?? row.enabled ?? row.active ?? 1)
&& normalizeEnabled(row.enabled ?? 1), && normalizeEnabled(row.enabled ?? 1),
note: row.note ?? row.description ?? "", note: row.note ?? row.description ?? "",
@@ -947,6 +1086,46 @@ function normalizeMappingSource(row, tableName) {
}; };
} }
function parseCapabilities(value) {
const parsed = parseJsonValue(value, []);
if (!Array.isArray(parsed)) return [];
return parsed
.filter(Boolean)
.filter((item) => item.capability)
.map((item) => ({
capability: item.capability,
enabled: normalizeEnabled(item.enabled ?? 1),
priority: Number(item.priority) || 100,
parserKey: item.parserKey ?? "",
parserModule: item.parserModule ?? "",
browserEngine: String(item.browserEngine ?? "").trim().toLowerCase(),
browserHeadless: item.browserHeadless === null ? null : normalizeEnabled(item.browserHeadless ?? 0),
waitAfterLoadMs: item.waitAfterLoadMs === null ? null : Number(item.waitAfterLoadMs) || null,
minWidth: item.minWidth === null ? null : Number(item.minWidth) || null,
minHeight: item.minHeight === null ? null : Number(item.minHeight) || null,
minLongestSide: item.minLongestSide === null ? null : Number(item.minLongestSide) || null,
defaultTargetStrategy: item.defaultTargetStrategy ?? "unknown",
targetStrategyOverride: item.targetStrategyOverride || "",
selectorConfig: parseJsonValue(item.selectorConfig, {}),
extractionConfig: parseJsonValue(item.extractionConfig, {}),
qualityConfig: parseJsonValue(item.qualityConfig, {}),
configOverride: parseJsonValue(item.configOverride, {}),
note: item.note ?? "",
}));
}
function parseJsonValue(value, fallback) {
if (value === null || value === undefined || value === "") return fallback;
if (typeof value === "object" && !Buffer.isBuffer(value)) return value;
try {
return JSON.parse(Buffer.isBuffer(value) ? value.toString("utf8") : String(value));
} catch {
return fallback;
}
}
function parseList(value) { function parseList(value) {
return String(value ?? "") return String(value ?? "")
.split(",") .split(",")
+98 -26
View File
@@ -93,13 +93,9 @@ export function productToDoByManufacturerSql({ manufacturerId, position = 1 }) {
tr.total, tr.total,
p.id_product, p.id_product,
p.price, p.price,
p.id_supplier,
( (
SELECT CONCAT('https://www.9b-plus.com/', i.id_image, '-home_default/', pl.link_rewrite, '.jpg') SELECT ${prestaImageUrl("i.id_image", "home_default")}
FROM ps_image i FROM ps_image i
JOIN ps_product_lang pl
ON pl.id_product = i.id_product
AND pl.id_lang = 2
WHERE i.id_product = p.id_product WHERE i.id_product = p.id_product
ORDER BY i.cover DESC, i.position ASC ORDER BY i.cover DESC, i.position ASC
LIMIT 1 LIMIT 1
@@ -155,13 +151,9 @@ export function productByEanSql({ manufacturerId, ean }) {
tr.total, tr.total,
p.id_product, p.id_product,
p.price, p.price,
p.id_supplier,
( (
SELECT CONCAT('https://www.9b-plus.com/', i.id_image, '-home_default/', pl.link_rewrite, '.jpg') SELECT ${prestaImageUrl("i.id_image", "home_default")}
FROM ps_image i FROM ps_image i
JOIN ps_product_lang pl
ON pl.id_product = i.id_product
AND pl.id_lang = 2
WHERE i.id_product = p.id_product WHERE i.id_product = p.id_product
ORDER BY i.cover DESC, i.position ASC ORDER BY i.cover DESC, i.position ASC
LIMIT 1 LIMIT 1
@@ -347,11 +339,8 @@ export function suggestedProductsSql({ manufacturerId, supplierReference, produc
p.supplier_reference, p.supplier_reference,
p.active, p.active,
( (
SELECT CONCAT('https://www.9b-plus.com/', i.id_image, '-cart_default/', pli.link_rewrite, '.jpg') SELECT ${prestaImageUrl("i.id_image", "cart_default")}
FROM ps_image i FROM ps_image i
JOIN ps_product_lang pli
ON pli.id_product = i.id_product
AND pli.id_lang = 2
WHERE i.id_product = p.id_product WHERE i.id_product = p.id_product
ORDER BY i.cover DESC, i.position ASC ORDER BY i.cover DESC, i.position ASC
LIMIT 1 LIMIT 1
@@ -373,25 +362,30 @@ export function mappingSourceTablesSql() {
FROM information_schema.tables FROM information_schema.tables
WHERE table_schema = DATABASE() WHERE table_schema = DATABASE()
AND table_name IN ( AND table_name IN (
'ps_product_catalog_source', 'scrape_sources',
'ps_product_catalog_manufacturer_source' 'scrape_manufacturer_sources',
'scrape_source_capabilities',
'scrape_manufacturer_source_capabilities'
) )
ORDER BY FIELD( ORDER BY FIELD(
table_name, table_name,
'ps_product_catalog_manufacturer_source', 'scrape_manufacturer_sources',
'ps_product_catalog_source' 'scrape_sources',
'scrape_manufacturer_source_capabilities',
'scrape_source_capabilities'
) )
`; `;
} }
export function mappingSourcesSql({ manufacturerId }) { export function mappingSourcesSql({ manufacturerId, scrapeDatabase = "catalog_scrape" }) {
const safeManufacturerId = parsePositiveInteger(manufacturerId, "manufacturerId"); const safeManufacturerId = parsePositiveInteger(manufacturerId, "manufacturerId");
const scrapeSchema = escapeSqlIdentifier(scrapeDatabase);
return ` return `
SELECT SELECT
ms.id_product_catalog_manufacturer_source AS mapping_id, ms.id AS mapping_id,
ms.id_manufacturer, ms.manufacturer_id AS id_manufacturer,
ms.id_product_catalog_source, s.id AS id_product_catalog_source,
ms.enabled AS mapping_enabled, ms.enabled AS mapping_enabled,
ms.priority AS mapping_priority, ms.priority AS mapping_priority,
s.source_key, s.source_key,
@@ -410,11 +404,67 @@ export function mappingSourcesSql({ manufacturerId }) {
s.browser_engine, s.browser_engine,
s.browser_headless, s.browser_headless,
s.wait_after_load_ms, s.wait_after_load_ms,
s.scraper_config,
s.note,
JSON_ARRAYAGG(
CASE
WHEN sc.id IS NULL THEN NULL
ELSE JSON_OBJECT(
'capability', sc.capability,
'enabled', COALESCE(msc.enabled, sc.enabled),
'priority', COALESCE(msc.priority, sc.priority),
'parserKey', sc.parser_key,
'parserModule', sc.parser_module,
'browserEngine', COALESCE(sc.browser_engine, ''),
'browserHeadless', sc.browser_headless,
'waitAfterLoadMs', sc.wait_after_load_ms,
'minWidth', sc.min_width,
'minHeight', sc.min_height,
'minLongestSide', sc.min_longest_side,
'defaultTargetStrategy', sc.default_target_strategy,
'targetStrategyOverride', COALESCE(msc.target_strategy_override, ''),
'selectorConfig', sc.selector_config,
'extractionConfig', sc.extraction_config,
'qualityConfig', sc.quality_config,
'configOverride', msc.config_override,
'note', sc.note
)
END
) AS capabilities
FROM ${scrapeSchema}.scrape_manufacturer_sources ms
JOIN ${scrapeSchema}.scrape_sources s
ON s.id = ms.source_id
LEFT JOIN ${scrapeSchema}.scrape_source_capabilities sc
ON sc.source_id = s.id
AND sc.enabled = 1
LEFT JOIN ${scrapeSchema}.scrape_manufacturer_source_capabilities msc
ON msc.source_capability_id = sc.id
AND msc.manufacturer_source_id = ms.id
WHERE ms.manufacturer_id = ${safeManufacturerId}
GROUP BY
ms.id,
ms.manufacturer_id,
s.id,
ms.enabled,
ms.priority,
s.source_key,
s.source_name,
s.source_type,
s.base_url,
s.enabled,
s.priority,
s.search_primary_key,
s.search_fallback_keys,
s.fallback_source_type,
s.search_url_template,
s.fallback_url_template,
s.search_context_keys,
s.access_mode,
s.browser_engine,
s.browser_headless,
s.wait_after_load_ms,
s.scraper_config,
s.note s.note
FROM ps_product_catalog_manufacturer_source ms
JOIN ps_product_catalog_source s
ON s.id_product_catalog_source = ms.id_product_catalog_source
WHERE ms.id_manufacturer = ${safeManufacturerId}
ORDER BY ORDER BY
ms.priority ASC, ms.priority ASC,
s.priority ASC, s.priority ASC,
@@ -422,6 +472,28 @@ export function mappingSourcesSql({ manufacturerId }) {
`; `;
} }
function escapeSqlIdentifier(value) {
const text = String(value ?? "").trim();
if (!/^[a-zA-Z0-9_]+$/.test(text)) {
throw new Error("SQL identifier contains unsupported characters.");
}
return `\`${text}\``;
}
function prestaImageUrl(idExpression, size) {
const safeSize = String(size ?? "").trim();
if (!/^[a-zA-Z0-9_]+$/.test(safeSize)) {
throw new Error("PrestaShop image size contains unsupported characters.");
}
const idText = `CAST(${idExpression} AS CHAR)`;
const pathParts = Array.from({ length: 10 }, (_, index) =>
`IF(CHAR_LENGTH(${idText}) > ${index}, CONCAT(SUBSTRING(${idText}, ${index + 1}, 1), '/'), '')`,
).join(", ");
return `CONCAT('https://www.9b-plus.com/img/p/', ${pathParts}, ${idExpression}, '-${safeSize}.jpg')`;
}
function escapeSqlString(value) { function escapeSqlString(value) {
return String(value ?? "").replaceAll("\\", "\\\\").replaceAll("'", "''").trim(); return String(value ?? "").replaceAll("\\", "\\\\").replaceAll("'", "''").trim();
} }
+73 -25
View File
@@ -58,10 +58,10 @@
.sidebar-drawer[aria-hidden="true"] { @apply -translate-x-full; } .sidebar-drawer[aria-hidden="true"] { @apply -translate-x-full; }
.sidebar-section { @apply border-t border-slate-700 pt-1; } .sidebar-section { @apply border-t border-slate-700 pt-1; }
.drawer-trigger { @apply relative flex min-h-8 w-full items-center justify-center rounded border border-slate-700 bg-slate-800 px-8 text-center text-[11px] font-semibold text-slate-100 transition hover:border-slate-500 hover:bg-slate-700; } .drawer-trigger { @apply relative flex min-h-8 w-full items-center justify-center rounded border border-slate-700 bg-slate-800 px-8 text-center text-[11px] font-semibold text-slate-100 transition hover:border-slate-500 hover:bg-slate-700; }
.drawer-trigger .sidebar-icon { @apply absolute left-2; } .drawer-trigger .sidebar-icon { @apply static; }
.drawer-trigger > span:not(.drawer-chevron) { @apply min-w-0 truncate text-center; } .drawer-trigger > span:not(.drawer-chevron) { @apply min-w-0 truncate text-center; }
.drawer-trigger .drawer-chevron { @apply inline-flex h-2 w-2 border-b-2 border-r-2 text-[0px] leading-none; } .drawer-trigger .drawer-chevron { @apply absolute right-2 inline-flex h-2 w-2 border-b-2 border-r-2 text-[0px] leading-none; }
.drawer-trigger .drawer-chevron { @apply ml-auto rotate-45 text-slate-300; } .drawer-trigger .drawer-chevron { @apply rotate-45 text-slate-300; }
.drawer-trigger[aria-expanded="true"] .drawer-chevron { @apply rotate-[225deg]; } .drawer-trigger[aria-expanded="true"] .drawer-chevron { @apply rotate-[225deg]; }
.parser-drawer[hidden] { display: none; } .parser-drawer[hidden] { display: none; }
.sidebar-icon { @apply inline-flex h-4 w-4 shrink-0 items-center justify-center text-sm leading-none text-slate-300; } .sidebar-icon { @apply inline-flex h-4 w-4 shrink-0 items-center justify-center text-sm leading-none text-slate-300; }
@@ -71,6 +71,11 @@
.action-button .sidebar-icon { @apply text-slate-400; } .action-button .sidebar-icon { @apply text-slate-400; }
.sidebar-footer { @apply mt-auto grid gap-1 border-t border-slate-700 pt-2; } .sidebar-footer { @apply mt-auto grid gap-1 border-t border-slate-700 pt-2; }
.app-status { @apply rounded border border-slate-700 bg-slate-950 p-1 text-[10px] leading-tight text-slate-100; } .app-status { @apply rounded border border-slate-700 bg-slate-950 p-1 text-[10px] leading-tight text-slate-100; }
.database-status-content { @apply flex items-center justify-center gap-1.5 text-center; }
.database-state-dot { @apply inline-block h-2.5 w-2.5 shrink-0 rounded-full border border-slate-500; }
.database-state-dot.is-ready { @apply border-emerald-300 bg-emerald-500; }
.database-state-dot.is-pending { @apply border-amber-300 bg-amber-400; }
.database-state-dot.is-error { @apply border-red-300 bg-red-500; }
.safe-mode, #modePill { @apply inline-block rounded bg-slate-100 px-1 py-0.5 text-[10px] font-semibold text-slate-800; } .safe-mode, #modePill { @apply inline-block rounded bg-slate-100 px-1 py-0.5 text-[10px] font-semibold text-slate-800; }
#databaseStatus.database-error { @apply bg-red-100 text-red-700; } #databaseStatus.database-error { @apply bg-red-100 text-red-700; }
.record-id { @apply flex h-10 items-center justify-center rounded border border-lime-700 bg-catalog-accent text-[26px] font-bold leading-none text-catalog-accent-text; } .record-id { @apply flex h-10 items-center justify-center rounded border border-lime-700 bg-catalog-accent text-[26px] font-bold leading-none text-catalog-accent-text; }
@@ -90,13 +95,15 @@
.arrow-button:disabled, .action-button:disabled { @apply cursor-not-allowed opacity-50; } .arrow-button:disabled, .action-button:disabled { @apply cursor-not-allowed opacity-50; }
.input-like { @apply h-7 px-1 text-[11px]; } .input-like { @apply h-7 px-1 text-[11px]; }
.select-control { @apply relative w-full; } .select-control { @apply relative w-full; }
.select-control .input-like { @apply appearance-none pr-7 pl-9 text-center; } .select-display { @apply pointer-events-none absolute inset-0 z-30 flex items-center justify-center gap-1.5 text-center text-slate-200; }
.select-icon { @apply pointer-events-none absolute left-3 top-1/2 z-10 h-4 w-4 -translate-y-1/2 text-slate-300; } .select-display span { @apply min-w-0 max-w-[calc(100%-3rem)] truncate; }
.select-chevron { @apply pointer-events-none absolute right-3 top-1/2 h-2 w-2 -translate-y-1/2 rotate-45 border-b-2 border-r-2 text-slate-200; } .language-native-select { @apply relative z-20 cursor-pointer appearance-none bg-transparent pr-7 pl-7 !text-transparent caret-transparent; }
.language-native-select:focus { @apply !text-transparent; }
.select-chevron { @apply pointer-events-none absolute right-3 top-1/2 z-30 h-2 w-2 -translate-y-1/2 rotate-45 border-b-2 border-r-2 text-slate-200; }
.native-manufacturer-host { @apply hidden; }
.native-manufacturer-select { @apply hidden; } .native-manufacturer-select { @apply hidden; }
.manufacturer-picker { @apply relative; } .manufacturer-picker { @apply relative; }
.manufacturer-trigger { @apply relative flex h-7 w-full items-center justify-center rounded border border-slate-700 bg-slate-800 px-9 text-center text-[11px] text-slate-200 shadow-sm; } .manufacturer-trigger { @apply relative flex h-7 w-full items-center justify-center gap-1.5 rounded border border-slate-700 bg-slate-800 px-9 text-center text-[11px] text-slate-200 shadow-sm; }
.manufacturer-trigger .select-icon { @apply left-3; }
.manufacturer-trigger > span:not(.select-chevron) { @apply min-w-0 truncate text-center; } .manufacturer-trigger > span:not(.select-chevron) { @apply min-w-0 truncate text-center; }
.manufacturer-trigger:hover { @apply border-slate-500 bg-slate-700; } .manufacturer-trigger:hover { @apply border-slate-500 bg-slate-700; }
.manufacturer-trigger[aria-expanded="true"] { @apply border-catalog-focus ring-2 ring-catalog-focus/25; } .manufacturer-trigger[aria-expanded="true"] { @apply border-catalog-focus ring-2 ring-catalog-focus/25; }
@@ -112,14 +119,24 @@
.input-like:focus { @apply border-catalog-focus outline-none ring-2 ring-catalog-focus/25; } .input-like:focus { @apply border-catalog-focus outline-none ring-2 ring-catalog-focus/25; }
.picture-slot { @apply aspect-square w-full overflow-hidden rounded border border-dashed border-slate-500 bg-catalog-panel-soft p-1 text-center text-[11px] text-slate-200; } .picture-slot { @apply aspect-square w-full overflow-hidden rounded border border-dashed border-slate-500 bg-catalog-panel-soft p-1 text-center text-[11px] text-slate-200; }
.picture-slot img { @apply h-full w-full object-contain; } .picture-slot img { @apply h-full w-full object-contain; }
.mapping-box { @apply rounded border border-slate-500 bg-catalog-panel-soft; } .mapping-box { @apply relative rounded border border-slate-500 bg-catalog-panel-soft; }
.mapping-details summary { @apply grid cursor-pointer grid-cols-[minmax(0,1fr)_22px] items-center px-1 py-1 list-none; } .sidebar-mapping-box { @apply overflow-hidden border-slate-700 bg-slate-900; }
.sidebar-mapping-box { @apply overflow-visible; }
.mapping-details { @apply relative; }
.mapping-summary { @apply relative flex min-h-8 items-center justify-center rounded bg-slate-800 px-9 text-center text-[11px] font-semibold text-slate-100; }
.mapping-label { @apply absolute left-1/2 top-1/2 inline-flex max-w-[calc(100%-4.5rem)] -translate-x-1/2 -translate-y-1/2 items-center justify-center gap-1.5 text-center; }
.mapping-title { @apply min-w-0 truncate text-center; }
.mapping-state-dot { @apply inline-block h-3 w-3 shrink-0 rounded-full border border-slate-500 shadow-sm; }
.mapping-state-dot.is-ready { @apply border-emerald-300 bg-emerald-500; }
.mapping-state-dot.is-warning { @apply border-amber-300 bg-amber-400; }
.mapping-state-dot.is-missing { @apply border-red-300 bg-red-500; }
.mapping-state-dot.is-loading { @apply border-slate-300 bg-slate-400; }
.mapping-panel { @apply absolute left-0 right-0 top-full z-50 border border-t-0 border-slate-700 bg-slate-950/95 shadow-lg; }
.mapping-detail-status { @apply truncate px-2 py-1 text-center text-[10px] font-normal text-slate-300; }
.mapping-summary .mapping-toggle { @apply absolute right-2 top-1/2 flex h-4 w-4 -translate-y-1/2 items-center justify-center; }
.mapping-details summary { @apply cursor-pointer list-none; }
.mapping-details summary::-webkit-details-marker { display: none; } .mapping-details summary::-webkit-details-marker { display: none; }
.mapping-toggle::before { content: ""; @apply inline-block text-center text-lg leading-none text-slate-100; } .mapping-toggle::before { content: ""; @apply block h-2 w-2 rotate-45 border-b-2 border-r-2 border-slate-100; }
.mapping-details[open] .mapping-toggle::before { content: "⌄"; }
.mapping-state { @apply block truncate rounded px-1 py-0.5 text-[10px] font-normal; }
.mapping-state.is-ready { @apply bg-slate-100 text-slate-800; }
.mapping-state.is-missing { @apply bg-red-100 text-red-700; }
.mapping-list { @apply grid gap-0.5 px-1 pb-1; } .mapping-list { @apply grid gap-0.5 px-1 pb-1; }
.mapping-source-row { @apply grid grid-cols-[minmax(0,1fr)_auto] items-center gap-1 rounded bg-slate-100 px-1 py-0.5 text-[10px] text-slate-800; } .mapping-source-row { @apply grid grid-cols-[minmax(0,1fr)_auto] items-center gap-1 rounded bg-slate-100 px-1 py-0.5 text-[10px] text-slate-800; }
.mapping-source-row.is-disabled { @apply opacity-50; } .mapping-source-row.is-disabled { @apply opacity-50; }
@@ -132,9 +149,9 @@
.column-header { @apply min-w-0; } .column-header { @apply min-w-0; }
.column-toggle { @apply inline-flex h-5 w-5 shrink-0 items-center justify-center rounded border border-slate-400 bg-white text-[0px] leading-none text-slate-700 shadow-sm transition hover:border-slate-500 hover:bg-slate-200 focus:outline-none focus:ring-2 focus:ring-catalog-focus/25; } .column-toggle { @apply inline-flex h-5 w-5 shrink-0 items-center justify-center rounded border border-slate-400 bg-white text-[0px] leading-none text-slate-700 shadow-sm transition hover:border-slate-500 hover:bg-slate-200 focus:outline-none focus:ring-2 focus:ring-catalog-focus/25; }
#secondaryPanelToggle[aria-label="Collapse product details"]::before, #secondaryPanelToggle[aria-label="Collapse product details"]::before,
#mainWorkspaceToggle[aria-label="Expand main workspace"]::before { transform: rotate(135deg); } #mainWorkspaceToggle[aria-label="Collapse main workspace"]::before { transform: rotate(135deg); }
#secondaryPanelToggle[aria-label="Expand product details"]::before, #secondaryPanelToggle[aria-label="Expand product details"]::before,
#mainWorkspaceToggle[aria-label="Collapse main workspace"]::before { transform: rotate(-45deg); } #mainWorkspaceToggle[aria-label="Expand main workspace"]::before { transform: rotate(-45deg); }
.secondary-panel .field-grid { @apply overflow-hidden; } .secondary-panel .field-grid { @apply overflow-hidden; }
.secondary-collapsed .secondary-panel .column-header { @apply h-full flex-col justify-start py-1; writing-mode: vertical-rl; } .secondary-collapsed .secondary-panel .column-header { @apply h-full flex-col justify-start py-1; writing-mode: vertical-rl; }
.secondary-collapsed .secondary-panel .column-header .column-toggle, .secondary-collapsed .secondary-panel .column-header .column-toggle,
@@ -148,9 +165,9 @@
.field-label { @apply bg-slate-200 text-slate-800; } .field-label { @apply bg-slate-200 text-slate-800; }
.field-value { @apply bg-teal-50 text-right text-slate-900; } .field-value { @apply bg-teal-50 text-right text-slate-900; }
.field-value[data-field="manufacturer_name"] { @apply font-medium; } .field-value[data-field="manufacturer_name"] { @apply font-medium; }
.main-workspace { @apply min-w-0 overflow-hidden bg-catalog-surface; } .main-workspace { @apply flex max-h-screen min-w-0 flex-col overflow-hidden bg-catalog-surface; }
.data-panel { @apply min-w-0 overflow-hidden bg-catalog-surface; } .data-panel { @apply min-w-0 overflow-hidden bg-catalog-surface; }
.table-wrap { @apply h-full overflow-auto; } .table-wrap { @apply min-h-0 flex-1 overflow-auto; scrollbar-gutter: stable; }
.variant-grid { @apply min-w-0 text-[11px]; min-width: var(--variant-min-width, 820px); } .variant-grid { @apply min-w-0 text-[11px]; min-width: var(--variant-min-width, 820px); }
.variant-header, .variant-row { @apply grid; grid-template-columns: var(--variant-columns); } .variant-header, .variant-row { @apply grid; grid-template-columns: var(--variant-columns); }
.variant-header { @apply sticky top-0 z-10; } .variant-header { @apply sticky top-0 z-10; }
@@ -165,17 +182,22 @@
.modal-backdrop { @apply fixed inset-0 z-50 flex items-center justify-center bg-slate-900/50 p-4; } .modal-backdrop { @apply fixed inset-0 z-50 flex items-center justify-center bg-slate-900/50 p-4; }
.modal-backdrop[hidden] { display: none; } .modal-backdrop[hidden] { display: none; }
.suggested-dialog, .source-dialog, .settings-dialog { @apply max-h-[90vh] w-full overflow-hidden rounded-lg border border-slate-300 bg-slate-50 shadow-2xl; } .suggested-dialog, .source-dialog, .settings-dialog { @apply max-h-[90vh] w-full overflow-hidden rounded-lg border border-slate-300 bg-slate-50 shadow-2xl; }
.source-dialog-actions { @apply flex items-center gap-1.5; }
.suggested-dialog { @apply max-w-2xl; } .suggested-dialog { @apply max-w-2xl; }
.source-dialog { @apply max-w-3xl; } .source-dialog { @apply max-w-3xl; }
.settings-dialog { @apply max-w-md; } .settings-dialog { @apply max-w-md; }
.suggested-header { @apply flex items-center justify-between border-b border-slate-300 bg-slate-200 px-3 py-2; } .suggested-header { @apply flex items-center justify-between border-b border-slate-300 bg-slate-200 px-3 py-2; }
.suggested-header h2 { @apply m-0 text-sm font-semibold text-slate-800; } .suggested-header h2 { @apply m-0 text-sm font-semibold text-slate-800; }
.modal-close, .settings-test-button { @apply rounded border border-slate-300 bg-white px-2 py-1 text-[11px] text-slate-800 shadow-sm hover:bg-slate-100; } .modal-close, .settings-test-button, .settings-inline-button { @apply inline-flex items-center justify-center gap-1.5 rounded border border-slate-300 bg-white px-2 py-1 text-[11px] text-slate-800 shadow-sm hover:bg-slate-100; }
.modal-close .sidebar-icon, .settings-test-button .sidebar-icon, .settings-inline-button .sidebar-icon { @apply h-3.5 w-3.5 text-slate-500; }
.suggested-list, .source-list, .settings-content { @apply max-h-[calc(90vh-48px)] overflow-y-auto p-2; } .suggested-list, .source-list, .settings-content { @apply max-h-[calc(90vh-48px)] overflow-y-auto p-2; }
.settings-content { @apply grid gap-2; } .settings-content { @apply grid gap-2; }
.settings-group { @apply m-0 rounded border border-slate-300 p-2; } .settings-group { @apply m-0 rounded border border-slate-300 p-2; }
.settings-group legend { @apply px-1 text-xs font-semibold text-slate-700; } .settings-group legend { @apply px-1 text-xs font-semibold text-slate-700; }
.settings-option { @apply flex items-center gap-1 py-0.5 text-[11px] text-slate-700; } .settings-option { @apply flex items-center gap-1 py-0.5 text-[11px] text-slate-700; }
.settings-option .settings-inline-button { @apply ml-auto py-0.5 text-[10px]; }
.settings-tool-actions { @apply ml-auto flex items-center gap-1; }
.settings-tool-actions .settings-inline-button { @apply ml-0; }
.settings-option input { @apply accent-catalog-focus; } .settings-option input { @apply accent-catalog-focus; }
.settings-option input:disabled + span { @apply text-slate-400; } .settings-option input:disabled + span { @apply text-slate-400; }
.local-db-fields { @apply grid gap-1; } .local-db-fields { @apply grid gap-1; }
@@ -185,6 +207,9 @@
.settings-select { @apply min-w-0 rounded border border-slate-300 bg-white px-1 py-1 text-[11px] text-slate-800; } .settings-select { @apply min-w-0 rounded border border-slate-300 bg-white px-1 py-1 text-[11px] text-slate-800; }
.settings-note, .settings-permissions { @apply m-0 text-[10px] leading-tight text-slate-500; } .settings-note, .settings-permissions { @apply m-0 text-[10px] leading-tight text-slate-500; }
.settings-permissions { @apply grid gap-0.5; } .settings-permissions { @apply grid gap-0.5; }
.adminer-login-hint { @apply mt-1 grid gap-0.5 rounded border border-slate-300 bg-slate-100 p-1.5 text-[10px] leading-tight text-slate-600; }
.adminer-login-row { @apply flex items-center justify-between gap-2; }
.adminer-login-row strong { @apply truncate text-right font-semibold text-slate-800; }
.connection-ok { @apply text-emerald-700; } .connection-ok { @apply text-emerald-700; }
.connection-error { @apply text-red-700; } .connection-error { @apply text-red-700; }
.suggested-row { @apply grid grid-cols-[40px_minmax(0,1fr)_auto] items-center gap-2 border-b border-slate-200 px-1 py-1 text-[11px]; } .suggested-row { @apply grid grid-cols-[40px_minmax(0,1fr)_auto] items-center gap-2 border-b border-slate-200 px-1 py-1 text-[11px]; }
@@ -202,9 +227,17 @@
.source-status, .source-row-action { @apply shrink-0 rounded px-1 text-[10px]; } .source-status, .source-row-action { @apply shrink-0 rounded px-1 text-[10px]; }
.source-status.is-ok { @apply bg-emerald-100 text-emerald-700; } .source-status.is-ok { @apply bg-emerald-100 text-emerald-700; }
.source-status.is-muted { @apply bg-slate-200 text-slate-600; } .source-status.is-muted { @apply bg-slate-200 text-slate-600; }
.picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)] items-center gap-2 border-b border-slate-200 p-1 text-[10px]; } .picture-candidate { @apply grid grid-cols-[72px_minmax(0,1fr)_auto] items-center gap-2 border-b border-slate-200 p-2 text-[10px]; }
.picture-candidate img { @apply h-16 w-[72px] object-contain; } .picture-candidate.is-too-small { @apply bg-amber-50; }
.picture-candidate span { @apply min-w-0 truncate; } .picture-candidate-preview { @apply flex h-16 w-[72px] items-center justify-center bg-white; }
.picture-candidate img { @apply max-h-16 w-[72px] object-contain; }
.picture-candidate-meta { @apply grid min-w-0 gap-0.5; }
.picture-candidate-title { @apply min-w-0 truncate font-semibold text-slate-800; }
.picture-candidate-dimensions,
.picture-candidate-target,
.picture-candidate-duplicate { @apply min-w-0 truncate text-slate-500; }
.picture-candidate.is-too-small .picture-candidate-dimensions { @apply font-semibold text-amber-700; }
.picture-candidate-save { @apply min-w-[76px]; }
@media (max-width: 980px) { @media (max-width: 980px) {
.catalog-screen { @apply flex flex-col pl-0; } .catalog-screen { @apply flex flex-col pl-0; }
.catalog-screen.sidebar-collapsed { @apply pl-0; } .catalog-screen.sidebar-collapsed { @apply pl-0; }
@@ -247,7 +280,8 @@
html:not(.dark) .sidebar-drawer-title { @apply text-slate-700; } html:not(.dark) .sidebar-drawer-title { @apply text-slate-700; }
html:not(.dark) .drawer-trigger .drawer-chevron, html:not(.dark) .drawer-trigger .drawer-chevron,
html:not(.dark) .select-chevron, html:not(.dark) .select-chevron,
html:not(.dark) .select-icon { @apply text-slate-700; } html:not(.dark) .select-display,
html:not(.dark) .manufacturer-trigger .sidebar-icon { @apply text-slate-700; }
html:not(.dark) .sidebar-close-button, html:not(.dark) .sidebar-close-button,
html:not(.dark) .sidebar-open-button, html:not(.dark) .sidebar-open-button,
html:not(.dark) .drawer-trigger, html:not(.dark) .drawer-trigger,
@@ -264,8 +298,12 @@
html:not(.dark) #scanInput { @apply bg-slate-200 placeholder:text-slate-500; } html:not(.dark) #scanInput { @apply bg-slate-200 placeholder:text-slate-500; }
html:not(.dark) .picture-slot, html:not(.dark) .picture-slot,
html:not(.dark) .mapping-box { @apply border-slate-300 bg-white text-slate-600; } html:not(.dark) .mapping-box { @apply border-slate-300 bg-white text-slate-600; }
html:not(.dark) .mapping-toggle::before { @apply text-slate-700; } html:not(.dark) .mapping-toggle::before { @apply border-slate-700; }
html:not(.dark) .app-status { @apply border-slate-300 bg-white text-slate-700; } html:not(.dark) .app-status { @apply border-slate-300 bg-white text-slate-700; }
html:not(.dark) .sidebar-mapping-box { @apply border-slate-300 bg-slate-100; }
html:not(.dark) .mapping-summary { @apply border border-slate-300 bg-white text-slate-700; }
html:not(.dark) .mapping-panel { @apply border-slate-300 bg-white; }
html:not(.dark) .mapping-detail-status { @apply text-slate-600; }
html:not(.dark) .manufacturer-trigger { @apply border-slate-300 bg-white text-slate-700; } html:not(.dark) .manufacturer-trigger { @apply border-slate-300 bg-white text-slate-700; }
html:not(.dark) .manufacturer-trigger:hover { @apply border-slate-400 bg-slate-200; } html:not(.dark) .manufacturer-trigger:hover { @apply border-slate-400 bg-slate-200; }
html.contrast-light-high:not(.dark) .control-panel { @apply bg-slate-200 text-slate-900; } html.contrast-light-high:not(.dark) .control-panel { @apply bg-slate-200 text-slate-900; }
@@ -315,8 +353,11 @@
html.dark .settings-field { @apply text-slate-100; } html.dark .settings-field { @apply text-slate-100; }
html.dark .settings-group { @apply border-slate-700; } html.dark .settings-group { @apply border-slate-700; }
html.dark .settings-field input { @apply border-slate-600 bg-slate-800 text-slate-100; } html.dark .settings-field input { @apply border-slate-600 bg-slate-800 text-slate-100; }
html.dark .adminer-login-hint { @apply border-slate-700 bg-slate-800 text-slate-400; }
html.dark .adminer-login-row strong { @apply text-slate-100; }
html.dark .modal-close, html.dark .modal-close,
html.dark .settings-test-button, html.dark .settings-test-button,
html.dark .settings-inline-button,
html.dark .suggested-pick, html.dark .suggested-pick,
html.dark .source-row-action, html.dark .source-row-action,
html.dark .source-picture-action { @apply border-slate-600 bg-slate-800 text-slate-100 hover:bg-slate-700; } html.dark .source-picture-action { @apply border-slate-600 bg-slate-800 text-slate-100 hover:bg-slate-700; }
@@ -325,5 +366,12 @@
html.dark .source-row { @apply border-slate-700 bg-slate-900 text-slate-100 hover:bg-slate-800; } html.dark .source-row { @apply border-slate-700 bg-slate-900 text-slate-100 hover:bg-slate-800; }
html.dark .suggested-row, html.dark .suggested-row,
html.dark .picture-candidate { @apply border-slate-700; } html.dark .picture-candidate { @apply border-slate-700; }
html.dark .picture-candidate.is-too-small { @apply bg-amber-950/40; }
html.dark .picture-candidate-preview { @apply bg-slate-800; }
html.dark .picture-candidate-title { @apply text-slate-100; }
html.dark .picture-candidate-dimensions,
html.dark .picture-candidate-target,
html.dark .picture-candidate-duplicate { @apply text-slate-400; }
html.dark .picture-candidate.is-too-small .picture-candidate-dimensions { @apply text-amber-300; }
html.dark .suggested-row:nth-child(even) { @apply bg-slate-800; } html.dark .suggested-row:nth-child(even) { @apply bg-slate-800; }
} }
+45
View File
@@ -0,0 +1,45 @@
// @ts-nocheck
import test from "node:test";
import assert from "node:assert/strict";
import { extractIdealoGalleryPictureUrls } from "../src/services/catalog-products.ts";
test("idealo gallery parser prefers large splide images over thumbnails", () => {
const html = `
<div class="splide__track container">
<ul class="splide__list">
<li class="splide__slide">
<img
src="//img.idealo.com/folder/Product/200123/4/200123456/s1_product_thumb.jpg"
srcset="//img.idealo.com/folder/Product/200123/4/200123456/s1_product_150x150.jpg 150w,
//img.idealo.com/folder/Product/200123/4/200123456/s1_product_1000x1000.jpg 1000w">
</li>
<li class="splide__slide">
<picture>
<source srcset="//img.idealo.com/folder/Product/200123/4/200123456/s2_product_400x400.jpg 400w,
//img.idealo.com/folder/Product/200123/4/200123456/s2_product_1200x1200.jpg 1200w">
<img src="//img.idealo.com/folder/Product/200123/4/200123456/s2_product_80x80.jpg">
</picture>
</li>
</ul>
</div>`;
const urls = extractIdealoGalleryPictureUrls(html, "https://www.idealo.de/product.html");
assert.deepEqual(urls, [
"https://img.idealo.com/folder/Product/200123/4/200123456/s1_product_1000x1000.jpg",
"https://img.idealo.com/folder/Product/200123/4/200123456/s2_product_1200x1200.jpg",
]);
});
test("idealo gallery parser does not fall back to search thumbnails", () => {
const html = `
<main>
<article class="search-result">
<img src="//img.idealo.com/folder/Product/200123/4/200123456/result_thumb_150x150.jpg">
</article>
</main>`;
const urls = extractIdealoGalleryPictureUrls(html, "https://www.idealo.de/preisvergleich/MainSearchProductCategory.html?q=123");
assert.deepEqual(urls, []);
});
+55
View File
@@ -0,0 +1,55 @@
// @ts-nocheck
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
const schema = fs.readFileSync("sql/local-scrape-db.sql", "utf8");
test("local scrape schema stores image binaries with exact and visual hashes", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_image_blobs`/);
assert.match(schema, /`sha256` CHAR\(64\) NOT NULL/);
assert.match(schema, /`perceptual_hash` CHAR\(16\) NULL/);
assert.match(schema, /`image_data` LONGBLOB NOT NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_image_blobs_sha256` \(`sha256`\)/);
});
test("local scrape schema tracks picture candidates and their target combination logic", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_candidates`/);
assert.match(schema, /`target_strategy` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\)/);
assert.match(schema, /`target_color` VARCHAR\(255\) NULL/);
assert.match(schema, /`target_size` VARCHAR\(128\) NULL/);
assert.match(schema, /`target_combi` VARCHAR\(255\) NULL/);
assert.match(schema, /`duplicate_status` ENUM\('unique', 'exact_duplicate', 'visual_duplicate', 'review'\)/);
});
test("local scrape schema can learn picture target rules from manual corrections", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_picture_target_rules`/);
assert.match(schema, /`product_name_pattern` VARCHAR\(255\) NULL/);
assert.match(schema, /`target_strategy` ENUM\('color', 'size', 'combination', 'manual'\) NOT NULL/);
assert.match(schema, /`examples_count` INT UNSIGNED NOT NULL DEFAULT 0/);
});
test("local scrape schema stores reusable source capabilities for pictures texts and features", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_source_capabilities`/);
assert.match(schema, /`capability` ENUM\('search', 'pictures', 'texts', 'features'\) NOT NULL/);
assert.match(schema, /`parser_key` VARCHAR\(128\) NOT NULL/);
assert.match(schema, /`selector_config` JSON NULL/);
assert.match(schema, /`extraction_config` JSON NULL/);
assert.match(schema, /`quality_config` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_source_capability_parser`/);
});
test("local scrape schema links manufacturer mappings to selected source capabilities", () => {
assert.match(schema, /CREATE TABLE IF NOT EXISTS `scrape_manufacturer_source_capabilities`/);
assert.match(schema, /`target_strategy_override` ENUM\('unknown', 'color', 'size', 'combination', 'manual'\) NULL/);
assert.match(schema, /`config_override` JSON NULL/);
assert.match(schema, /UNIQUE KEY `uq_scrape_manufacturer_source_capability`/);
});
test("local scrape schema seeds Hudy and Idealo extraction capabilities", () => {
assert.match(schema, /'hudy-gallery'/);
assert.match(schema, /'hudy-product-texts'/);
assert.match(schema, /'hudy-product-params'/);
assert.match(schema, /'idealo-splide-gallery'/);
assert.match(schema, /'productGalleryOnly'/);
});
+3
View File
@@ -59,6 +59,9 @@ test("mapping source queries are read-only", () => {
assert.doesNotThrow(() => assert.doesNotThrow(() =>
assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })), assertReadOnlySql(mappingSourcesSql({ manufacturerId: 13 })),
); );
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_manufacturer_sources/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /scrape_source_capabilities/);
assert.match(mappingSourcesSql({ manufacturerId: 13 }), /capabilities/);
}); });
test("manufacturer refresh updates are narrowly allowlisted", () => { test("manufacturer refresh updates are narrowly allowlisted", () => {
+269
View File
@@ -33,6 +33,52 @@ test("manufacturer picker shows ten results and supports search", async ({ page
await expect(page.locator("#manufacturerOptions .manufacturer-option")).toHaveText(["La Sportiva"]); await expect(page.locator("#manufacturerOptions .manufacturer-option")).toHaveText(["La Sportiva"]);
}); });
test("mapping uses cache on reload and reloads from DB on manual manufacturer change", async ({ page }) => {
let mappingRequests = 0;
await page.route("**/api/catalog-maker/mapping-sources**", async (route) => {
mappingRequests += 1;
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
mapped: true,
items: [{ key: "hudy", name: "Hudy", enabled: true }],
}),
});
});
await page.addInitScript(() => {
localStorage.setItem("catalog-maker:selected-manufacturer", "13");
localStorage.setItem(
"catalog-maker:mapping-cache",
JSON.stringify({
13: {
mapped: true,
items: [{ key: "cached-hudy", name: "Cached Hudy", enabled: true }],
},
}),
);
});
await page.goto("/");
await page.waitForFunction(() => document.querySelector("#manufacturerSelect")?.value === "13");
await expect(page.locator("#mappingStatusText")).toContainText("Mapping OK");
await page.waitForTimeout(300);
expect(mappingRequests).toBe(0);
await page.locator("#manufacturerSelect").evaluate((select) => {
const nextOption = [...select.options].find((option) => option.value && option.value !== "13");
select.value = nextOption?.value || "";
select.dispatchEvent(new Event("change", { bubbles: true }));
});
await expect.poll(() => mappingRequests).toBe(1);
await page.locator("#manufacturerSelect").evaluate((select) => {
select.value = "13";
select.dispatchEvent(new Event("change", { bubbles: true }));
});
await expect.poll(() => mappingRequests).toBe(2);
});
test("catalog maker remains usable on a narrow viewport", async ({ page }) => { test("catalog maker remains usable on a narrow viewport", async ({ page }) => {
await page.setViewportSize({ width: 390, height: 844 }); await page.setViewportSize({ width: 390, height: 844 });
await page.goto("/"); await page.goto("/");
@@ -75,6 +121,229 @@ test("workspace columns collapse and expand without leaving a gap", async ({ pag
await expect(page.locator(".catalog-screen")).not.toHaveClass(/workspace-collapsed/); await expect(page.locator(".catalog-screen")).not.toHaveClass(/workspace-collapsed/);
}); });
test("main workspace keeps a bottom horizontal scrollbar for wide data", async ({ page }) => {
await page.setViewportSize({ width: 1061, height: 912 });
await page.goto("/");
const scrollState = await page.locator(".main-workspace .table-wrap").evaluate((element) => {
const style = getComputedStyle(element);
return {
overflowX: style.overflowX,
clientWidth: element.clientWidth,
scrollWidth: element.scrollWidth,
};
});
expect(scrollState.overflowX).toBe("auto");
expect(scrollState.scrollWidth).toBeGreaterThan(scrollState.clientWidth);
});
test("picture candidates show source, dimensions, target assignment and save action", async ({ page }) => {
await page.route("**/api/manufacturers", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({ items: [{ id: 13, name: "La Sportiva" }] }),
});
});
await page.route("**/api/languages", async (route) => {
await route.fulfill({ contentType: "application/json", body: JSON.stringify({ items: [] }) });
});
await page.route("**/api/catalog-maker/mapping-sources**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
mapped: true,
items: [{ key: "hudy", name: "Hudy", enabled: true, health: "ok" }],
}),
});
});
await page.route("**/api/catalog-maker/product**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
product: {
idProductCatalog: 135520,
idProduct: 110927,
idManufacturer: 13,
codeManufacture: "LAS",
supplierReference: "ZFHS099",
name: "Aequilibrium Trek Woman GTX",
position: 1,
total: 380,
},
combinations: [
{
idProductCatalog: 110923,
status: "SS25/Boty",
color: "Carbon/Malibu Blue",
combination: "36",
ean13: "8058428191284",
supplierStock: 1,
},
],
}),
});
});
await page.route("**/api/catalog-maker/get-pictures**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
items: [
{
source: { key: "hudy", name: "Hudy", type: "supplier" },
searchUrl: "https://www.hudy.cz/vyhledavani?q=8058428191284",
candidates: [
{
rawTitle: "Aequilibrium Trek Woman GTX",
url: "https://www.hudy.cz/product",
pictures: [
"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='80' height='64'%3E%3Crect width='80' height='64' fill='red'/%3E%3C/svg%3E",
"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='120' height='90'%3E%3Crect width='120' height='90' fill='blue'/%3E%3C/svg%3E",
],
},
],
},
],
}),
});
});
let saveRequests = 0;
await page.route("**/api/catalog-maker/save-picture", async (route) => {
saveRequests += 1;
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
saved: true,
candidateId: saveRequests,
duplicateStatus: "unique",
}),
});
});
await page.addInitScript(() => {
localStorage.setItem("catalog-maker:selected-manufacturer", "13");
});
await page.goto("/");
await page.waitForFunction(() => document.querySelector("#manufacturerSelect")?.value === "13");
await page.locator("#getPicturesButton").click();
await expect(page.locator("#sourceModal")).toBeVisible();
await expect(page.locator(".picture-candidate-title").first()).toHaveText("Hudy picture #1");
await expect(page.locator(".picture-candidate-dimensions").first()).toContainText("too small, min 800 x 800");
await expect(page.locator(".picture-candidate").first()).toHaveClass(/is-too-small/);
await expect(page.locator(".picture-candidate-target").first()).toContainText("Target: color Carbon/Malibu Blue");
await expect(page.locator(".picture-candidate-save")).toHaveCount(2);
await expect(page.locator("#sourceBatchActionButton")).toBeVisible();
await expect(page.locator("#sourceBatchActionButton")).toContainText("Save all");
await page.locator("#sourceBatchActionButton").click();
await expect(page.locator(".picture-candidate-save")).toHaveText(["Saved", "Saved"]);
expect(saveRequests).toBe(2);
});
test("source candidates can fetch all pictures before saving candidates", async ({ page }) => {
await page.route("**/api/manufacturers", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({ items: [{ id: 13, name: "La Sportiva" }] }),
});
});
await page.route("**/api/languages", async (route) => {
await route.fulfill({ contentType: "application/json", body: JSON.stringify({ items: [] }) });
});
await page.route("**/api/catalog-maker/mapping-sources**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
mapped: true,
items: [{ key: "hudy", name: "Hudy", enabled: true, health: "ok" }],
}),
});
});
await page.route("**/api/catalog-maker/product**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
product: {
idProductCatalog: 135520,
idProduct: 110927,
idManufacturer: 13,
codeManufacture: "LAS",
supplierReference: "ZFHS099",
name: "Aequilibrium Trek Woman GTX",
position: 1,
total: 380,
},
combinations: [
{
idProductCatalog: 110923,
status: "SS25/Boty",
color: "Carbon/Malibu Blue",
combination: "36",
ean13: "8058428191284",
supplierStock: 1,
},
],
}),
});
});
await page.route("**/api/catalog-maker/find-sources**", async (route) => {
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
items: [
{
status: "found",
source: { key: "hudy", name: "Hudy", type: "supplier" },
candidates: [
{
title: "Direct Hudy / EAN: 8058428191284 / Color: Carbon/Malibu Blue",
url: "https://www.hudy.cz/product",
},
],
},
],
}),
});
});
await page.route("**/api/catalog-maker/get-pictures**", async (route) => {
expect(route.request().url()).not.toContain("sourceKey=");
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
items: [
{
source: { key: "hudy", name: "Hudy", type: "supplier" },
candidates: [
{
rawTitle: "Aequilibrium Trek Woman GTX",
url: "https://www.hudy.cz/product",
pictures: [
"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='80' height='64'%3E%3Crect width='80' height='64' fill='red'/%3E%3C/svg%3E",
],
},
],
},
],
}),
});
});
await page.addInitScript(() => {
localStorage.setItem("catalog-maker:selected-manufacturer", "13");
});
await page.goto("/");
await page.waitForFunction(() => document.querySelector("#manufacturerSelect")?.value === "13");
await page.locator("#findSourcesButton").click();
await expect(page.locator("#sourceModal")).toBeVisible();
await expect(page.locator("#sourceBatchActionButton")).toBeVisible();
await expect(page.locator("#sourceBatchActionButton")).toContainText("Get all pictures");
await page.locator("#sourceBatchActionButton").click();
await expect(page.locator(".picture-candidate-title").first()).toHaveText("Hudy picture #1");
await expect(page.locator("#sourceBatchActionButton")).toContainText("Save all");
});
test("uses one shared height for the top panel headers", async ({ page }) => { test("uses one shared height for the top panel headers", async ({ page }) => {
await page.goto("/"); await page.goto("/");
const heights = await page.locator("#sidebarDrawer .sidebar-drawer-header, .secondary-panel .column-header, .main-workspace .column-header").evaluateAll((elements) => elements.map((element) => Math.round(element.getBoundingClientRect().height))); const heights = await page.locator("#sidebarDrawer .sidebar-drawer-header, .secondary-panel .column-header, .main-workspace .column-header").evaluateAll((elements) => elements.map((element) => Math.round(element.getBoundingClientRect().height)));
+34 -1
View File
@@ -17,7 +17,7 @@ test.describe("database connection", () => {
await expect(page.locator("#localDbTestResult")).toContainText("Connected:"); await expect(page.locator("#localDbTestResult")).toContainText("Connected:");
}); });
test("keeps the scrape database as a separate read-only mode", async ({ page }) => { test("keeps the scrape database as a separate local mode with its own permissions", async ({ page }) => {
await page.goto("/"); await page.goto("/");
await page.locator("#settingsButton").click(); await page.locator("#settingsButton").click();
@@ -25,6 +25,22 @@ test.describe("database connection", () => {
await expect(page.locator('input[name="databaseMode"][value="live"] + span')).toHaveText("Live 9bplus DB"); await expect(page.locator('input[name="databaseMode"][value="live"] + span')).toHaveText("Live 9bplus DB");
await page.locator('input[name="databaseMode"][value="scrape"]').check(); await page.locator('input[name="databaseMode"][value="scrape"]').check();
await expect(page.locator("#databaseModeNote")).toContainText("Local scrape DB selected"); await expect(page.locator("#databaseModeNote")).toContainText("Local scrape DB selected");
await expect(page.locator("#allowInsertToggle")).toBeEnabled();
await expect(page.locator("#allowUpdateToggle")).toBeEnabled();
await expect(page.locator("#allowDeleteToggle")).toBeEnabled();
await page.locator("#allowUpdateToggle").check();
await page.reload();
await page.locator("#settingsButton").click();
await page.locator('input[name="databaseMode"][value="scrape"]').check();
await expect(page.locator("#allowUpdateToggle")).toBeChecked();
await page.locator('input[name="databaseMode"][value="local"]').check();
await expect(page.locator("#allowInsertToggle")).not.toBeChecked();
await expect(page.locator("#allowUpdateToggle")).not.toBeChecked();
await expect(page.locator("#allowDeleteToggle")).not.toBeChecked();
await page.locator('input[name="databaseMode"][value="live"]').check();
await expect(page.locator("#allowInsertToggle")).toBeDisabled(); await expect(page.locator("#allowInsertToggle")).toBeDisabled();
await expect(page.locator("#allowUpdateToggle")).toBeDisabled(); await expect(page.locator("#allowUpdateToggle")).toBeDisabled();
await expect(page.locator("#allowDeleteToggle")).toBeDisabled(); await expect(page.locator("#allowDeleteToggle")).toBeDisabled();
@@ -40,4 +56,21 @@ test.describe("database connection", () => {
expect(data.items.length).toBeGreaterThan(0); expect(data.items.length).toBeGreaterThan(0);
expect(data.items.some((item) => item.name === "La Sportiva")).toBeTruthy(); expect(data.items.some((item) => item.name === "La Sportiva")).toBeTruthy();
}); });
test("persists Local 9bplus write permissions across reloads", async ({ page }) => {
await page.goto("/");
await page.locator("#settingsButton").click();
await page.locator('input[name="databaseMode"][value="local"]').check();
await page.locator("#allowInsertToggle").check();
await page.locator("#allowUpdateToggle").check();
await page.locator("#allowDeleteToggle").check();
await page.reload();
await page.locator("#settingsButton").click();
await expect(page.locator('input[name="databaseMode"][value="local"]')).toBeChecked();
await expect(page.locator("#allowInsertToggle")).toBeChecked();
await expect(page.locator("#allowUpdateToggle")).toBeChecked();
await expect(page.locator("#allowDeleteToggle")).toBeChecked();
});
}); });