Stabilize visitor language detection

This commit is contained in:
rajchales-monito
2026-08-06 15:52:15 +02:00
parent bf2c58f90a
commit 17b4665e23
2 changed files with 107 additions and 4 deletions
+54 -4
View File
@@ -1114,13 +1114,16 @@ async function translateStoredMessage(conversationId, messageId, senderType, bod
try {
const result = await translateText(message.body, targetLanguage, settings.model, settings.openaiApiKey);
const sourceLanguage = normalizeLanguage(result.sourceLanguage);
const detectedSourceLanguage = normalizeDetectedLanguage(result.sourceLanguage);
const sourceLanguage = senderType === "visitor"
? stableVisitorLanguage(conversation, message.body, detectedSourceLanguage)
: detectedSourceLanguage;
const translatedText = String(result.translatedText || "").trim();
const translatedLanguage = normalizeLanguage(result.translatedLanguage || targetLanguage);
const translatedLanguage = normalizeDetectedLanguage(result.translatedLanguage || targetLanguage);
const sameLanguage = sourceLanguage && translatedLanguage && sourceLanguage === translatedLanguage;
if (senderType === "visitor" && sourceLanguage) {
db.prepare("UPDATE conversations SET customer_language = COALESCE(?, customer_language), updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(sourceLanguage, conversationId);
db.prepare("UPDATE conversations SET customer_language = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(sourceLanguage, conversationId);
}
if (!translatedText || sameLanguage) {
@@ -1157,7 +1160,7 @@ async function translateText(text, targetLanguage, model, apiKey) {
input: [
{
role: "system",
content: "You translate live chat messages for an e-commerce support team. Return only compact JSON with keys sourceLanguage, translatedLanguage, translatedText. Preserve meaning, product terms, numbers, names, URLs, and tone. Do not answer the customer."
content: "You translate live chat messages for an e-commerce support team. Return only compact JSON with keys sourceLanguage, translatedLanguage, translatedText. Preserve meaning, product terms, numbers, names, URLs, and tone. Do not answer the customer. If the source language is unclear because the text is a greeting, product name, SKU, brand, color, URL, or otherwise language-neutral, set sourceLanguage to unknown."
},
{
role: "user",
@@ -1768,6 +1771,53 @@ function normalizeLanguage(value) {
return text.split("-")[0].slice(0, 8);
}
function normalizeDetectedLanguage(value) {
const language = normalizeLanguage(value);
return ["unknown", "und", "auto", "n/a", "none"].includes(language) ? "" : language;
}
function stableVisitorLanguage(conversation, text, detectedLanguage) {
const currentLanguage = normalizeDetectedLanguage(conversation?.customer_language);
const browserLanguage = normalizeDetectedLanguage(conversation?.language);
if (!currentLanguage) {
if (browserLanguage && weakLanguageEvidence(text)) return browserLanguage;
return detectedLanguage;
}
if (!detectedLanguage || detectedLanguage === currentLanguage) return currentLanguage;
return weakLanguageEvidence(text) ? currentLanguage : detectedLanguage;
}
function weakLanguageEvidence(value) {
const text = String(value || "").trim();
if (!text) return true;
const words = text.match(/[\p{L}\p{N}][\p{L}\p{N}'-]*/gu) || [];
const lowerWords = words.map((word) => word.toLowerCase());
if (!words.length) return true;
if (hasStrongLanguageEvidence(text, lowerWords)) return false;
const compact = lowerWords.join(" ");
const weakPhrases = new Set([
"hi", "hello", "hey", "ok", "okay", "yes", "no", "thanks", "thank you",
"hallo", "servus", "danke", "ja", "nein", "bitte"
]);
if (weakPhrases.has(compact)) return true;
if (words.length <= 2) return true;
const startsLikeProductName = /^[\p{Lu}\p{N}]/u.test(words[0]);
const sentencePunctuation = /[?!.]/.test(text);
return words.length <= 6 && startsLikeProductName && !sentencePunctuation;
}
function hasStrongLanguageEvidence(text, lowerWords) {
const strongWords = new Set([
"i", "you", "we", "need", "want", "would", "could", "please", "what", "which", "size", "have", "has", "is", "are",
"ich", "sie", "wir", "brauche", "benotige", "möchte", "mochte", "bitte", "welche", "welcher", "welches", "größe", "groesse", "hat", "haben", "ist", "sind", "wie", "was", "kann"
]);
const hits = lowerWords.filter((word) => strongWords.has(word)).length;
return hits >= 2 || (hits >= 1 && /[?]/.test(text));
}
function truncateText(value, maxLength) {
const text = String(value || "").trim();
if (!text) return null;