Stabilize visitor language detection
This commit is contained in:
@@ -1114,13 +1114,16 @@ async function translateStoredMessage(conversationId, messageId, senderType, bod
|
||||
|
||||
try {
|
||||
const result = await translateText(message.body, targetLanguage, settings.model, settings.openaiApiKey);
|
||||
const sourceLanguage = normalizeLanguage(result.sourceLanguage);
|
||||
const detectedSourceLanguage = normalizeDetectedLanguage(result.sourceLanguage);
|
||||
const sourceLanguage = senderType === "visitor"
|
||||
? stableVisitorLanguage(conversation, message.body, detectedSourceLanguage)
|
||||
: detectedSourceLanguage;
|
||||
const translatedText = String(result.translatedText || "").trim();
|
||||
const translatedLanguage = normalizeLanguage(result.translatedLanguage || targetLanguage);
|
||||
const translatedLanguage = normalizeDetectedLanguage(result.translatedLanguage || targetLanguage);
|
||||
const sameLanguage = sourceLanguage && translatedLanguage && sourceLanguage === translatedLanguage;
|
||||
|
||||
if (senderType === "visitor" && sourceLanguage) {
|
||||
db.prepare("UPDATE conversations SET customer_language = COALESCE(?, customer_language), updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(sourceLanguage, conversationId);
|
||||
db.prepare("UPDATE conversations SET customer_language = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?").run(sourceLanguage, conversationId);
|
||||
}
|
||||
|
||||
if (!translatedText || sameLanguage) {
|
||||
@@ -1157,7 +1160,7 @@ async function translateText(text, targetLanguage, model, apiKey) {
|
||||
input: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You translate live chat messages for an e-commerce support team. Return only compact JSON with keys sourceLanguage, translatedLanguage, translatedText. Preserve meaning, product terms, numbers, names, URLs, and tone. Do not answer the customer."
|
||||
content: "You translate live chat messages for an e-commerce support team. Return only compact JSON with keys sourceLanguage, translatedLanguage, translatedText. Preserve meaning, product terms, numbers, names, URLs, and tone. Do not answer the customer. If the source language is unclear because the text is a greeting, product name, SKU, brand, color, URL, or otherwise language-neutral, set sourceLanguage to unknown."
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@@ -1768,6 +1771,53 @@ function normalizeLanguage(value) {
|
||||
return text.split("-")[0].slice(0, 8);
|
||||
}
|
||||
|
||||
function normalizeDetectedLanguage(value) {
|
||||
const language = normalizeLanguage(value);
|
||||
return ["unknown", "und", "auto", "n/a", "none"].includes(language) ? "" : language;
|
||||
}
|
||||
|
||||
function stableVisitorLanguage(conversation, text, detectedLanguage) {
|
||||
const currentLanguage = normalizeDetectedLanguage(conversation?.customer_language);
|
||||
const browserLanguage = normalizeDetectedLanguage(conversation?.language);
|
||||
if (!currentLanguage) {
|
||||
if (browserLanguage && weakLanguageEvidence(text)) return browserLanguage;
|
||||
return detectedLanguage;
|
||||
}
|
||||
if (!detectedLanguage || detectedLanguage === currentLanguage) return currentLanguage;
|
||||
return weakLanguageEvidence(text) ? currentLanguage : detectedLanguage;
|
||||
}
|
||||
|
||||
function weakLanguageEvidence(value) {
|
||||
const text = String(value || "").trim();
|
||||
if (!text) return true;
|
||||
|
||||
const words = text.match(/[\p{L}\p{N}][\p{L}\p{N}'-]*/gu) || [];
|
||||
const lowerWords = words.map((word) => word.toLowerCase());
|
||||
if (!words.length) return true;
|
||||
if (hasStrongLanguageEvidence(text, lowerWords)) return false;
|
||||
|
||||
const compact = lowerWords.join(" ");
|
||||
const weakPhrases = new Set([
|
||||
"hi", "hello", "hey", "ok", "okay", "yes", "no", "thanks", "thank you",
|
||||
"hallo", "servus", "danke", "ja", "nein", "bitte"
|
||||
]);
|
||||
if (weakPhrases.has(compact)) return true;
|
||||
if (words.length <= 2) return true;
|
||||
|
||||
const startsLikeProductName = /^[\p{Lu}\p{N}]/u.test(words[0]);
|
||||
const sentencePunctuation = /[?!.]/.test(text);
|
||||
return words.length <= 6 && startsLikeProductName && !sentencePunctuation;
|
||||
}
|
||||
|
||||
function hasStrongLanguageEvidence(text, lowerWords) {
|
||||
const strongWords = new Set([
|
||||
"i", "you", "we", "need", "want", "would", "could", "please", "what", "which", "size", "have", "has", "is", "are",
|
||||
"ich", "sie", "wir", "brauche", "benotige", "möchte", "mochte", "bitte", "welche", "welcher", "welches", "größe", "groesse", "hat", "haben", "ist", "sind", "wie", "was", "kann"
|
||||
]);
|
||||
const hits = lowerWords.filter((word) => strongWords.has(word)).length;
|
||||
return hits >= 2 || (hits >= 1 && /[?]/.test(text));
|
||||
}
|
||||
|
||||
function truncateText(value, maxLength) {
|
||||
const text = String(value || "").trim();
|
||||
if (!text) return null;
|
||||
|
||||
Reference in New Issue
Block a user