Broaden clustered bot filtering

server.js

@@ -1838,10 +1838,26 @@ function currentVisitors() {
     ORDER BY datetime(vs.first_seen_at) ASC, vs.id ASC
     LIMIT 100
   `).all();
-  const clusters = suspiciousVisitorClusters(rows, presence);
+  const clusters = suspiciousVisitorClusters(recentVisitorClusterRows(presence), presence);
   return rows.map((row) => formatVisitorSession(row, presence, clusters));
 }
 
+function recentVisitorClusterRows(settings) {
+  if (!settings.clusterBotFilterEnabled) return [];
+  const lookbackSeconds = Math.max(settings.offlineSeconds, settings.clusterBotWindowSeconds * 4, 600);
+  return db.prepare(`
+    SELECT vs.*, s.site_key, vp.display_name AS visitor_display_name,
+      (SELECT COUNT(*) FROM conversations c WHERE c.site_id = vs.site_id AND c.visitor_token = vs.visitor_token) AS visitor_conversation_count
+    FROM visitor_sessions vs
+    JOIN sites s ON s.id = vs.site_id
+    LEFT JOIN visitor_profiles vp ON vp.site_id = vs.site_id AND vp.visitor_token = vs.visitor_token
+    WHERE datetime(COALESCE(vs.last_activity_at, vs.last_seen_at, vs.first_seen_at)) >= datetime('now', '-${lookbackSeconds} seconds')
+       OR datetime(vs.first_seen_at) >= datetime('now', '-${lookbackSeconds} seconds')
+    ORDER BY datetime(vs.first_seen_at) ASC, vs.id ASC
+    LIMIT 3000
+  `).all();
+}
+
 function formatVisitorSession(row, thresholds = adminPresenceSettings(), clusters = new Map()) {
   const history = parseJson(row.browsing_history_json, []);
   const device = parseJson(row.device_json, {});
@@ -1855,7 +1871,7 @@ function formatVisitorSession(row, thresholds = adminPresenceSettings(), cluster
     sessionSeconds: sessionSecondsValue,
     referrer: row.referrer
   });
-  const clusterReason = clusters.get(visitorClusterKey(row, device));
+  const clusterReason = visitorClusterReason(row, device, clusters);
   const finalBot = clusterReason && conversationCountValue === 0 ? { isBot: true, reason: clusterReason } : bot;
   const countryCode = row.country_code || countryFromTimezone(row.timezone) || countryFromLanguage(row.language);
   const presence = visitorPresence(row, thresholds);
@@ -1903,21 +1919,24 @@ function suspiciousVisitorClusters(rows, settings) {
     if (Number(row.visitor_conversation_count || 0) > 0) continue;
     if (String(row.referrer || "").trim()) continue;
     const device = parseJson(row.device_json, {});
-    const key = visitorClusterKey(row, device);
-    if (!key) continue;
     const firstSeen = parseTimestampMs(row.first_seen_at);
     if (!Number.isFinite(firstSeen)) continue;
+    for (const key of visitorClusterKeys(row, device)) {
       if (!groups.has(key)) groups.set(key, []);
       groups.get(key).push(firstSeen);
     }
+  }
   const suspicious = new Map();
   const windowMs = settings.clusterBotWindowSeconds * 1000;
   for (const [key, times] of groups.entries()) {
+    const requiredCount = key.startsWith("device|")
+      ? Math.max(settings.clusterBotCount * 3, settings.clusterBotCount + 20)
+      : settings.clusterBotCount;
     times.sort((a, b) => a - b);
     let start = 0;
     for (let end = 0; end < times.length; end += 1) {
       while (times[end] - times[start] > windowMs) start += 1;
-      if (end - start + 1 >= settings.clusterBotCount) {
+      if (end - start + 1 >= requiredCount) {
         suspicious.set(key, `cluster ${end - start + 1}/${settings.clusterBotWindowSeconds}s`);
         break;
       }
@@ -1926,9 +1945,16 @@ function suspiciousVisitorClusters(rows, settings) {
   return suspicious;
 }
 
-function visitorClusterKey(row, device = {}) {
+function visitorClusterReason(row, device, clusters) {
+  for (const key of visitorClusterKeys(row, device)) {
+    const reason = clusters.get(key);
+    if (reason) return reason;
+  }
+  return "";
+}
+
+function visitorClusterKeys(row, device = {}) {
   const ip = String(row.ip || "").trim();
-  if (!ip) return "";
   const country = String(row.country_code || countryFromTimezone(row.timezone) || countryFromLanguage(row.language) || "").toUpperCase();
   const platform = String(device.platform || "").trim();
   const ua = String(device.userAgent || "").trim();
@@ -1937,7 +1963,10 @@ function visitorClusterKey(row, device = {}) {
     : ua.includes("Firefox/") ? "Firefox"
     : ua.includes("Safari/") ? "Safari"
     : ua.slice(0, 80);
-  return [ip, country, platform, browser].join("|");
+  const deviceKey = [country, platform, browser].join("|");
+  const keys = [`device|${deviceKey}`];
+  if (ip) keys.unshift(`ip|${ip}|${deviceKey}`);
+  return keys.filter((key) => key.replace(/^(ip\|[^|]*\||device\|)/, "").replace(/\|/g, "").trim());
 }
 
 function conversationDetails(conversation) {