diff --git a/src/VahterBanBot/Bot.fs b/src/VahterBanBot/Bot.fs index 0e181a9..f77e4b3 100644 --- a/src/VahterBanBot/Bot.fs +++ b/src/VahterBanBot/Bot.fs @@ -256,6 +256,10 @@ module private BotHelpers = let prefix = actor |> Option.map (fun a -> $"{a.DisplayName}, ") |> Option.defaultValue "" match reason with | AutoDeleteReason.MlSpam r -> $"{prefix}score: {r.score}" + // Same "score: x" shape as MlSpam — the actor prefix (already "LLM/{modelName}, " via + // Actor.LLM.DisplayName) is what tells a human this was the LLM's own kill call, not a + // plain ML-threshold verdict; formatReasonStr keeps that wording stable on purpose. + | AutoDeleteReason.LlmSpam r -> $"{prefix}score: {r.score}" | AutoDeleteReason.ReactionSpam r -> $"{prefix}reactions: {r.reactionCount}" | AutoDeleteReason.InvisibleMention -> $"{prefix}invisible mention" | AutoDeleteReason.SpamTextCacheHit r -> $"{prefix}spam-text cache hit, seeded by ban of {r.seedChatId}/{r.seedMessageId}" @@ -268,6 +272,18 @@ module private BotHelpers = else photos |> Array.maxBy (fun p -> p.Width * p.Height) +/// Picks the `AutoDeleteReason` case for an `AutoVerdict.Spam` kill, based on which actor made +/// the call: `Actor.LLM` means `LlmVerdict.Kill` decided it (see `GetAutoVerdict`), so it's +/// `LlmSpam`; anything else (`Actor.ML`, the only other actor `AutoVerdict.Spam` carries) is a +/// plain ML-threshold verdict, `MlSpam`. Deliberately public — unlike BotHelpers' predicates, +/// which are private to this file — so the 2026-08-18 misattribution incident (an LLM kill +/// recorded/rendered as a plain `MlSpam` verdict) is unit-testable without a container: see +/// VahterBanBot.Unit.Tests/SpamDeleteReasonTests.fs. +let spamDeleteReason (score: float) (actor: Actor) : AutoDeleteReason = + match actor with + | Actor.LLM l -> AutoDeleteReason.LlmSpam {| score = score; modelName = l.modelName |} + | _ -> AutoDeleteReason.MlSpam {| score = score |} + /// True if the message's first token is the "/vahter_report" command (mention-tolerant, /// same tokenizing pattern as BotHelpers.isVahterCommand above — e.g. "/vahter_report@my_bot" /// matches). Deliberately public — unlike BotHelpers' predicates, which are private to this @@ -1358,7 +1374,10 @@ type BotService( | Some (AutoVerdict.Spam (score, actor)) -> %mlActivity.SetTag("spamScoreMl", score) %mlActivity.SetTag("autoVerdict", "spam") - do! enforceSpam actor (MlSpam {| score = score |}) + // The LLM itself said SPAM (LlmVerdict.Kill) vs. crossing the ML score + // threshold on its own — attribute the reason accordingly (2026-08-18 + // incident: an LLM kill was mislabeled as a plain MlSpam verdict). + do! enforceSpam actor (spamDeleteReason score actor) | Some (AutoVerdict.ContentFilterSpam (score, actor, triggers)) -> %mlActivity.SetTag("spamScoreMl", score) %mlActivity.SetTag("autoVerdict", "contentFilterSpam") diff --git a/src/VahterBanBot/LlmTriage.fs b/src/VahterBanBot/LlmTriage.fs index 355930b..c6fc87a 100644 --- a/src/VahterBanBot/LlmTriage.fs +++ b/src/VahterBanBot/LlmTriage.fs @@ -18,6 +18,7 @@ open VahterBanBot.Telemetry open VahterBanBot.Types open VahterBanBot.Utils open VahterBanBot.LlmVerdictCache +open VahterBanBot.ProfileFetcher open BotInfra // ── Dedup helpers ───────────────────────────────────────────────────────────── @@ -237,6 +238,65 @@ let private logContentFilterRejection (logger: ILogger) (pathLabel: string) (tri "{TriagePath} content_filter rejection (HTTP 400): Azure RAI policy flagged the prompt as harmful. Triggers: {ContentFilterTriggers}. Raw response: {RawResponseBody}", pathLabel, triggers, defaultArg rawBody "(raw response body unavailable)") +// ── Empty-text media placeholder (message LLM triage only) ──────────────────── +// +// Incident (2026-08-18, @AvaloniaRU, msg 217142): a caption-less spam-check-worthy sticker had +// no OCR text, so `msg.Text` stayed null. The LLM user-content rendered `Message:` with an EMPTY +// body (null interpolates to ""), so gpt-4o-mini judged a blank message on username/display-name +// alone and said SPAM — an innocent static cat sticker got auto-deleted. Real spammers DO post +// content-less stickers/photos with the spam in their NAME or BIO, so the fix is to give the LLM +// honest context about WHAT the message is, not to skip triage on empty text. +// +// CRITICAL: `mediaPlaceholder` is read ONLY when building the LLM prompt below — it must never be +// written back via `msg.AppendText`/`msg.PrependText`. `msg.Text` also feeds the ML scorer, the +// spam-text cache, the verdict-cache key (see `hasStableTextCacheKey` / Classify below), and the +// deleted-spam channel post. If the placeholder ever leaked into `msg.Text`, every photo/sticker +// would collapse onto the SAME cache key (e.g. "[photo, no readable text]") and — because +// SPAM/SKIP verdicts are cached GLOBALLY by text hash (see the module doc comment at the top of +// this file) — one SPAM verdict on a single photo would globally condemn every future photo. + +/// Descriptive placeholder for the LLM prompt's `Message:` body when the message has no readable +/// text (`msg.Text` is null/empty) — `None` when there IS real text, in which case the caller +/// should render `msg.Text` as-is. Degrades gracefully when sticker emoji/set_name are absent +/// (the 2026-08-12 prod spam sticker had neither — see StickerOcrTests.fs). +let mediaPlaceholder (msg: TgMessage) : string option = + if not (String.IsNullOrEmpty msg.Text) then None + else + match msg.Sticker with + | Some s -> + let emojiPart = s.Emoji |> Option.map (fun e -> $" \"{e}\"") |> Option.defaultValue "" + let setPart = s.SetName |> Option.map (fun n -> $" from set \"{n}\"") |> Option.defaultValue "" + Some $"[sticker{emojiPart}{setPart}, no readable text]" + | None -> + if msg.Photos.Length > 0 then + Some "[photo, no readable text]" + else + // RawMessage is `internal` (same-assembly access only — see TgMessage.fs), so this + // generic media check lives here rather than as a public TgMessage member. + let raw = msg.RawMessage + if raw.Video.IsSome then Some "[video, no readable text]" + elif raw.Animation.IsSome then Some "[animation, no readable text]" + elif raw.VideoNote.IsSome then Some "[video note, no readable text]" + elif raw.Voice.IsSome then Some "[voice message, no readable text]" + elif raw.Audio.IsSome then Some "[audio, no readable text]" + elif raw.Document.IsSome then Some "[document, no readable text]" + else Some "[empty message]" + +/// Whether `msg.Text` alone yields a stable cache key — mirrors the guard `Classify` uses to pick +/// `NoCache` (LlmTriage.fs's `CacheRouting`). Pulled out as a pure predicate so "a placeholder- +/// rendered message still hits NoCache" is unit-testable without a live Azure client: `msg.Text` +/// is never mutated by `mediaPlaceholder` above, so a message that gets a placeholder in the +/// prompt still reports `false` here, exactly as before this change. +let hasStableTextCacheKey (msg: TgMessage) : bool = + not (String.IsNullOrEmpty msg.Text) + +/// Renders a fetched sender bio for the LLM prompt's "Bio:" line — `(none)` for null/empty/ +/// whitespace, the bio text otherwise. `IUserProfileFetcher.Fetch` never throws (see +/// ProfileFetcher.fs) and already degrades any fetch failure to `Bio = ""`, so blank-vs-real is +/// the only distinction left to make here. Pulled out as a pure function purely for unit testing. +let formatBioLine (bio: string) : string = + if String.IsNullOrWhiteSpace bio then "(none)" else bio + // ── Interface + implementation ──────────────────────────────────────────────── type ILlmTriage = @@ -244,7 +304,7 @@ type ILlmTriage = abstract member PromptHash: string abstract member Classify: msg: TgMessage * userMsgCount: int64 * ct: CancellationToken -> Task -type AzureLlmTriage(botConf: IOptions, logger: ILogger, db: DbService, cache: ILlmVerdictCache) = +type AzureLlmTriage(botConf: IOptions, logger: ILogger, db: DbService, cache: ILlmVerdictCache, profileFetcher: IUserProfileFetcher) = // Coalesces concurrent identical-text classifications (same spam across channels at once). let inflight = ConcurrentDictionary>>() @@ -262,6 +322,12 @@ Message count context (provided as "Total messages seen from this user"): - 10-20 messages: could be a hidden spammer who posted random stuff to blend in - 20-50 messages: most probably not a spammer — message must be really advertising something or be malicious +A media-only message with no readable text (rendered below as e.g. "[sticker ..., no readable +text]" or "[photo, no readable text]") is NOT, by itself, a spam signal — real spammers do this, +but so do ordinary members posting a reaction sticker/photo with nothing to OCR. For such +messages, judge only the sender signals (username, display name, bio); when those look normal, +prefer NOT_SPAM/SKIP. + Classify the message as exactly one of: - SPAM : obvious advertising/bot/malicious content — delete and reduce user karma - SKIP : not sure — route to human moderators for review @@ -296,13 +362,27 @@ Respond with exactly: {"verdict":"SPAM"} or {"verdict":"SKIP"} or {"verdict":"NO let username = if isNull msg.SenderUsername then "(none)" else $"@{msg.SenderUsername}" let displayName = msg.SenderDisplayName + + // Fetched only here — at the point of actual LLM escalation, not for every message. + // IUserProfileFetcher.Fetch never throws (see ProfileFetcher.fs); an empty/missing bio + // still degrades to "(none)" below. + let! profile = profileFetcher.Fetch(msg.SenderId) + let bio = formatBioLine profile.Bio + + // See the module doc comment above `mediaPlaceholder`: this placeholder is rendered ONLY + // in the prompt string below — msg.Text itself is never touched, so the ML scorer / spam- + // text cache / verdict-cache key / deleted-spam channel post all keep seeing the real + // (empty) text. + let messageBody = mediaPlaceholder msg |> Option.defaultValue msg.Text + let userPrompt = $"""Username: {username} Display name: {displayName} +Bio: {bio} Total messages seen from this user: {userMsgCount} Message: -{msg.Text}""" +{messageBody}""" let options = ChatCompletionOptions( @@ -395,7 +475,9 @@ Message: else // Photo-only / empty-text messages have no stable text key → classify directly, no cache. - match (if String.IsNullOrEmpty msg.Text then None else Some (md5Hex msg.Text)) with + // (Unchanged by the media-placeholder prompt rendering above — hasStableTextCacheKey + // reads msg.Text, never the placeholder; see that function's doc comment.) + match (if hasStableTextCacheKey msg then Some (md5Hex msg.Text) else None) with | None -> return! classifyUncached msg userMsgCount NoCache ct | Some hash -> let senderKey = sprintf "text:%d:%s" msg.SenderId hash diff --git a/src/VahterBanBot/Types.fs b/src/VahterBanBot/Types.fs index 44c35da..d57350a 100644 --- a/src/VahterBanBot/Types.fs +++ b/src/VahterBanBot/Types.fs @@ -210,6 +210,14 @@ type VahterAction = type AutoDeleteReason = | MlSpam of {| score: float |} + /// The kill decision came from LLM triage (LlmVerdict.Kill — the LLM itself said SPAM), not + /// from crossing the ML score threshold on its own. Distinct from MlSpam so stats/rendering + /// don't mislabel an LLM call as a plain ML verdict — see the 2026-08-18 incident + /// (@AvaloniaRU msg 217142): an innocent caption-less sticker was auto-deleted with + /// `reason = MlSpam` even though the LLM, not the ML threshold, made the kill call. + /// `score` is still the ML score that triggered LLM escalation (for the same human-facing + /// "score: x" rendering as MlSpam); `modelName` names which deployment decided. + | LlmSpam of {| score: float; modelName: string |} | ReactionSpam of {| reactionCount: int |} | InvisibleMention /// Ban-seeded spam-text cache hit (see SpamTextCache.fs) — the normalized text exactly diff --git a/tests/VahterBanBot.Tests/EventSerializationTests.fs b/tests/VahterBanBot.Tests/EventSerializationTests.fs index 56d7788..fdc0e94 100644 --- a/tests/VahterBanBot.Tests/EventSerializationTests.fs +++ b/tests/VahterBanBot.Tests/EventSerializationTests.fs @@ -193,6 +193,57 @@ let ``LlmReactionTriageClassified round-trips with reason and shadowMode`` () = Assert.True(e.shadowMode) | other -> Assert.Fail $"Expected LlmReactionTriageClassified but got {other}" +// --------------------------------------------------------------------------- +// AutoDeleteReason.LlmSpam — 2026-08-18 deletion-reason attribution fix (@AvaloniaRU msg 217142: +// an LLM kill verdict was recorded/rendered as a plain MlSpam verdict). Old stored events with +// `reason.Case = "MlSpam"` must keep deserializing exactly as before — LlmSpam is purely additive. +// --------------------------------------------------------------------------- + +[] +let ``New BotAutoDeleted with LlmSpam reason round-trips with score and modelName`` () = + let original = + BotAutoDeleted {| chatId = -666L; messageId = 217142L; userId = 8931498652L; reason = AutoDeleteReason.LlmSpam {| score = 0.31478; modelName = "gpt-4o-mini" |} |} + let json = JsonSerializer.Serialize(original, eventJsonOpts) + let roundtripped = JsonSerializer.Deserialize(json, eventJsonOpts) + match roundtripped with + | BotAutoDeleted e -> + Assert.Equal(-666L, e.chatId) + Assert.Equal(217142L, e.messageId) + match e.reason with + | AutoDeleteReason.LlmSpam r -> + Assert.Equal(0.31478, r.score) + Assert.Equal("gpt-4o-mini", r.modelName) + | other -> Assert.Fail $"Expected AutoDeleteReason.LlmSpam but got {other}" + | other -> Assert.Fail $"Expected BotAutoDeleted but got {other}" + +[] +let ``Old BotAutoDeleted event with reason.Case=MlSpam (pre-LlmSpam) still deserializes`` () = + // Simulates an event stored in the DB before AutoDeleteReason.LlmSpam existed. + let json = + """{"Case":"BotAutoDeleted","chatId":-666,"messageId":217142,"userId":8931498652,"reason":{"Case":"MlSpam","score":0.31478}}""" + let event = JsonSerializer.Deserialize(json, eventJsonOpts) + match event with + | BotAutoDeleted e -> + Assert.Equal(-666L, e.chatId) + match e.reason with + | AutoDeleteReason.MlSpam r -> Assert.Equal(0.31478, r.score) + | other -> Assert.Fail $"Expected AutoDeleteReason.MlSpam but got {other}" + | other -> Assert.Fail $"Expected BotAutoDeleted but got {other}" + +[] +let ``BotAutoDeleted with LlmSpam reason folds into Moderation and FoldTimeline just like MlSpam`` () = + let llmDeleted = + FromModeration ( + BotAutoDeleted {| chatId = -1L; messageId = 1; userId = 5L; reason = AutoDeleteReason.LlmSpam {| score = 0.31478; modelName = "gpt-4o-mini" |} |}) + let recv = FromMessage (MessageReceived {| chatId = -1L; messageId = 1; userId = 5L; text = Some "x"; rawMessage = "{}" |}) + let m = [ recv; llmDeleted ] |> List.fold (fun s e -> Message.FoldTimeline(s, e)) Message.Zero + Assert.Equal(SpamClassification.Spam, m.Classification) + + let moderation = + [ BotAutoDeleted {| chatId = -1L; messageId = 1; userId = 5L; reason = AutoDeleteReason.LlmSpam {| score = 0.31478; modelName = "gpt-4o-mini" |} |} ] + |> List.fold (fun s e -> Moderation.Fold(s, e)) Moderation.Zero + Assert.Equal(1, moderation.BotAutoDeletedCount) + [] let ``Old UserUnbanned without actor deserializes correctly`` () = let json = diff --git a/tests/VahterBanBot.Unit.Tests/LlmMediaPlaceholderTests.fs b/tests/VahterBanBot.Unit.Tests/LlmMediaPlaceholderTests.fs new file mode 100644 index 0000000..2bbbd2b --- /dev/null +++ b/tests/VahterBanBot.Unit.Tests/LlmMediaPlaceholderTests.fs @@ -0,0 +1,102 @@ +/// Pure unit coverage for `LlmTriage.mediaPlaceholder` / `hasStableTextCacheKey` / `formatBioLine` +/// — extracted specifically so the 2026-08-18 empty-text-media incident fix (@AvaloniaRU msg +/// 217142: an innocent caption-less sticker's blank `Message:` body let gpt-4o-mini judge SPAM +/// on sender signals alone) is unit-testable without a live Azure client. See LlmTriage.fs's +/// "Empty-text media placeholder" section for the full incident/cache-key writeup. +module VahterBanBot.Unit.Tests.LlmMediaPlaceholderTests + +open BotTestInfra +open Funogram.Telegram.Types +open VahterBanBot +open VahterBanBot.LlmTriage +open Xunit + +let private msgOf (update: Funogram.Telegram.Types.Update) = TgMessage.Create(update.Message.Value) + +[] +let ``sticker with no emoji and no set_name (2026-08-12 prod spam sticker shape): generic placeholder`` () = + let sticker = Tg.staticSticker() // no emoji/set_name — see StickerOcrTests.fs's doc comment + let msg = msgOf (Tg.quickMsg(sticker = sticker, text = null)) + Assert.Equal(Some "[sticker, no readable text]", mediaPlaceholder msg) + +[] +let ``sticker with emoji and set_name: placeholder names both`` () = + let sticker = + Sticker.Create( + fileId = "cat-sticker", fileUniqueId = "cat-sticker-uid", ``type`` = "regular", + width = 512L, height = 512L, isAnimated = false, isVideo = false, + emoji = "🐈‍⬛️", setName = "catssenseoflife") + let msg = msgOf (Tg.quickMsg(sticker = sticker, text = null)) + Assert.Equal(Some "[sticker \"🐈‍⬛️\" from set \"catssenseoflife\", no readable text]", mediaPlaceholder msg) + +[] +let ``sticker with emoji only (no set_name): placeholder degrades gracefully`` () = + let sticker = + Sticker.Create( + fileId = "s", fileUniqueId = "s-uid", ``type`` = "regular", + width = 512L, height = 512L, isAnimated = false, isVideo = false, + emoji = "😀") + let msg = msgOf (Tg.quickMsg(sticker = sticker, text = null)) + Assert.Equal(Some "[sticker \"😀\", no readable text]", mediaPlaceholder msg) + +[] +let ``photo with no OCR text: generic photo placeholder`` () = + let msg = msgOf (Tg.quickMsg(text = null, photos = [| Tg.spamPhoto |])) + Assert.Equal(Some "[photo, no readable text]", mediaPlaceholder msg) + +[] +let ``truly empty message (no text, no media): empty-message placeholder`` () = + let msg = msgOf (Tg.quickMsg(text = null)) + Assert.Equal(Some "[empty message]", mediaPlaceholder msg) + +[] +let ``message with real text: no placeholder, caller uses msg.Text as-is`` () = + let msg = msgOf (Tg.quickMsg(text = "buy crypto now")) + Assert.Equal(None, mediaPlaceholder msg) + +[] +let ``mediaPlaceholder never mutates msg.Text — it stays null for a sticker-only message`` () = + // Critical invariant (see LlmTriage.fs's module doc comment above mediaPlaceholder): the + // placeholder must exist ONLY in the LLM prompt string. msg.Text also feeds the ML scorer, + // spam-text cache, verdict-cache key, and the deleted-spam channel post — if the placeholder + // ever mutated msg.Text, every photo/sticker would collapse onto ONE cache key and (since + // SPAM/SKIP is cached globally by text hash) one verdict would condemn every future photo. + let sticker = Tg.staticSticker() + let msg = msgOf (Tg.quickMsg(sticker = sticker, text = null)) + let before = msg.Text + Assert.Null(before) + let placeholder = mediaPlaceholder msg + Assert.True(placeholder.IsSome) + Assert.Null(msg.Text) // unchanged after computing the placeholder + Assert.Null(msg.OriginalText) // and the original wire text is untouched too + +[] +let ``hasStableTextCacheKey: true when msg.Text is non-empty`` () = + let msg = msgOf (Tg.quickMsg(text = "buy crypto now")) + Assert.True(hasStableTextCacheKey msg) + +[] +let ``hasStableTextCacheKey: false for an empty-text message (photo-only)`` () = + let msg = msgOf (Tg.quickMsg(text = null, photos = [| Tg.spamPhoto |])) + Assert.False(hasStableTextCacheKey msg) + +[] +let ``sticker-only message that gets a rendered placeholder still has no stable text cache key (NoCache branch)`` () = + // Proves the placeholder-rendering change does NOT widen what Classify treats as cacheable: + // a message that gets a non-None mediaPlaceholder (because msg.Text is empty) must still + // report false here, so Classify's `if hasStableTextCacheKey msg then ... else NoCache` + // routes it to NoCache exactly as before this change. + let sticker = Tg.staticSticker() + let msg = msgOf (Tg.quickMsg(sticker = sticker, text = null)) + Assert.True((mediaPlaceholder msg).IsSome) + Assert.False(hasStableTextCacheKey msg) + +[] +let ``formatBioLine: null/empty/whitespace bio renders as (none)`` () = + Assert.Equal("(none)", formatBioLine null) + Assert.Equal("(none)", formatBioLine "") + Assert.Equal("(none)", formatBioLine " ") + +[] +let ``formatBioLine: a real bio is rendered verbatim`` () = + Assert.Equal("Зайди в мой био", formatBioLine "Зайди в мой био") diff --git a/tests/VahterBanBot.Unit.Tests/SpamDeleteReasonTests.fs b/tests/VahterBanBot.Unit.Tests/SpamDeleteReasonTests.fs new file mode 100644 index 0000000..d70ff68 --- /dev/null +++ b/tests/VahterBanBot.Unit.Tests/SpamDeleteReasonTests.fs @@ -0,0 +1,31 @@ +/// Pure unit coverage for `Bot.spamDeleteReason` — the 2026-08-18 deletion-reason attribution +/// fix. Before this fix, `AutoVerdict.Spam`'s enforcement site always recorded `AutoDeleteReason. +/// MlSpam`, even when `Actor.LLM` (i.e. `LlmVerdict.Kill`) made the actual kill call — see the +/// @AvaloniaRU msg 217142 incident, where a kitten-sticker false positive was logged as a plain +/// ML-threshold verdict when the LLM had in fact decided. +module VahterBanBot.Unit.Tests.SpamDeleteReasonTests + +open VahterBanBot.Bot +open VahterBanBot.Types +open Xunit + +[] +let ``LLM kill verdict (Actor.LLM) attributes to LlmSpam, carrying score and modelName`` () = + let actor = Actor.LLM {| modelName = "gpt-4o-mini"; promptHash = "abc123" |} + match spamDeleteReason 0.31478 actor with + | AutoDeleteReason.LlmSpam r -> + Assert.Equal(0.31478, r.score) + Assert.Equal("gpt-4o-mini", r.modelName) + | other -> Assert.Fail $"Expected LlmSpam but got {other}" + +[] +let ``plain ML-threshold verdict (Actor.ML) attributes to MlSpam`` () = + match spamDeleteReason 0.87 Actor.ML with + | AutoDeleteReason.MlSpam r -> Assert.Equal(0.87, r.score) + | other -> Assert.Fail $"Expected MlSpam but got {other}" + +[] +let ``any non-LLM actor (defensive: AutoVerdict.Spam never actually carries these) falls back to MlSpam`` () = + match spamDeleteReason 0.6 (Actor.Bot None) with + | AutoDeleteReason.MlSpam r -> Assert.Equal(0.6, r.score) + | other -> Assert.Fail $"Expected MlSpam but got {other}" diff --git a/tests/VahterBanBot.Unit.Tests/VahterBanBot.Unit.Tests.fsproj b/tests/VahterBanBot.Unit.Tests/VahterBanBot.Unit.Tests.fsproj index 06cca1b..01c91be 100644 --- a/tests/VahterBanBot.Unit.Tests/VahterBanBot.Unit.Tests.fsproj +++ b/tests/VahterBanBot.Unit.Tests/VahterBanBot.Unit.Tests.fsproj @@ -13,8 +13,10 @@ + +