FEATURE: Skip PM scanning in LLM triage by default (#966)

SamSaffron · web-flow · commit fbcc8e493a19 · 2024-11-28T09:25:29.000+11:00
Usually people do not want to scan personal messages. Sometimes
they may. In that case they can enable triage on personal messages.
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
@@ -113,6 +113,9 @@ en:
             flag_post:
               label: "Flag post"
               description: "Flags post (either as spam or for review)"
+            include_personal_messages:
+              label: "Include personal messages"
+              description: "Also scan and triage personal messages"
             model:
               label: "Model"
               description: "Language model used for triage"
diff --git a/discourse_automation/llm_triage.rb b/discourse_automation/llm_triage.rb
@@ -22,6 +22,7 @@
     field :tags, component: :tags
     field :hide_topic, component: :boolean
     field :flag_post, component: :boolean
+    field :include_personal_messages, component: :boolean
     field :flag_type,
           component: :choices,
           required: false,
@@ -54,6 +55,11 @@
 
       max_post_tokens = nil if max_post_tokens <= 0
 
+      if post.topic.private_message?
+        include_personal_messages = fields.dig("include_personal_messages", "value")
+        next if !include_personal_messages
+      end
+
       begin
         RateLimiter.new(
           Discourse.system_user,
diff --git a/spec/lib/discourse_automation/llm_triage_spec.rb b/spec/lib/discourse_automation/llm_triage_spec.rb
@@ -5,6 +5,8 @@
 describe DiscourseAi::Automation::LlmTriage do
   fab!(:category)
   fab!(:reply_user) { Fabricate(:user) }
+  fab!(:personal_message) { Fabricate(:private_message_topic) }
+  let(:canned_reply_text) { "Hello, this is a reply" }
 
   let(:automation) { Fabricate(:automation, script: "llm_triage", enabled: true) }
 
@@ -30,7 +32,7 @@ def add_automation_field(name, value, type: "text")
     add_automation_field("tags", %w[aaa bbb], type: "tags")
     add_automation_field("hide_topic", true, type: "boolean")
     add_automation_field("flag_post", true, type: "boolean")
-    add_automation_field("canned_reply", "Yo this is a reply")
+    add_automation_field("canned_reply", canned_reply_text)
     add_automation_field("canned_reply_user", reply_user.username, type: "user")
     add_automation_field("max_post_tokens", 100)
   end
@@ -63,7 +65,7 @@ def add_automation_field(name, value, type: "text")
     expect(topic.tags.pluck(:name)).to contain_exactly("aaa", "bbb")
     expect(topic.visible).to eq(false)
     reply = topic.posts.order(:post_number).last
-    expect(reply.raw).to eq("Yo this is a reply")
+    expect(reply.raw).to eq(canned_reply_text)
     expect(reply.user.id).to eq(reply_user.id)
 
     ai_log = AiApiAuditLog.order("id desc").first
@@ -79,6 +81,30 @@ def add_automation_field(name, value, type: "text")
     expect(count).to be > (50)
   end
 
+  it "does not triage PMs by default" do
+    post = Fabricate(:post, topic: personal_message)
+    automation.running_in_background!
+    automation.trigger!({ "post" => post })
+
+    # nothing should happen, no classification, its a PM
+  end
+
+  it "will triage PMs if automation allows it" do
+    # needs to be admin or it will not be able to just step in to
+    # PM
+    reply_user.update!(admin: true)
+    add_automation_field("include_personal_messages", true, type: :boolean)
+    post = Fabricate(:post, topic: personal_message)
+
+    DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
+      automation.running_in_background!
+      automation.trigger!({ "post" => post })
+    end
+
+    last_post = post.topic.reload.posts.order(:post_number).last
+    expect(last_post.raw).to eq(canned_reply_text)
+  end
+
   it "does not reply to the canned_reply_user" do
     post = Fabricate(:post, user: reply_user)
 
