Skip to content

Commit 616b990

Browse files
SamSaffronSamSaffron
authored
FEATURE: LLM mentions and auto silence (#949)
* FEATURE: allow mentioning an LLM mid conversation to switch This is a edgecase feature that allow you to start a conversation in a PM with LLM1 and then use LLM2 to evaluation or continue the conversation * FEATURE: allow auto silencing of spam accounts New rule can also allow for silencing an account automatically This can prevent spammers from creating additional posts.
1 parent 6c25718 commit 616b990

File tree

6 files changed

+112
-5
lines changed

6 files changed

+112
-5
lines changed

‎config/locales/server.en.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ en:
44
flag_types:
55
review: "Add post to review queue"
66
spam: "Flag as spam and hide post"
7+
spam_silence: "Flag as spam, hide post and silence user"
78
scriptables:
89
llm_triage:
910
title: Triage posts using AI

‎lib/ai_bot/playground.rb

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,16 @@ def self.schedule_reply(post)
7878
bot_user = nil
7979
mentioned = nil
8080

81-
all_llm_user_ids = LlmModel.joins(:user).pluck("users.id")
81+
all_llm_users =
82+
LlmModel
83+
.where(enabled_chat_bot: true)
84+
.joins(:user)
85+
.pluck("users.id", "users.username_lower")
8286

8387
if post.topic.private_message?
8488
# this is an edge case, you started a PM with a different bot
85-
bot_user = post.topic.topic_allowed_users.where(user_id: all_llm_user_ids).first&.user
89+
bot_user =
90+
post.topic.topic_allowed_users.where(user_id: all_llm_users.map(&:first)).first&.user
8691
bot_user ||=
8792
post
8893
.topic
@@ -92,14 +97,17 @@ def self.schedule_reply(post)
9297
&.user
9398
end
9499

95-
if mentionables.present?
100+
mentions = nil
101+
if mentionables.present? || (bot_user && post.topic.private_message?)
96102
mentions = post.mentions.map(&:downcase)
97103

98104
# in case we are replying to a post by a bot
99105
if post.reply_to_post_number && post.reply_to_post&.user
100106
mentions << post.reply_to_post.user.username_lower
101107
end
108+
end
102109

110+
if mentionables.present?
103111
mentioned = mentionables.find { |mentionable| mentions.include?(mentionable[:username]) }
104112

105113
# direct PM to mentionable
@@ -117,7 +125,9 @@ def self.schedule_reply(post)
117125
end
118126

119127
if bot_user
120-
persona_id = mentioned&.dig(:id) || post.topic.custom_fields["ai_persona_id"]
128+
topic_persona_id = post.topic.custom_fields["ai_persona_id"]
129+
persona_id = mentioned&.dig(:id) || topic_persona_id
130+
121131
persona = nil
122132

123133
if persona_id
@@ -130,6 +140,19 @@ def self.schedule_reply(post)
130140
DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, name: persona_name)
131141
end
132142

143+
# edge case, llm was mentioned in an ai persona conversation
144+
if persona_id == topic_persona_id.to_i && post.topic.private_message? && persona &&
145+
all_llm_users.present?
146+
if !persona.force_default_llm && mentions.present?
147+
mentioned_llm_user_id, _ =
148+
all_llm_users.find { |id, username| mentions.include?(username) }
149+
150+
if mentioned_llm_user_id
151+
bot_user = User.find_by(id: mentioned_llm_user_id) || bot_user
152+
end
153+
end
154+
end
155+
133156
persona ||= DiscourseAi::AiBot::Personas::General
134157

135158
bot_user = User.find(persona.user_id) if persona && persona.force_default_llm

‎lib/automation.rb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ def self.flag_types
66
[
77
{ id: "review", translated_name: I18n.t("discourse_automation.ai.flag_types.review") },
88
{ id: "spam", translated_name: I18n.t("discourse_automation.ai.flag_types.spam") },
9+
{
10+
id: "spam_silence",
11+
translated_name: I18n.t("discourse_automation.ai.flag_types.spam_silence"),
12+
},
913
]
1014
end
1115
def self.available_models

‎lib/automation/llm_triage.rb

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,16 @@ def self.handle(
8383
.sub("%%AUTOMATION_ID%%", automation&.id.to_s)
8484
.sub("%%AUTOMATION_NAME%%", automation&.name.to_s)
8585

86-
if flag_type == :spam
86+
if flag_type == :spam || flag_type == :spam_silence
8787
PostActionCreator.new(
8888
Discourse.system_user,
8989
post,
9090
PostActionType.types[:spam],
9191
message: score_reason,
9292
queue_for_review: true,
9393
).perform
94+
95+
SpamRule::AutoSilence.new(post.user, post).silence_user if flag_type == :spam_silence
9496
else
9597
reviewable =
9698
ReviewablePost.needs_review!(target: post, created_by: Discourse.system_user)

‎spec/lib/modules/ai_bot/playground_spec.rb

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -622,6 +622,65 @@
622622
expect(post.topic.posts.last.post_number).to eq(1)
623623
end
624624

625+
it "allows swapping a llm mid conversation using a mention" do
626+
SiteSetting.ai_bot_enabled = true
627+
628+
post = nil
629+
DiscourseAi::Completions::Llm.with_prepared_responses(
630+
["Yes I can", "Magic Title"],
631+
llm: "custom:#{claude_2.id}",
632+
) do
633+
post =
634+
create_post(
635+
title: "I just made a PM",
636+
raw: "Hey there #{persona.user.username}, can you help me?",
637+
target_usernames: "#{user.username},#{persona.user.username}",
638+
archetype: Archetype.private_message,
639+
user: admin,
640+
)
641+
end
642+
643+
post.topic.custom_fields["ai_persona_id"] = persona.id
644+
post.topic.save_custom_fields
645+
646+
llm2 = Fabricate(:llm_model, enabled_chat_bot: true)
647+
648+
llm2.toggle_companion_user
649+
650+
DiscourseAi::Completions::Llm.with_prepared_responses(
651+
["Hi from bot two"],
652+
llm: "custom:#{llm2.id}",
653+
) do
654+
create_post(
655+
user: admin,
656+
raw: "hi @#{llm2.user.username.capitalize} how are you",
657+
topic_id: post.topic_id,
658+
)
659+
end
660+
661+
last_post = post.topic.reload.posts.order("id desc").first
662+
expect(last_post.raw).to eq("Hi from bot two")
663+
expect(last_post.user_id).to eq(persona.user_id)
664+
665+
# tether llm, so it can no longer be switched
666+
persona.update!(force_default_llm: true, default_llm: "custom:#{claude_2.id}")
667+
668+
DiscourseAi::Completions::Llm.with_prepared_responses(
669+
["Hi from bot one"],
670+
llm: "custom:#{claude_2.id}",
671+
) do
672+
create_post(
673+
user: admin,
674+
raw: "hi @#{llm2.user.username.capitalize} how are you",
675+
topic_id: post.topic_id,
676+
)
677+
end
678+
679+
last_post = post.topic.reload.posts.order("id desc").first
680+
expect(last_post.raw).to eq("Hi from bot one")
681+
expect(last_post.user_id).to eq(persona.user_id)
682+
end
683+
625684
it "allows PMing a persona even when no particular bots are enabled" do
626685
SiteSetting.ai_bot_enabled = true
627686
toggle_enabled_bots(bots: [])

‎spec/lib/modules/automation/llm_triage_spec.rb

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,24 @@ def triage(**args)
110110
expect(post.topic.reload.visible).to eq(false)
111111
end
112112

113+
it "can handle spam+silence flags" do
114+
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
115+
triage(
116+
post: post,
117+
model: "custom:#{llm_model.id}",
118+
system_prompt: "test %%POST%%",
119+
search_for_text: "bad",
120+
flag_post: true,
121+
flag_type: :spam_silence,
122+
automation: nil,
123+
)
124+
end
125+
126+
expect(post.reload).to be_hidden
127+
expect(post.topic.reload.visible).to eq(false)
128+
expect(post.user.silenced?).to eq(true)
129+
end
130+
113131
it "can handle garbled output from LLM" do
114132
DiscourseAi::Completions::Llm.with_prepared_responses(["Bad.\n\nYo"]) do
115133
triage(

0 commit comments

Comments
 (0)