From c8d7ea4ee6371adb181c0e7f6586158dee61750f Mon Sep 17 00:00:00 2001 From: Roman Rizzi Date: Thu, 31 Jul 2025 11:49:41 -0300 Subject: [PATCH 1/2] FEATURE: Use a persona when running the AI triage automation script --- .../discourse-ai/config/locales/client.en.yml | 12 +--- ...80444_seed_personas_from_triage_scripts.rb | 72 +++++++++++++++++++ .../discourse_automation/llm_triage.rb | 24 +++---- .../discourse-ai/lib/ai_helper/assistant.rb | 2 +- .../discourse-ai/lib/automation/llm_triage.rb | 64 ++++++++++------- .../lib/embeddings/semantic_search.rb | 2 +- plugins/discourse-ai/lib/summarization.rb | 2 +- .../discourse_automation/llm_triage_spec.rb | 29 ++++---- .../lib/modules/automation/llm_triage_spec.rb | 49 ++++++------- 9 files changed, 159 insertions(+), 97 deletions(-) create mode 100644 plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb diff --git a/plugins/discourse-ai/config/locales/client.en.yml b/plugins/discourse-ai/config/locales/client.en.yml index eecd0ad1230c2..619d9e859191c 100644 --- a/plugins/discourse-ai/config/locales/client.en.yml +++ b/plugins/discourse-ai/config/locales/client.en.yml @@ -132,9 +132,9 @@ en: description: "In silent mode persona will receive the content but will not post anything on the forum - useful when performing triage using tools" llm_triage: fields: - system_prompt: - label: "System Prompt" - description: "The prompt that will be used to triage, be sure for it to reply with a single word you can use to trigger the action" + triage_persona: + label: "Persona" + description: "Persona used to triage, be sure for it to reply with a single word you can use to trigger the action" max_post_tokens: label: "Max Post Tokens" description: "The maximum number of tokens to scan using LLM triage" @@ -174,12 +174,6 @@ en: reply_persona: label: "Reply Persona" description: "AI Persona to use for replies (must have default LLM), will be prioritized over canned reply" - model: - label: "Model" - description: "Language model used for triage" - temperature: - label: "Temperature" - description: "Temperature to use for the LLM. Increase to increase randomness (leave empty to use model default)" max_output_tokens: label: "Max output tokens" description: "When specified, sets an upper bound to the maximum number of tokens the model can generate. Respects LLM's max output tokens limit" diff --git a/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb b/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb new file mode 100644 index 0000000000000..88e4d35cbb66d --- /dev/null +++ b/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class SeedPersonasFromTriageScripts < ActiveRecord::Migration[8.0] + def up + script_fields = DB.query <<~SQL + SELECT fields.id, fields.name, (fields.metadata->>'value') AS value, automations.name AS automation_name, automations.id AS automation_id + FROM discourse_automation_fields fields + INNER JOIN discourse_automation_automations automations ON automations.id = fields.automation_id + WHERE fields.name IN ('model', 'system_prompt', 'temperature') + AND automations.script = 'llm_triage' + SQL + + return if script_fields.empty? + + script_fields = + script_fields.reduce({}) do |acc, field| + id = field.automation_id + acc[id] = { "automation_id" => id, "name" => field.automation_name } if acc[id].nil? + + acc[field.automation_id].merge!(field.name => field.value) + + acc + end + + automation_to_persona_ids = + script_fields.transform_values do |field| + if field["system_prompt"].blank? + nil + else + name = + ( + if field["name"].pesent? + "#{field["name"]} triage automation" + else + "Unnamed triage automation script ID #{field["automation_id"]}" + end + ) + temp = field["temperature"] || "NULL" + row = + "'#{name}', 'Seeded Persona for an LLM Triage script', FALSE, '#{field["system_prompt"]}', #{temp}, #{field["model"]}, NOW(), NOW()" + + DB.query_single(<<~SQL)&.first + INSERT INTO ai_personas (name, description, enabled, system_prompt, temperature, default_llm_id, created_at, updated_at) + VALUES (#{row}) + RETURNING id + SQL + end + end + + pp automation_to_persona_ids + + new_fields = + automation_to_persona_ids + .map do |k, v| + if v.blank? + nil + else + "(#{k}, 'triage_persona', json_build_object('value', #{v}), 'choices', 'script', NOW(), NOW())" + end + end + .compact + + DB.exec <<~SQL + INSERT INTO discourse_automation_fields (automation_id, name, metadata, component, target, created_at, updated_at) + VALUES #{new_fields.join(",")} + SQL + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/plugins/discourse-ai/discourse_automation/llm_triage.rb b/plugins/discourse-ai/discourse_automation/llm_triage.rb index cda48c84760ba..253021ff88f8b 100644 --- a/plugins/discourse-ai/discourse_automation/llm_triage.rb +++ b/plugins/discourse-ai/discourse_automation/llm_triage.rb @@ -13,17 +13,19 @@ field :include_personal_messages, component: :boolean # Inputs - field :model, + field :triage_persona, component: :choices, required: true, extra: { - content: DiscourseAi::Automation.available_models, + content: + DiscourseAi::Automation.available_persona_choices( + require_user: false, + require_default_llm: true, + ), } - field :system_prompt, component: :message, required: false field :search_for_text, component: :text, required: true field :max_post_tokens, component: :text field :stop_sequences, component: :text_list, required: false - field :temperature, component: :text field :max_output_tokens, component: :text # Actions @@ -60,6 +62,8 @@ next if !include_personal_messages end + triage_persona_id = fields.dig("triage_persona", "value") + canned_reply = fields.dig("canned_reply", "value") canned_reply_user = fields.dig("canned_reply_user", "value") reply_persona_id = fields.dig("reply_persona", "value") @@ -69,9 +73,7 @@ next if post.user.username == canned_reply_user next if post.raw.strip == canned_reply.to_s.strip - system_prompt = fields.dig("system_prompt", "value") search_for_text = fields.dig("search_for_text", "value") - model = fields.dig("model", "value") category_id = fields.dig("category", "value") tags = fields.dig("tags", "value") @@ -79,12 +81,6 @@ flag_post = fields.dig("flag_post", "value") flag_type = fields.dig("flag_type", "value") max_post_tokens = fields.dig("max_post_tokens", "value").to_i - temperature = fields.dig("temperature", "value") - if temperature == "" || temperature.nil? - temperature = nil - else - temperature = temperature.to_f - end max_output_tokens = fields.dig("max_output_tokens", "value").to_i max_output_tokens = nil if max_output_tokens <= 0 @@ -110,9 +106,8 @@ DiscourseAi::Automation::LlmTriage.handle( post: post, - model: model, + triage_persona_id: triage_persona_id, search_for_text: search_for_text, - system_prompt: system_prompt, category_id: category_id, tags: tags, canned_reply: canned_reply, @@ -125,7 +120,6 @@ max_post_tokens: max_post_tokens, stop_sequences: stop_sequences, automation: self.automation, - temperature: temperature, max_output_tokens: max_output_tokens, action: context["action"], ) diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rb index 01a27675ea473..3926f2a792eb0 100644 --- a/plugins/discourse-ai/lib/ai_helper/assistant.rb +++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb @@ -312,7 +312,7 @@ def find_ai_helper_model(helper_mode, persona_klass) # Priorities are: # 1. Persona's default LLM - # 2. SiteSetting.ai_default_llm_id (or newest LLM if not set) + # 2. SiteSetting.ai_default_llm_model (or newest LLM if not set) def self.find_ai_helper_model(helper_mode, persona_klass) model_id = persona_klass.default_llm_id || SiteSetting.ai_default_llm_model diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rb index 93f06712f39dc..934b45996f357 100644 --- a/plugins/discourse-ai/lib/automation/llm_triage.rb +++ b/plugins/discourse-ai/lib/automation/llm_triage.rb @@ -5,9 +5,8 @@ module Automation module LlmTriage def self.handle( post:, - model:, + triage_persona_id:, search_for_text:, - system_prompt:, category_id: nil, tags: nil, canned_reply: nil, @@ -18,7 +17,6 @@ def self.handle( automation: nil, max_post_tokens: nil, stop_sequences: nil, - temperature: nil, whisper: nil, reply_persona_id: nil, max_output_tokens: nil, @@ -34,42 +32,55 @@ def self.handle( return end - llm = DiscourseAi::Completions::Llm.proxy(model) + triage_persona = AiPersona.find(triage_persona_id) + model_id = triage_persona.default_llm_id || SiteSetting.ai_default_llm_model + return if model_id.blank? + model = LlmModel.find(model_id) - s_prompt = system_prompt.to_s.sub("%%POST%%", "") # Backwards-compat. We no longer sub this. - prompt = DiscourseAi::Completions::Prompt.new(s_prompt) + bot = + DiscourseAi::Personas::Bot.as( + Discourse.system_user, + persona: triage_persona.class_instance.new, + model: model, + ) - content = "title: #{post.topic.title}\n#{post.raw}" + input = "title: #{post.topic.title}\n#{post.raw}" - content = - llm.tokenizer.truncate( - content, + input = + model.tokenizer_class.truncate( + input, max_post_tokens, strict: SiteSetting.ai_strict_token_counting, ) if max_post_tokens.present? if post.upload_ids.present? - content = [content] - content.concat(post.upload_ids.map { |upload_id| { upload_id: upload_id } }) + input = [input] + input.concat(post.upload_ids.map { |upload_id| { upload_id: upload_id } }) end - prompt.push(type: :user, content: content) + bot_ctx = + DiscourseAi::Personas::BotContext.new( + user: Discourse.system_user, + skip_tool_details: true, + feature_name: "llm_triage", + messages: [{ type: :user, content: input }], + ) result = nil - result = - llm.generate( - prompt, - max_tokens: max_output_tokens, - temperature: temperature, - user: Discourse.system_user, - stop_sequences: stop_sequences, - feature_name: "llm_triage", - feature_context: { - automation_id: automation&.id, - automation_name: automation&.name, - }, - )&.strip + llm_args = { + max_tokens: max_output_tokens, + stop_sequences: stop_sequences, + feature_context: { + automation_id: automation&.id, + automation_name: automation&.name, + }, + } + + result = +"" + bot.reply(bot_ctx, llm_args: llm_args) do |partial, _, type| + result << partial if type.blank? + end if result.present? && result.downcase.include?(search_for_text.downcase) user = User.find_by_username(canned_reply_user) if canned_reply_user.present? @@ -92,6 +103,7 @@ def self.handle( end elsif canned_reply.present? && action != :edit post_type = whisper ? Post.types[:whisper] : Post.types[:regular] + PostCreator.create!( user, topic_id: post.topic_id, diff --git a/plugins/discourse-ai/lib/embeddings/semantic_search.rb b/plugins/discourse-ai/lib/embeddings/semantic_search.rb index 6d44c1f7356dc..1cf75a42a5661 100644 --- a/plugins/discourse-ai/lib/embeddings/semantic_search.rb +++ b/plugins/discourse-ai/lib/embeddings/semantic_search.rb @@ -201,7 +201,7 @@ def hypothetical_post_from(search_term) # Priorities are: # 1. Persona's default LLM - # 2. SiteSetting.ai_default_llm_id (or newest LLM if not set) + # 2. SiteSetting.ai_default_llm_model (or newest LLM if not set) def find_ai_hyde_model(persona_klass) model_id = persona_klass.default_llm_id || SiteSetting.ai_default_llm_model diff --git a/plugins/discourse-ai/lib/summarization.rb b/plugins/discourse-ai/lib/summarization.rb index f8f71cab2da3c..97be410ed3f32 100644 --- a/plugins/discourse-ai/lib/summarization.rb +++ b/plugins/discourse-ai/lib/summarization.rb @@ -54,7 +54,7 @@ def chat_channel_summary(channel, time_window_in_hours) # Priorities are: # 1. Persona's default LLM - # 2. SiteSetting.ai_default_llm_id (or newest LLM if not set) + # 2. SiteSetting.ai_default_llm_model (or newest LLM if not set) def find_summarization_model(persona_klass) model_id = persona_klass.default_llm_id || SiteSetting.ai_default_llm_model diff --git a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb index 44ece3a0ef50b..f31489da4b9aa 100644 --- a/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb +++ b/plugins/discourse-ai/spec/lib/discourse_automation/llm_triage_spec.rb @@ -11,6 +11,7 @@ let(:automation) { Fabricate(:automation, script: "llm_triage", enabled: true) } fab!(:llm_model) + fab!(:ai_persona) def add_automation_field(name, value, type: "text") automation.fields.create!( @@ -27,9 +28,11 @@ def add_automation_field(name, value, type: "text") enable_current_plugin SiteSetting.tagging_enabled = true - add_automation_field("system_prompt", "hello %%POST%%") + + ai_persona.update!(default_llm: llm_model) + + add_automation_field("triage_persona", ai_persona.id) add_automation_field("search_for_text", "bad") - add_automation_field("model", llm_model.id) add_automation_field("category", category.id, type: "category") add_automation_field("tags", %w[aaa bbb], type: "tags") add_automation_field("hide_topic", true, type: "boolean") @@ -42,21 +45,17 @@ def add_automation_field(name, value, type: "text") it "can trigger via automation" do post = Fabricate(:post, raw: "hello " * 5000) - body = { - model: "gpt-3.5-turbo-0301", - usage: { - prompt_tokens: 337, - completion_tokens: 162, - total_tokens: 499, - }, - choices: [ - { message: { role: "assistant", content: "bad" }, finish_reason: "stop", index: 0 }, - ], - }.to_json + chunks = <<~RESPONSE + data: {"id":"chatcmpl-B2VwlY6KzSDtHvg8pN1VAfRhhLFgn","object":"chat.completion.chunk","created":1739939159,"model": "gpt-3.5-turbo-0301","service_tier":"default","system_fingerprint":"fp_ef58bd3122","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-B2VwlY6KzSDtHvg8pN1VAfRhhLFgn","object":"chat.completion.chunk","created":1739939159,"model": "gpt-3.5-turbo-0301","service_tier":"default","system_fingerprint":"fp_ef58bd3122","choices":[{"index":0,"delta":{"content":"bad"},"finish_reason":null}],"usage":null} + + data: [DONE] + RESPONSE WebMock.stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return( status: 200, - body: body, + body: chunks, ) automation.running_in_background! @@ -96,7 +95,7 @@ def add_automation_field(name, value, type: "text") # PM reply_user.update!(admin: true) add_automation_field("include_personal_messages", true, type: :boolean) - add_automation_field("temperature", "0.2") + ai_persona.update!(temperature: 0.2) add_automation_field("max_output_tokens", "700") post = Fabricate(:post, topic: personal_message) diff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb index bb8073a3ab805..c51251b2b2c21 100644 --- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb +++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb @@ -4,19 +4,23 @@ fab!(:reply) { Fabricate(:post, topic: post.topic, user: Fabricate(:user)) } fab!(:llm_model) + fab!(:ai_persona) + def triage(**args) DiscourseAi::Automation::LlmTriage.handle(**args) end - before { enable_current_plugin } + before do + enable_current_plugin + ai_persona.update!(default_llm: llm_model) + end it "does nothing if it does not pass triage" do DiscourseAi::Completions::Llm.with_prepared_responses(["good"]) do triage( post: post, - model: llm_model.id.to_s, + triage_persona_id: ai_persona.id, hide_topic: true, - system_prompt: "test %%POST%%", search_for_text: "bad", automation: nil, ) @@ -29,9 +33,8 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, + triage_persona_id: ai_persona.id, hide_topic: true, - system_prompt: "test %%POST%%", search_for_text: "bad", automation: nil, ) @@ -46,9 +49,8 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, + triage_persona_id: ai_persona.id, category_id: category.id, - system_prompt: "test %%POST%%", search_for_text: "bad", automation: nil, ) @@ -62,8 +64,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", canned_reply: "test canned reply 123", canned_reply_user: user.username, @@ -81,8 +82,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, automation: nil, @@ -99,8 +99,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, flag_type: :spam, @@ -116,8 +115,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, flag_type: :spam_silence, @@ -134,8 +132,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, flag_type: :review_hide, @@ -160,8 +157,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, flag_type: :spam_silence, @@ -176,8 +172,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["Bad.\n\nYo"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, automation: nil, @@ -193,8 +188,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "BAD", flag_post: true, automation: nil, @@ -212,8 +206,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, automation: nil, @@ -231,8 +224,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do |spy| triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, automation: nil, @@ -251,8 +243,7 @@ def triage(**args) DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do triage( post: post, - model: llm_model.id.to_s, - system_prompt: "test %%POST%%", + triage_persona_id: ai_persona.id, search_for_text: "bad", flag_post: true, tags: [tag_2.name], From b75cf8e3bbef9c537731cbec4cd3c539d546cb1c Mon Sep 17 00:00:00 2001 From: Roman Rizzi Date: Thu, 31 Jul 2025 14:59:07 -0300 Subject: [PATCH 2/2] update plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb Co-authored-by: Rafael dos Santos Silva --- .../20250721080444_seed_personas_from_triage_scripts.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb b/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb index 88e4d35cbb66d..f436c41c4f3b7 100644 --- a/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb +++ b/plugins/discourse-ai/db/migrate/20250721080444_seed_personas_from_triage_scripts.rb @@ -29,7 +29,7 @@ def up else name = ( - if field["name"].pesent? + if field["name"].present? "#{field["name"]} triage automation" else "Unnamed triage automation script ID #{field["automation_id"]}" @@ -47,8 +47,6 @@ def up end end - pp automation_to_persona_ids - new_fields = automation_to_persona_ids .map do |k, v|