FIX: Default prompts have to be explicit about the expected format.

romanrizzi · romanrizzi · commit 000f318b80c3 · 2025-08-20T11:37:03.000-03:00
Wrapping the expected response with `&lt;output&gt;` tags confuses models, especially those from the Claude family, which don't have schema support and rely on prefilling. Relying on prefilling means they only know the response must start with `{`, and how the JSON looks is only hinted at in the system prompt.
diff --git a/plugins/discourse-ai/lib/personas/concept_deduplicator.rb b/plugins/discourse-ai/lib/personas/concept_deduplicator.rb
@@ -32,16 +32,17 @@ def system_prompt
           Example Output:
           AI, AJAX, API, APK, APT Package Manager, ARIA, ARM Architecture, AWS, Abbreviations
 
-          Please provide your streamlined list of tags within <streamlined_tags> key.
+          Please provide your streamlined list of tags within "streamlined_tags" key.
 
           Remember, the goal is to create a more focused and effective set of tags while maintaining the essence of the original list.
 
           Your output should be in the following format:
-            <o>
-              {
-                "streamlined_tags": ["tag1", "tag3"]
-              }
-            </o>
+
+          {
+            "streamlined_tags": ["tag1", "tag3"]
+          }
+
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/concept_finder.rb b/plugins/discourse-ai/lib/personas/concept_finder.rb
@@ -33,11 +33,11 @@ def system_prompt
           #{existing_concepts_text}
           Format your response as a JSON object with a single key named "concepts", which has an array of concept strings as the value.
           Your output should be in the following format:
-            <o>
-              {"concepts": ["concept1", "concept2", "concept3"]}
-            </o>
+
+          {"concepts": ["concept1", "concept2", "concept3"]}
 
           Where the concepts are replaced by the actual concepts you've identified.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/concept_matcher.rb b/plugins/discourse-ai/lib/personas/concept_matcher.rb
@@ -27,11 +27,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "matching_concepts", which has an array of concept strings from the provided list.
           Your output should be in the following format:
-            <o>
-              {"matching_concepts": ["concept1", "concept3", "concept5"]}
-            </o>
+
+          {"matching_concepts": ["concept1", "concept3", "concept5"]}
 
           Only include concepts from the provided list that match the content. If no concepts match, return an empty array.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/content_creator.rb b/plugins/discourse-ai/lib/personas/content_creator.rb
@@ -17,11 +17,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the created content.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the content.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/custom_prompt.rb b/plugins/discourse-ai/lib/personas/custom_prompt.rb
@@ -14,11 +14,12 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the result as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
+
 
           Where "xx" is replaced by the result.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/image_captioner.rb b/plugins/discourse-ai/lib/personas/image_captioner.rb
@@ -13,11 +13,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the caption as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the caption.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/markdown_table_generator.rb b/plugins/discourse-ai/lib/personas/markdown_table_generator.rb
@@ -13,11 +13,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the formatted table as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the formatted table.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/post_illustrator.rb b/plugins/discourse-ai/lib/personas/post_illustrator.rb
@@ -14,11 +14,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the generated prompt as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the generated prompt.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/proofreader.rb b/plugins/discourse-ai/lib/personas/proofreader.rb
@@ -15,11 +15,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the proofreaded version as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the proofreaded version.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/short_summarizer.rb b/plugins/discourse-ai/lib/personas/short_summarizer.rb
@@ -24,11 +24,12 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "summary", which has the summary as the value.
           Your output should be in the following format:
-            <output>
-              {"summary": "xx"}
-            </output>
+
+          {"summary": "xx"}
+
 
           Where "xx" is replaced by the summary.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/smart_dates.rb b/plugins/discourse-ai/lib/personas/smart_dates.rb
@@ -24,11 +24,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the formatted result as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+
+          {"output": "xx"}
 
           Where "xx" is replaced by the formatted result.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/spam_detector.rb b/plugins/discourse-ai/lib/personas/spam_detector.rb
@@ -46,11 +46,12 @@ def system_prompt
 
           Format your response as a JSON object with a one key named "spam", which is a boolean that indicates if a post is spam or legitimate.
           Your output should be in the following format:
-            <output>
-              {"spam": xx}
-            </output>
+
+          {"spam": xx}
 
           Where xx is true if the post is spam, or false if it's legitimate.
+          reply with valid JSON only
+
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/summarizer.rb b/plugins/discourse-ai/lib/personas/summarizer.rb
@@ -24,11 +24,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "summary", which has the summary as the value.
           Your output should be in the following format:
-            <output>
-              {"summary": "xx"}
-            </output>
+          
+          {"summary": "xx"}
 
           Where "xx" is replaced by the summary.
+          reply with valid JSON only
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/lib/personas/tutor.rb b/plugins/discourse-ai/lib/personas/tutor.rb
@@ -22,11 +22,11 @@ def system_prompt
 
           Format your response as a JSON object with a single key named "output", which has the explanation as the value.
           Your output should be in the following format:
-            <output>
-              {"output": "xx"}
-            </output>
+          
+          {"output": "xx"}
 
           Where "xx" is replaced by the explanation.
+          reply with valid JSON only.
         PROMPT
       end
 
diff --git a/plugins/discourse-ai/spec/lib/personas/concept_deduplicator_spec.rb b/plugins/discourse-ai/spec/lib/personas/concept_deduplicator_spec.rb
@@ -15,10 +15,8 @@
     let(:prompt) { persona.system_prompt }
 
     it "specifies output format" do
-      expect(prompt).to include("<streamlined_tags>")
-      expect(prompt).to include("<o>")
+      expect(prompt).to include("streamlined_tags")
       expect(prompt).to include('"streamlined_tags": ["tag1", "tag3"]')
-      expect(prompt).to include("</o>")
     end
   end
 
diff --git a/plugins/discourse-ai/spec/lib/personas/concept_finder_spec.rb b/plugins/discourse-ai/spec/lib/personas/concept_finder_spec.rb
@@ -43,9 +43,7 @@
     it "includes format instructions" do
       prompt = persona.system_prompt
 
-      expect(prompt).to include("<o>")
       expect(prompt).to include('{"concepts": ["concept1", "concept2", "concept3"]}')
-      expect(prompt).to include("</o>")
     end
 
     it "includes language preservation instruction" do
diff --git a/plugins/discourse-ai/spec/lib/personas/concept_matcher_spec.rb b/plugins/discourse-ai/spec/lib/personas/concept_matcher_spec.rb
@@ -20,9 +20,7 @@
 
     it "specifies output format" do
       expect(prompt).to include("matching_concepts")
-      expect(prompt).to include("<o>")
       expect(prompt).to include('{"matching_concepts": ["concept1", "concept3", "concept5"]}')
-      expect(prompt).to include("</o>")
     end
   end
 
