fix(generative-ai): Fix flaky tests (GoogleCloudPlatform#11757)

gericdong · web-flow · commit ecabbe688cd2 · 2024-05-13T19:15:48.000-07:00
* fix(generative-ai): Fix a flaky test

* Skip PDF sample test for now as model output is likely blocked.

* Skip reasoning engine advanced sample test

* Remove unreliable asserts

* Skip reasoning engine basic test for now
diff --git a/generative_ai/gemini_reasoning_engine_test.py b/generative_ai/gemini_reasoning_engine_test.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import os
-import sys
+
 from typing import Generator
 
 import pytest
@@ -37,13 +37,12 @@ def reasoning_engine_id() -> Generator[str, None, None]:
     )
 
 
-@pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="requires Python version lower than 3.12"
-)
+@pytest.mark.skip("TODO: Reasoning Engine Deployment Issue b/339643184")
 def test_create_reasoning_engine_basic(reasoning_engine_id: str) -> None:
     assert reasoning_engine_id
 
 
+@pytest.mark.skip("TODO: Reasoning Engine Deployment Issue b/339643184")
 def test_create_reasoning_engine_advanced() -> None:
     reasoning_engine = gemini_reasoning_engine.create_reasoning_engine_advanced(
         PROJECT_ID, REGION, STAGING_BUCKET
diff --git a/generative_ai/test_gemini_examples.py b/generative_ai/test_gemini_examples.py
@@ -44,7 +44,6 @@ def test_gemini_guide_example() -> None:
     text = gemini_guide_example.generate_text(PROJECT_ID)
     text = text.lower()
     assert len(text) > 0
-    assert "scones" in text
 
 
 def test_gemini_text_input_example() -> None:
@@ -69,7 +68,6 @@ def test_gemini_pro_config_example() -> None:
         text = gemini_pro_config_example.generate_text(PROJECT_ID)
         text = text.lower()
         assert len(text) > 0
-        assert any(e in text for e in ("blueberry", "coffee", "flower", "table"))
 
         # clean-up
         os.remove(fname)
@@ -107,6 +105,7 @@ def test_gemini_single_turn_video_example() -> None:
     assert any([_ in text for _ in ("zoo", "tiger", "leaf", "water")])
 
 
+@pytest.mark.skip("TODO: Safty filters are likely blocking model output b/339985493")
 def test_gemini_pdf_example() -> None:
     text = gemini_pdf_example.analyze_pdf(PROJECT_ID)
     assert len(text) > 0
