Skip to content

Commit ea2cb42

Browse files
feat(Gen AI): Add new sample for Evaluation feature (GoogleCloudPlatform#12602)
* Add sample for Gen AI 'Evaluation - get rouge score' section * refresh CI builds * Update comments to follow the runnable-snippets-plan standard * refresh CI builds * refresh CI checks * move get_rouge_score sample into a dedicated dir
1 parent 16e3610 commit ea2cb42

File tree

2 files changed

+124
-0
lines changed

2 files changed

+124
-0
lines changed
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import os
15+
16+
from vertexai.preview.evaluation import EvalResult
17+
18+
PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
19+
20+
21+
def get_rouge_score() -> EvalResult:
22+
# [START generativeaionvertexai_evaluation_get_rouge_score]
23+
import pandas as pd
24+
25+
import vertexai
26+
from vertexai.preview.evaluation import EvalTask
27+
28+
# TODO(developer): Update & uncomment line below
29+
# PROJECT_ID = "your-project-id"
30+
vertexai.init(project=PROJECT_ID, location="us-central1")
31+
32+
text_to_summarize = """
33+
The Great Barrier Reef, located off the coast of Queensland in northeastern
34+
Australia, is the world's largest coral reef system. Stretching over 2,300
35+
kilometers, it is composed of over 2,900 individual reefs and 900 islands.
36+
The reef is home to a wide variety of marine life, including many endangered
37+
species. However, climate change, ocean acidification, and coral bleaching
38+
pose significant threats to its ecosystem."""
39+
40+
prompt = f"Summarize the following text:\n\n{text_to_summarize}"
41+
42+
reference_summarization = """
43+
The Great Barrier Reef, the world's largest coral reef system, is
44+
located off the coast of Queensland, Australia. It's a vast
45+
ecosystem spanning over 2,300 kilometers with thousands of reefs
46+
and islands. While it harbors an incredible diversity of marine
47+
life, including endangered species, it faces serious threats from
48+
climate change, ocean acidification, and coral bleaching."""
49+
50+
# Use pre-generated model responses to compare different summarization outputs
51+
# against a consistent reference.
52+
eval_dataset = pd.DataFrame(
53+
{
54+
"prompt": [prompt] * 3,
55+
"response": [
56+
"""The Great Barrier Reef, the world's largest coral reef system located
57+
in Australia, is a vast and diverse ecosystem. However, it faces serious
58+
threats from climate change, ocean acidification, and coral bleaching,
59+
endangering its rich marine life.""",
60+
"""The Great Barrier Reef, a vast coral reef system off the coast of
61+
Queensland, Australia, is the world's largest. It's a complex ecosystem
62+
supporting diverse marine life, including endangered species. However,
63+
climate change, ocean acidification, and coral bleaching are serious
64+
threats to its survival.""",
65+
"""The Great Barrier Reef, the world's largest coral reef system off the
66+
coast of Australia, is a vast and diverse ecosystem with thousands of
67+
reefs and islands. It is home to a multitude of marine life, including
68+
endangered species, but faces serious threats from climate change, ocean
69+
acidification, and coral bleaching.""",
70+
],
71+
"reference": [reference_summarization] * 3,
72+
}
73+
)
74+
75+
eval_task = EvalTask(
76+
dataset=eval_dataset,
77+
metrics=[
78+
"rouge_1",
79+
"rouge_2",
80+
"rouge_l",
81+
"rouge_l_sum",
82+
],
83+
)
84+
result = eval_task.evaluate()
85+
86+
print("Summary Metrics:\n")
87+
88+
for key, value in result.summary_metrics.items():
89+
print(f"{key}: \t{value}")
90+
91+
print("\n\nMetrics Table:\n")
92+
print(result.metrics_table)
93+
# Example response:
94+
# prompt ... rouge_1/score rouge_2/score ...
95+
# 0 Summarize the following text:\n\n\n ... 0.659794 0.484211 ...
96+
# 1 Summarize the following text:\n\n\n ... 0.704762 0.524272 ...
97+
# ...
98+
99+
# [END generativeaionvertexai_evaluation_get_rouge_score]
100+
return result
101+
102+
103+
if __name__ == "__main__":
104+
get_rouge_score()
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import get_rouge_score
16+
17+
18+
def test_create_evaluation_task() -> None:
19+
response = get_rouge_score.get_rouge_score()
20+
assert response

0 commit comments

Comments
 (0)