feat: GenAI SDK client - Add automatic candidate naming and creation timestamp to evaluation dataset metadata

PiperOrigin-RevId: 773786705

Author: jsondai

tests/unit/vertexai/genai/test_evals.py

@@ -154,6 +154,49 @@ def test_inference_with_string_model_success(
                 }
             ),
         )
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
+
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
+    def test_inference_with_callable_model_sets_candidate_name(
+        self, mock_eval_dataset_loader
+    ):
+        mock_df = pd.DataFrame({"prompt": ["test prompt"]})
+        mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
+            orient="records"
+        )
+
+        def my_model_fn(contents):
+            return "callable response"
+
+        inference_result = self.client.evals.run_inference(
+            model=my_model_fn,
+            src=mock_df,
+        )
+        assert inference_result.candidate_name == "my_model_fn"
+        assert inference_result.gcs_source is None
+
+    @mock.patch.object(_evals_utils, "EvalDatasetLoader")
+    def test_inference_with_lambda_model_candidate_name_is_none(
+        self, mock_eval_dataset_loader
+    ):
+        mock_df = pd.DataFrame({"prompt": ["test prompt"]})
+        mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
+            orient="records"
+        )
+
+        inference_result = self.client.evals.run_inference(
+            model=lambda x: "lambda response",  # pylint: disable=unnecessary-lambda
+            src=mock_df,
+        )
+        # Lambdas may or may not have a __name__ depending on Python version/env
+        # but it's typically '<lambda>' if it exists.
+        # The code under test uses getattr(model, "__name__", None)
+        assert (
+            inference_result.candidate_name == "<lambda>"
+            or inference_result.candidate_name is None
+        )
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
     def test_inference_with_callable_model_success(self, mock_eval_dataset_loader):
@@ -179,6 +222,8 @@ def mock_model_fn(contents):
                 }
             ),
         )
+        assert inference_result.candidate_name == "mock_model_fn"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -224,6 +269,8 @@ def test_inference_with_prompt_template(
                 }
             ),
         )
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -273,6 +320,10 @@ def test_inference_with_gcs_destination(
         pd.testing.assert_frame_equal(
             inference_result.eval_dataset_df, expected_df_to_save
         )
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source == vertexai_genai_types.GcsSource(
+            uris=[gcs_dest_path]
+        )
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -322,6 +373,8 @@ def test_inference_with_local_destination(
             }
         )
         pd.testing.assert_frame_equal(inference_result.eval_dataset_df, expected_df)
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -405,6 +458,8 @@ def test_inference_from_request_column_save_locally(
             expected_records, key=lambda x: x["request"]
         )
         os.remove(local_dest_path)
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     def test_inference_from_local_jsonl_file(self, mock_models):
@@ -478,6 +533,8 @@ def test_inference_from_local_jsonl_file(self, mock_models):
             any_order=True,
         )
         os.remove(local_src_path)
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     def test_inference_from_local_csv_file(self, mock_models):
@@ -548,6 +605,8 @@ def test_inference_from_local_csv_file(self, mock_models):
             any_order=True,
         )
         os.remove(local_src_path)
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -719,6 +778,8 @@ def mock_generate_content_logic(*args, **kwargs):
             expected_df.sort_values(by="id").reset_index(drop=True),
             check_dtype=False,
         )
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
     @mock.patch.object(_evals_common, "Models")
     @mock.patch.object(_evals_utils, "EvalDatasetLoader")
@@ -794,6 +855,8 @@ def test_inference_with_multimodal_content(
                 }
             ),
         )
+        assert inference_result.candidate_name == "gemini-pro"
+        assert inference_result.gcs_source is None
 
 
 class TestMetricPromptBuilder:
@@ -3295,3 +3358,76 @@ def test_execute_evaluation_multiple_datasets(
         assert summary_metric.mean_score == 1.0
 
         assert mock_eval_dependencies["mock_evaluate_instances"].call_count == 2
+
+    def test_execute_evaluation_deduplicates_candidate_names(
+        self, mock_api_client_fixture, mock_eval_dependencies
+    ):
+        """Tests that duplicate candidate names are indexed."""
+        dataset1 = vertexai_genai_types.EvaluationDataset(
+            eval_dataset_df=pd.DataFrame(
+                [{"prompt": "p1", "response": "r1", "reference": "ref1"}]
+            ),
+            candidate_name="gemini-pro",
+        )
+        dataset2 = vertexai_genai_types.EvaluationDataset(
+            eval_dataset_df=pd.DataFrame(
+                [{"prompt": "p1", "response": "r2", "reference": "ref1"}]
+            ),
+            candidate_name="gemini-flash",
+        )
+        dataset3 = vertexai_genai_types.EvaluationDataset(
+            eval_dataset_df=pd.DataFrame(
+                [{"prompt": "p1", "response": "r3", "reference": "ref1"}]
+            ),
+            candidate_name="gemini-pro",
+        )
+
+        mock_eval_dependencies[
+            "mock_evaluate_instances"
+        ].return_value = vertexai_genai_types.EvaluateInstancesResponse(
+            exact_match_results=vertexai_genai_types.ExactMatchResults(
+                exact_match_metric_values=[
+                    vertexai_genai_types.ExactMatchMetricValue(score=1.0)
+                ]
+            )
+        )
+
+        result = _evals_common._execute_evaluation(
+            api_client=mock_api_client_fixture,
+            dataset=[dataset1, dataset2, dataset3],
+            metrics=[vertexai_genai_types.Metric(name="exact_match")],
+        )
+
+        assert result.metadata.candidate_names == [
+            "gemini-pro #1",
+            "gemini-flash",
+            "gemini-pro #2",
+        ]
+
+    @mock.patch("vertexai._genai._evals_common.datetime")
+    def test_execute_evaluation_adds_creation_timestamp(
+        self, mock_datetime, mock_api_client_fixture, mock_eval_dependencies
+    ):
+        """Tests that creation_timestamp is added to the result metadata."""
+        import datetime
+
+        mock_now = datetime.datetime(
+            2025, 6, 18, 12, 0, 0, tzinfo=datetime.timezone.utc
+        )
+        mock_datetime.datetime.now.return_value = mock_now
+
+        dataset = vertexai_genai_types.EvaluationDataset(
+            eval_dataset_df=pd.DataFrame(
+                [{"prompt": "p", "response": "r", "reference": "r"}]
+            )
+        )
+        metric = vertexai_genai_types.Metric(name="exact_match")
+
+        result = _evals_common._execute_evaluation(
+            api_client=mock_api_client_fixture,
+            dataset=dataset,
+            metrics=[metric],
+        )
+
+        assert result.metadata is not None
+        assert result.metadata.creation_timestamp == mock_now

vertexai/_genai/_evals_common.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 #
 """Common utilities for evals."""
+import collections
 import concurrent.futures
+import datetime
 import json
 import logging
 import os
@@ -475,6 +477,17 @@ def _execute_inference(
     end_time = time.time()
     logger.info("Inference completed in %.2f seconds.", end_time - start_time)
 
+    candidate_name = None
+    if isinstance(model, str):
+        candidate_name = model
+    elif callable(model):
+        candidate_name = getattr(model, "__name__", None)
+
+    evaluation_dataset = types.EvaluationDataset(
+        eval_dataset_df=results_df,
+        candidate_name=candidate_name,
+    )
+
     if dest:
         file_name = "inference_results.jsonl"
         full_dest_path = dest
@@ -500,13 +513,14 @@ def _execute_inference(
                     file_type="jsonl",
                 )
                 logger.info("Results saved to GCS: %s", full_dest_path)
+                evaluation_dataset.gcs_source = types.GcsSource(uris=[full_dest_path])
             else:
                 results_df.to_json(full_dest_path, orient="records", lines=True)
                 logger.info("Results saved locally to: %s", full_dest_path)
         except Exception as e:  # pylint: disable=broad-exception-caught
             logger.error("Failed to save results to %s. Error: %s", full_dest_path, e)
 
-    return types.EvaluationDataset(eval_dataset_df=results_df)
+    return evaluation_dataset
 
 
 def _get_dataset_source(
@@ -690,6 +704,19 @@ def _execute_evaluation(
             f"Unsupported dataset type: {type(dataset)}. Must be an"
             " EvaluationDataset or a list of EvaluationDataset."
         )
+    original_candidate_names = [
+        ds.candidate_name or f"candidate_{i+1}" for i, ds in enumerate(dataset_list)
+    ]
+    name_counts = collections.Counter(original_candidate_names)
+    deduped_candidate_names = []
+    current_name_counts = collections.defaultdict(int)
+
+    for name in original_candidate_names:
+        if name_counts[name] > 1:
+            current_name_counts[name] += 1
+            deduped_candidate_names.append(f"{name} #{current_name_counts[name]}")
+        else:
+            deduped_candidate_names.append(name)
 
     loader = _evals_utils.EvalDatasetLoader(api_client=api_client)
     processed_eval_dataset, num_response_candidates = _resolve_dataset_inputs(
@@ -714,6 +741,17 @@ def _execute_evaluation(
     logger.info("Evaluation took: %f seconds", t2 - t1)
 
     evaluation_result.evaluation_dataset = dataset_list
+
+    if not evaluation_result.metadata:
+        evaluation_result.metadata = types.EvaluationRunMetadata()
+
+    evaluation_result.metadata.creation_timestamp = datetime.datetime.now(
+        datetime.timezone.utc
+    )
+
+    if deduped_candidate_names:
+        evaluation_result.metadata.candidate_names = deduped_candidate_names
+
     logger.info("Evaluation run completed.")
 
     if dest:

vertexai/_genai/types.py

@@ -2064,6 +2064,10 @@ class EvaluationDataset(_common.BaseModel):
         default=None,
         description="""The evaluation dataset in the form of a Pandas DataFrame.""",
     )
+    candidate_name: Optional[str] = Field(
+        default=None,
+        description="""The name of the candidate model or agent for this evaluation dataset.""",
+    )
     gcs_source: Optional[GcsSource] = Field(
         default=None,
         description="""The GCS source for the evaluation dataset.""",
@@ -2089,6 +2093,9 @@ class EvaluationDatasetDict(TypedDict, total=False):
     eval_dataset_df: Optional[pd.DataFrame]
     """The evaluation dataset in the form of a Pandas DataFrame."""
 
+    candidate_name: Optional[str]
+    """The name of the candidate model or agent for this evaluation dataset."""
+
     gcs_source: Optional[GcsSourceDict]
     """The GCS source for the evaluation dataset."""