feat(GenerativeAI): Add new sample for embeddings feature (GoogleCloudPlatform#12629)

* Add new code samples
* Add minor code refactoring & improving visibility

Author: Valeriy-Burlaka

generative_ai/embeddings/batch_example.py

@@ -28,7 +28,7 @@ def embed_text_batch() -> BatchPredictionJob:
     import vertexai
     from vertexai.preview import language_models
 
-    # TODO(developer): Uncomment and set your project ID
+    # TODO(developer): Update & uncomment line below
     # PROJECT_ID = "your-project-id"
     vertexai.init(project=PROJECT_ID, location="us-central1")
     input_uri = (
@@ -38,7 +38,7 @@ def embed_text_batch() -> BatchPredictionJob:
     output_uri = OUTPUT_URI
 
     textembedding_model = language_models.TextEmbeddingModel.from_pretrained(
-        "textembedding-gecko"
+        "textembedding-gecko@003"
     )
 
     batch_prediction_job = textembedding_model.batch_predict(

generative_ai/embeddings/batch_example_test.py

@@ -13,32 +13,33 @@
 # limitations under the License.
 
 # [START generativeaionvertexai_embedding]
-from typing import List, Optional
+from __future__ import annotations
 
 from vertexai.language_models import TextEmbeddingInput, TextEmbeddingModel
 
 
-def embed_text(
-    texts: list = None,
-    task: str = "RETRIEVAL_DOCUMENT",
-    dimensionality: Optional[int] = 256,
-) -> List[List[float]]:
+def embed_text() -> list[list[float]]:
     """Embeds texts with a pre-trained, foundational model.
-    Args:
-        texts (List[str]): A list of texts to be embedded.
-        task (str): The task type for embedding. Check the available tasks in the model's documentation.
-        dimensionality (Optional[int]): The dimensionality of the output embeddings.
+
     Returns:
-        List[List[float]]: A list of lists containing the embedding vectors for each input text
+        A list of lists containing the embedding vectors for each input text
     """
-    if texts is None:
-        texts = ["banana muffins? ", "banana bread? banana muffins?"]
+
+    # A list of texts to be embedded.
+    texts = ["banana muffins? ", "banana bread? banana muffins?"]
+    # The dimensionality of the output embeddings.
+    dimensionality = 256
+    # The task type for embedding. Check the available tasks in the model's documentation.
+    task = "RETRIEVAL_DOCUMENT"
+
     model = TextEmbeddingModel.from_pretrained("text-embedding-004")
     inputs = [TextEmbeddingInput(text, task) for text in texts]
     kwargs = dict(output_dimensionality=dimensionality) if dimensionality else {}
     embeddings = model.get_embeddings(inputs, **kwargs)
+
+    print(embeddings)
     # Example response:
-    # [[0.006135190837085247, -0.01462465338408947, 0.004978656303137541, ...],
+    # [[0.006135190837085247, -0.01462465338408947, 0.004978656303137541, ...], [0.1234434666, ...]],
     return [embedding.values for embedding in embeddings]
 
 

generative_ai/embeddings/document_retrieval_example.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from vertexai.vision_models import MultiModalEmbeddingResponse
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def generate_embeddings_with_lower_dimension() -> MultiModalEmbeddingResponse:
+    """Example of how to use lower dimensions when creating multimodal embeddings.
+
+    Read more @ https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings#low-dimension
+
+    Returns:
+        The multimodal embedding response.
+    """
+    # [START generativeaionvertexai_embeddings_specify_lower_dimension]
+    import vertexai
+
+    from vertexai.vision_models import Image, MultiModalEmbeddingModel
+
+    # TODO(developer): Update & uncomment line below
+    # PROJECT_ID = "your-project-id"
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # TODO(developer): Try different dimenions: 128, 256, 512, 1408
+    embedding_dimension = 128
+
+    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
+    image = Image.load_from_file(
+        "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png")
+
+    embeddings = model.get_embeddings(
+        image=image,
+        contextual_text="Colosseum",
+        dimension=embedding_dimension,
+    )
+
+    print(f"Image Embedding: {embeddings.image_embedding}")
+    print(f"Text Embedding: {embeddings.text_embedding}")
+
+    # Example response:
+    # Image Embedding: [0.0622573346, -0.0406507477, 0.0260440577, ...]
+    # Text Embedding: [0.27469793, -0.146258667, 0.0222803634, ...]
+
+    # [END generativeaionvertexai_embeddings_specify_lower_dimension]
+
+    return embeddings
+
+
+if __name__ == "__main__":
+    generate_embeddings_with_lower_dimension()

generative_ai/embeddings/generate_embeddings_with_lower_dimension.py

@@ -22,19 +22,19 @@
 def get_image_video_text_embeddings() -> MultiModalEmbeddingResponse:
     """Example of how to generate multimodal embeddings from image, video, and text.
 
-    Read more @ https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-multimodal-embeddings#video-best-practices
+    Read more @ https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings#img-txt-vid-embedding
     """
     # [START generativeaionvertexai_multimodal_embedding_image_video_text]
     import vertexai
 
     from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
     from vertexai.vision_models import VideoSegmentConfig
 
-    # TODO(developer): Uncomment and set your project ID
+    # TODO(developer): Update & uncomment line below
     # PROJECT_ID = "your-project-id"
     vertexai.init(project=PROJECT_ID, location="us-central1")
 
-    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
+    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
 
     image = Image.load_from_file(
         "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"

generative_ai/embeddings/model_tuning_test.py

@@ -19,20 +19,20 @@
 PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
 
 
-def get_image_embeddings() -> MultiModalEmbeddingResponse:
+def get_image_text_embeddings() -> MultiModalEmbeddingResponse:
     """Example of how to generate multimodal embeddings from image and text.
 
-    Read more @ https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-multimodal-embeddings#low-dimension
+    Read more @ https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings#text-image-embedding
     """
     # [START generativeaionvertexai_multimodal_embedding_image]
     import vertexai
     from vertexai.vision_models import Image, MultiModalEmbeddingModel
 
-    # TODO(developer): Uncomment and set your project ID
+    # TODO(developer): Update & uncomment line below
     # PROJECT_ID = "your-project-id"
     vertexai.init(project=PROJECT_ID, location="us-central1")
 
-    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
+    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
     image = Image.load_from_file(
         "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
     )
@@ -54,4 +54,4 @@ def get_image_embeddings() -> MultiModalEmbeddingResponse:
 
 
 if __name__ == "__main__":
-    get_image_embeddings()
+    get_image_text_embeddings()

generative_ai/embeddings/multimodal_example.py

@@ -19,28 +19,27 @@
 
 
 def get_video_embeddings() -> MultiModalEmbeddingResponse:
-    """Example of how to generate multimodal embeddings from video and text.
+    """Example of how to use the multimodal embedding model to get embeddings only for video content.
 
-    Read more at https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-multimodal-embeddings#video-best-practices
+    Read more at https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings#vid-embedding
     """
     # [START generativeaionvertexai_multimodal_embedding_video]
     import vertexai
 
     from vertexai.vision_models import MultiModalEmbeddingModel, Video
     from vertexai.vision_models import VideoSegmentConfig
 
-    # TODO(developer): Uncomment and set your project ID
+    # TODO(developer): Update & uncomment line below
     # PROJECT_ID = "your-project-id"
     vertexai.init(project=PROJECT_ID, location="us-central1")
 
-    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
+    model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding@001")
 
     embeddings = model.get_embeddings(
         video=Video.load_from_file(
             "gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4"
         ),
         video_segment_config=VideoSegmentConfig(end_offset_sec=1),
-        contextual_text="Cars on Highway",
     )
 
     # Video Embeddings are segmented based on the video_segment_config.
@@ -51,12 +50,10 @@ def get_video_embeddings() -> MultiModalEmbeddingResponse:
         )
         print(f"Embedding: {video_embedding.embedding}")
 
-    print(f"Text Embedding: {embeddings.text_embedding}")
     # Example response:
     # Video Embeddings:
     # Video Segment: 0.0 - 1.0
     # Embedding: [-0.0206376351, 0.0123456789, ...]
-    # Text Embedding: [-0.0207006913, -0.00251061679, ...]
 
     # [END generativeaionvertexai_multimodal_embedding_video]