gemini/evaluation/gemini_function_calling_evaluation_rapid_evaluation_sdk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OsXAs2gcIpbC"
   },
   "outputs": [],
   "source": [
    "# Copyright 2024 Google LLC\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Evaluate Generative Model Tool Use | Rapid Evaluation SDK Tutorial \n",
    "\n",
    " <table align=\"left\">\n",
    "  <td style=\"text-align: center\">\n",
    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/gemini_function_calling_evaluation_rapid_evaluation_sdk.ipynb\">\n",
    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
    "    </a>\n",
    "  </td>\n",
    "  <td style=\"text-align: center\">\n",
    "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fgemini_function_calling_evaluation_rapid_evaluation_sdk.ipynb\">\n",
    "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
    "    </a>\n",
    "  </td>    \n",
    "  <td style=\"text-align: center\">\n",
    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/gemini_function_calling_evaluation_rapid_evaluation_sdk.ipynb\">\n",
    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Workbench\n",
    "    </a>\n",
    "  </td>\n",
    "  <td style=\"text-align: center\">\n",
    "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/gemini_function_calling_evaluation_rapid_evaluation_sdk.ipynb\">\n",
    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
    "    </a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0922hY8RI11o"
   },
   "source": [
    "| | |\n",
    "|-|-|\n",
    "|Author(s) | [Jason Dai](https://github.com/jsondai) |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview\n",
    "\n",
    "* Define an API function and a Tool for Gemini model, and evaluate the Gemini tool use quality."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting Started"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Install Vertex AI SDK for Rapid Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6DnneQRilTFW"
   },
   "outputs": [],
   "source": [
    "%pip install --upgrade --user --quiet google-cloud-aiplatform[rapid_evaluation]\n",
    "%pip install --quiet --upgrade nest_asyncio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restart runtime\n",
    "To use the newly installed packages in this Jupyter runtime, you must restart the runtime. You can do this by running the cell below, which restarts the current kernel.\n",
    "\n",
    "The restart might take a minute or longer. After it's restarted, continue to the next step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import IPython\n",
    "\n",
    "# app = IPython.Application.instance()\n",
    "# app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"alert alert-block alert-warning\">\n",
    "<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
    "</div>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Authenticate your notebook environment (Colab only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "if \"google.colab\" in sys.modules:\n",
    "    from google.colab import auth\n",
    "\n",
    "    auth.authenticate_user()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Google Cloud project information and initialize Vertex AI SDK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
    "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
    "\n",
    "\n",
    "import vertexai\n",
    "\n",
    "vertexai.init(project=PROJECT_ID, location=LOCATION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eHDPrnbQlTFX"
   },
   "outputs": [],
   "source": [
    "# General\n",
    "import inspect\n",
    "from uuid import uuid4\n",
    "from google.colab import auth\n",
    "from IPython.display import display, Markdown, HTML\n",
    "import json\n",
    "import plotly.graph_objects as go\n",
    "import matplotlib.pyplot as plt\n",
    "import logging\n",
    "from tqdm import tqdm\n",
    "import nest_asyncio\n",
    "import warnings\n",
    "import random\n",
    "import string\n",
    "import os\n",
    "\n",
    "# Main\n",
    "import vertexai\n",
    "from vertexai.preview.evaluation import (\n",
    "    EvalTask,\n",
    "    PromptTemplate,\n",
    "    CustomMetric,\n",
    "    make_metric,\n",
    ")\n",
    "import pandas as pd\n",
    "from google.cloud import aiplatform\n",
    "from vertexai.generative_models import GenerativeModel, HarmCategory, HarmBlockThreshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GEv1ZxXPlTFX"
   },
   "source": [
    "### Library settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ilX3gNbllTFX"
   },
   "outputs": [],
   "source": [
    "logging.getLogger(\"urllib3.connectionpool\").setLevel(logging.ERROR)\n",
    "nest_asyncio.apply()\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o4nrCUUnlTFX"
   },
   "source": [
    "### Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HohglC48lTFX"
   },
   "outputs": [],
   "source": [
    "def generate_uuid(length: int = 8) -> str:\n",
    "    \"\"\"Generate a uuid of a specified length (default=8).\"\"\"\n",
    "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
    "\n",
    "\n",
    "def print_doc(function):\n",
    "    print(f\"{function.__name__}:\\n{inspect.getdoc(function)}\\n\")\n",
    "\n",
    "\n",
    "def display_eval_report(eval_result, metrics=None):\n",
    "    \"\"\"Display the evaluation results.\"\"\"\n",
    "\n",
    "    title, summary_metrics, report_df = eval_result\n",
    "    metrics_df = pd.DataFrame.from_dict(summary_metrics, orient=\"index\").T\n",
    "    if metrics:\n",
    "        metrics_df = metrics_df.filter(\n",
    "            [\n",
    "                metric\n",
    "                for metric in metrics_df.columns\n",
    "                if any(selected_metric in metric for selected_metric in metrics)\n",
    "            ]\n",
    "        )\n",
    "        report_df = report_df.filter(\n",
    "            [\n",
    "                metric\n",
    "                for metric in report_df.columns\n",
    "                if any(selected_metric in metric for selected_metric in metrics)\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    # Display the title with Markdown for emphasis\n",
    "    display(Markdown(f\"## {title}\"))\n",
    "\n",
    "    # Display the metrics DataFrame\n",
    "    display(Markdown(\"### Summary Metrics\"))\n",
    "    display(metrics_df)\n",
    "\n",
    "    # Display the detailed report DataFrame\n",
    "    display(Markdown(f\"### Report Metrics\"))\n",
    "    display(report_df)\n",
    "\n",
    "\n",
    "def display_explanations(df, metrics=None, n=1):\n",
    "    style = \"white-space: pre-wrap; width: 800px; overflow-x: auto;\"\n",
    "    df = df.sample(n=n)\n",
    "    if metrics:\n",
    "        df = df.filter(\n",
    "            [\"instruction\", \"context\", \"reference\", \"completed_prompt\", \"response\"]\n",
    "            + [\n",
    "                metric\n",
    "                for metric in df.columns\n",
    "                if any(selected_metric in metric for selected_metric in metrics)\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        for col in df.columns:\n",
    "            display(HTML(f\"<h2>{col}:</h2> <div style='{style}'>{row[col]}</div>\"))\n",
    "        display(HTML(\"<hr>\"))\n",
    "\n",
    "\n",
    "def plot_radar_plot(eval_results, metrics=None):\n",
    "    fig = go.Figure()\n",
    "\n",
    "    for eval_result in eval_results:\n",
    "        title, summary_metrics, report_df = eval_result\n",
    "\n",
    "        if metrics:\n",
    "            summary_metrics = {\n",
    "                k: summary_metrics[k]\n",
    "                for k, v in summary_metrics.items()\n",
    "                if any(selected_metric in k for selected_metric in metrics)\n",
    "            }\n",
    "\n",
    "        fig.add_trace(\n",
    "            go.Scatterpolar(\n",
    "                r=list(summary_metrics.values()),\n",
    "                theta=list(summary_metrics.keys()),\n",
    "                fill=\"toself\",\n",
    "                name=title,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    fig.update_layout(\n",
    "        polar=dict(radialaxis=dict(visible=True, range=[0, 5])), showlegend=True\n",
    "    )\n",
    "\n",
    "    fig.show()\n",
    "\n",
    "\n",
    "def plot_bar_plot(eval_results, metrics=None):\n",
    "    fig = go.Figure()\n",
    "    data = []\n",
    "\n",
    "    for eval_result in eval_results:\n",
    "        title, summary_metrics, _ = eval_result\n",
    "        if metrics:\n",
    "            summary_metrics = {\n",
    "                k: summary_metrics[k]\n",
    "                for k, v in summary_metrics.items()\n",
    "                if any(selected_metric in k for selected_metric in metrics)\n",
    "            }\n",
    "\n",
    "        data.append(\n",
    "            go.Bar(\n",
    "                x=list(summary_metrics.keys()),\n",
    "                y=list(summary_metrics.values()),\n",
    "                name=title,\n",
    "            )\n",
    "        )\n",
    "\n",
    "    fig = go.Figure(data=data)\n",
    "\n",
    "    # Change the bar mode\n",
    "    fig.update_layout(barmode=\"group\")\n",
    "    fig.show()\n",
    "\n",
    "\n",
    "def print_aggregated_metrics(job):\n",
    "    \"\"\"Print AutoMetrics\"\"\"\n",
    "\n",
    "    rougeLSum = round(job.rougeLSum, 3) * 100\n",
    "    display(\n",
    "        HTML(\n",
    "            f\"<h3>The {rougeLSum}% of the reference summary is represented by LLM when considering the longest common subsequence (LCS) of words.</h3>\"\n",
    "        )\n",
    "    )\n",
    "\n",
    "\n",
    "def print_autosxs_judgments(df, n=3):\n",
    "    \"\"\"Print AutoSxS judgments in the notebook\"\"\"\n",
    "\n",
    "    style = \"white-space: pre-wrap; width: 800px; overflow-x: auto;\"\n",
    "    df = df.sample(n=n)\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        if row[\"confidence\"] >= 0.5:\n",
    "            display(\n",
    "                HTML(\n",
    "                    f\"<h2>Document:</h2> <div style='{style}'>{row['id_columns']['document']}</div>\"\n",
    "                )\n",
    "            )\n",
    "            display(\n",
    "                HTML(\n",
    "                    f\"<h2>Response A:</h2> <div style='{style}'>{row['response_a']}</div>\"\n",
    "                )\n",
    "            )\n",
    "            display(\n",
    "                HTML(\n",
    "                    f\"<h2>Response B:</h2> <div style='{style}'>{row['response_b']}</div>\"\n",
    "                )\n",
    "            )\n",
    "            display(\n",
    "                HTML(\n",
    "                    f\"<h2>Explanation:</h2> <div style='{style}'>{row['explanation']}</div>\"\n",
    "                )\n",
    "            )\n",
    "            display(\n",
    "                HTML(\n",
    "                    f\"<h2>Confidence score:</h2> <div style='{style}'>{row['confidence']}</div>\"\n",
    "                )\n",
    "            )\n",
    "            display(HTML(\"<hr>\"))\n",
    "\n",
    "\n",
    "def print_autosxs_win_metrics(scores):\n",
    "    \"\"\"Print AutoSxS aggregated metrics\"\"\"\n",
    "\n",
    "    score_b = round(scores[\"autosxs_model_b_win_rate\"] * 100)\n",
    "    display(\n",
    "        HTML(\n",
    "            f\"<h3>AutoSxS Autorater prefers {score_b}% of time Model B over Model A </h3>\"\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RPBjzk96w9gV"
   },
   "source": [
    "## Evaluate Tool use and Function Calling quality for Gemini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iY5hG6VMnS_s"
   },
   "source": [
    "#### Use Metric Bundle\n",
    "\n",
    "Metric Bundle `tool_call_quality` contains 4 metrics:\n",
    "* `tool_call_valid`\n",
    "* `tool_name_match`\n",
    "* `tool_parameter_key_match`\n",
    "* `tool_parameter_kv_match`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vcZ2UikInZe0"
   },
   "outputs": [],
   "source": [
    "metrics = [\"tool_call_quality\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dJgpakfIw9gc"
   },
   "source": [
    "### 1. Evaluate a Bring-Your-Own-Prediction dataset\n",
    "\n",
    "Generative model's tool use quality can be evaluated if the eval dataset contains saved model tool call responses, and expected references."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "d6Euv-SQw9gc"
   },
   "outputs": [],
   "source": [
    "response = [\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"7:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"7:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Cinemark\", \"location\": \"Mountain View CA\", \"showtime\": \"5:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "]\n",
    "\n",
    "reference = [\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"7:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Godzilla\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"9:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"7:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "    '{\"content\": \"\", \"tool_calls\": [{\"name\": \"book_tickets\", \"arguments\": {\"movie\": \"Mission Impossible Dead Reckoning Part 1\", \"theater\": \"Regal Edwards 14\", \"location\": \"Mountain View CA\", \"showtime\": \"7:30\", \"date\": \"2024-03-30\", \"num_tix\": \"2\"}}]}',\n",
    "]\n",
    "\n",
    "eval_dataset = pd.DataFrame(\n",
    "    {\n",
    "        \"response\": response,\n",
    "        \"reference\": reference,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PyUj7kOnnNtH"
   },
   "source": [
    "#### Define EvalTask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tYl6A1zInObM"
   },
   "outputs": [],
   "source": [
    "experiment_name = \"eval-saved-llm-tool-use\"  # @param {type:\"string\"}\n",
    "\n",
    "tool_use_eval_task = EvalTask(\n",
    "    dataset=eval_dataset,\n",
    "    metrics=metrics,\n",
    "    experiment=experiment_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 420
    },
    "id": "l8U4KVzAhsP2",
    "outputId": "046f5084-2cd3-43ef-bead-65815068fe3c"
   },
   "outputs": [],
   "source": [
    "run_id = generate_uuid()\n",
    "\n",
    "experiment_run_name = f\"eval-{run_id}\"\n",
    "\n",
    "eval_result = tool_use_eval_task.evaluate(experiment_run_name=experiment_run_name)\n",
    "display_eval_report(\n",
    "    (\n",
    "        f\"Tool Use Quality Evaluation Metrics\",\n",
    "        eval_result.summary_metrics,\n",
    "        eval_result.metrics_table,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 118
    },
    "id": "3WXnsT2qo-OD",
    "outputId": "ccfddcc7-0664-4e3e-8d7a-319804a014dd"
   },
   "outputs": [],
   "source": [
    "tool_use_eval_task.display_runs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oxow7GrIRgxo"
   },
   "source": [
    "## 2. Tool Use and Function Calling with Gemini\n",
    "\n",
    "[Function Calling Documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ew82QX0QWxYa"
   },
   "source": [
    "### Define a function and tool\n",
    "\n",
    "Define an API specification and register the function in a tool with the latest version of [Vertex AI SDK for Python](https://cloud.google.com/vertex-ai/docs/python-sdk/use-vertex-ai-python-sdk).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LiuafKieQgjF"
   },
   "outputs": [],
   "source": [
    "from vertexai.generative_models import (\n",
    "    Content,\n",
    "    FunctionDeclaration,\n",
    "    GenerativeModel,\n",
    "    Part,\n",
    "    Tool,\n",
    ")\n",
    "\n",
    "book_tickets_func = FunctionDeclaration(\n",
    "    name=\"book_tickets\",\n",
    "    description=\"Book movie tickets\",\n",
    "    parameters={\n",
    "        \"type\": \"object\",\n",
    "        \"properties\": {\n",
    "            \"movie\": {\"type\": \"string\", \"description\": \"The title of the movie.\"},\n",
    "            \"theater\": {\n",
    "                \"type\": \"string\",\n",
    "                \"description\": \"The name of the movie theater.\",\n",
    "            },\n",
    "            \"location\": {\n",
    "                \"type\": \"string\",\n",
    "                \"description\": \"The location of the movie theater.\",\n",
    "            },\n",
    "            \"showtime\": {\n",
    "                \"type\": \"string\",\n",
    "                \"description\": \"The showtime of the movie in ISO 8601 format.\",\n",
    "            },\n",
    "            \"date\": {\n",
    "                \"type\": \"string\",\n",
    "                \"description\": \"The date of the movie in ISO 8601 format.\",\n",
    "            },\n",
    "            \"num_tix\": {\n",
    "                \"type\": \"string\",\n",
    "                \"description\": \"The integer number of tickets to book.\",\n",
    "            },\n",
    "        },\n",
    "        \"required\": [\n",
    "            \"movie\",\n",
    "            \"theater\",\n",
    "            \"location\",\n",
    "            \"showtime\",\n",
    "            \"date\",\n",
    "            \"num_tix\",\n",
    "        ],\n",
    "    },\n",
    ")\n",
    "\n",
    "\n",
    "book_tickets_tool = Tool(\n",
    "    function_declarations=[book_tickets_func],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QQgreOFGXJNF"
   },
   "source": [
    "### Generate a function call\n",
    "\n",
    "Prompt the Gemini model and include the tool that you defined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pNevmS2ngB0Z",
    "outputId": "3aba951a-7f7c-4256-84c1-bd1da0b5eb28"
   },
   "outputs": [],
   "source": [
    "prompt = \"\"\"I'd like to book 2 tickets for the movie \"Mission Impossible Dead Reckoning Part 1\"\n",
    "at the Regal Edwards 14 theater in Mountain View, CA. The showtime is 7:30 PM on March 30th, 2024.\n",
    "\"\"\"\n",
    "\n",
    "gemini_model = GenerativeModel(\"gemini-pro\")\n",
    "\n",
    "gemini_response = gemini_model.generate_content(\n",
    "    prompt,\n",
    "    tools=[book_tickets_tool],\n",
    ")\n",
    "\n",
    "gemini_response.candidates[0].content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lOAbbxFFXkdn"
   },
   "source": [
    "###  Unpack the Gemini response into a Python dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "x84XHpuWXmu6",
    "outputId": "d3ce54d1-e0b8-4a4b-b39a-263e5ee163fa"
   },
   "outputs": [],
   "source": [
    "def unpack_response(response):\n",
    "    output = {}\n",
    "    function_call = {}\n",
    "    for key, value in response.candidates[0].content.parts[0].to_dict().items():\n",
    "        function_call[key] = value\n",
    "    output[\"content\"] = \"\"\n",
    "    output[\"tool_calls\"] = [function_call[\"function_call\"]]\n",
    "    output[\"tool_calls\"][0][\"arguments\"] = output[\"tool_calls\"][0].pop(\"args\")\n",
    "    return json.dumps(output)\n",
    "\n",
    "\n",
    "response = unpack_response(gemini_response)\n",
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fsLT0B75aJYz"
   },
   "source": [
    "### Evaluate the Gemini's Function Call Response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kgzK90mybqbK"
   },
   "outputs": [],
   "source": [
    "reference = json.dumps(\n",
    "    {\n",
    "        \"content\": \"\",\n",
    "        \"tool_calls\": [\n",
    "            {\n",
    "                \"name\": \"book_tickets\",\n",
    "                \"arguments\": {\n",
    "                    \"movie\": \"Mission Impossible Dead Reckoning Part 1\",\n",
    "                    \"theater\": \"Regal Edwards 14\",\n",
    "                    \"location\": \"Mountain View CA\",\n",
    "                    \"showtime\": \"7:30\",\n",
    "                    \"date\": \"2024-03-30\",\n",
    "                    \"num_tix\": \"2\",\n",
    "                },\n",
    "            }\n",
    "        ],\n",
    "    }\n",
    ")\n",
    "\n",
    "eval_dataset = pd.DataFrame({\"response\": [response], \"reference\": [reference]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ttRycKaCg6GG",
    "outputId": "f6bda339-834b-468d-9a04-178873e8d67a"
   },
   "outputs": [],
   "source": [
    "# Expected Tool Call Response\n",
    "json.loads(eval_dataset.reference[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "k5SmLtFAdVCh",
    "outputId": "ec321830-88b7-4b1a-9fce-72d7add3ef95"
   },
   "outputs": [],
   "source": [
    "# Actual Gemini Tool Call Response\n",
    "json.loads(eval_dataset.response[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gUqjfQH5aJE2"
   },
   "outputs": [],
   "source": [
    "experiment_name = \"eval-gemini-model-function-call\"  # @param {type:\"string\"}\n",
    "\n",
    "gemini_functiona_call_eval_task = EvalTask(\n",
    "    dataset=eval_dataset,\n",
    "    metrics=[\"tool_call_quality\"],\n",
    "    experiment=experiment_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 344
    },
    "id": "r3MfX9QCZxHW",
    "outputId": "d2270889-6762-43c5-a1bd-60f1391ed2c6"
   },
   "outputs": [],
   "source": [
    "run_id = generate_uuid()\n",
    "\n",
    "eval_result = gemini_functiona_call_eval_task.evaluate(\n",
    "    experiment_run_name=f\"eval-{run_id}\"\n",
    ")\n",
    "\n",
    "display_eval_report(\n",
    "    (\n",
    "        f\"Gemini Tool Use Quality Evaluation Metrics\",\n",
    "        eval_result.summary_metrics,\n",
    "        eval_result.metrics_table,\n",
    "    )\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "NeGjzkqKlTFW",
    "GEv1ZxXPlTFX",
    "o4nrCUUnlTFX"
   ],
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}