Unlocking Multimodal Video Transcription with Gemini

Requirement:

• A Google Cloud undertaking
• The Vertex AI API should be enabled for this undertaking

Gen AI SDK setting variables:

Study extra about setting up a project and a development environment.

Requirement:

Gen AI SDK setting variables:

• GOOGLE_GENAI_USE_VERTEXAI="False"
• GOOGLE_API_KEY=""

Study extra about getting a Gemini API key from Google AI Studio.

import os
import sys
from collections.abc import Callable

from google import genai

# Guide setup (go away unchanged if setup is environment-defined)

# @markdown **Which API: Vertex AI or Google AI Studio?**
GOOGLE_GENAI_USE_VERTEXAI = True  # @param {kind: "boolean"}

# @markdown **Possibility A - Google Cloud undertaking [+location]**
GOOGLE_CLOUD_PROJECT = ""  # @param {kind: "string"}
GOOGLE_CLOUD_LOCATION = "international"  # @param {kind: "string"}

# @markdown **Possibility B - Google AI Studio API key**
GOOGLE_API_KEY = ""  # @param {kind: "string"}


def check_environment() -> bool:
    check_colab_user_authentication()
    return check_manual_setup() or check_vertex_ai() or check_colab() or check_local()


def check_manual_setup() -> bool:
    return check_define_env_vars(
        GOOGLE_GENAI_USE_VERTEXAI,
        GOOGLE_CLOUD_PROJECT.strip(),  # May need been pasted with line return
        GOOGLE_CLOUD_LOCATION,
        GOOGLE_API_KEY,
    )


def check_vertex_ai() -> bool:
    # Workbench and Colab Enterprise
    match os.getenv("VERTEX_PRODUCT", ""):
        case "WORKBENCH_INSTANCE":
            go
        case "COLAB_ENTERPRISE":
            if not running_in_colab_env():
                return False
        case _:
            return False

    return check_define_env_vars(
        True,
        os.getenv("GOOGLE_CLOUD_PROJECT", ""),
        os.getenv("GOOGLE_CLOUD_REGION", ""),
        "",
    )


def check_colab() -> bool:
    if not running_in_colab_env():
        return False

    # Colab Enterprise was checked earlier than, so that is Colab solely
    from google.colab import auth as colab_auth  # kind: ignore

    colab_auth.authenticate_user()

    # Use Colab Secrets and techniques (🗝️ icon in left panel) to retailer the setting variables
    # Secrets and techniques are personal, seen solely to you and the notebooks that you choose
    # - Vertex AI: Retailer your settings as secrets and techniques
    # - Google AI: Immediately import your Gemini API key from the UI
    vertexai, undertaking, location, api_key = get_vars(get_colab_secret)

    return check_define_env_vars(vertexai, undertaking, location, api_key)


def check_local() -> bool:
    vertexai, undertaking, location, api_key = get_vars(os.getenv)

    return check_define_env_vars(vertexai, undertaking, location, api_key)


def running_in_colab_env() -> bool:
    # Colab or Colab Enterprise
    return "google.colab" in sys.modules


def check_colab_user_authentication() -> None:
    if running_in_colab_env():
        from google.colab import auth as colab_auth  # kind: ignore

        colab_auth.authenticate_user()


def get_colab_secret(secret_name: str, default: str) -> str:
    from google.colab import userdata  # kind: ignore

    strive:
        return userdata.get(secret_name)
    besides Exception as e:
        return default


def get_vars(getenv: Callable[[str, str], str]) -> tuple[bool, str, str, str]:
    # Restrict getenv calls to the minimal (could set off UI affirmation for secret entry)
    vertexai_str = getenv("GOOGLE_GENAI_USE_VERTEXAI", "")
    if vertexai_str:
        vertexai = vertexai_str.decrease() in ["true", "1"]
    else:
        vertexai = bool(getenv("GOOGLE_CLOUD_PROJECT", ""))

    undertaking = getenv("GOOGLE_CLOUD_PROJECT", "") if vertexai else ""
    location = getenv("GOOGLE_CLOUD_LOCATION", "") if undertaking else ""
    api_key = getenv("GOOGLE_API_KEY", "") if not undertaking else ""

    return vertexai, undertaking, location, api_key


def check_define_env_vars(
    vertexai: bool,
    undertaking: str,
    location: str,
    api_key: str,
) -> bool:
    match (vertexai, bool(undertaking), bool(location), bool(api_key)):
        case (True, True, _, _):
            # Vertex AI - Google Cloud undertaking [+location]
            location = location or "international"
            define_env_vars(vertexai, undertaking, location, "")
        case (True, False, _, True):
            # Vertex AI - API key
            define_env_vars(vertexai, "", "", api_key)
        case (False, _, _, True):
            # Google AI Studio - API key
            define_env_vars(vertexai, "", "", api_key)
        case _:
            return False

    return True


def define_env_vars(vertexai: bool, undertaking: str, location: str, api_key: str) -> None:
    os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = str(vertexai)
    os.environ["GOOGLE_CLOUD_PROJECT"] = undertaking
    os.environ["GOOGLE_CLOUD_LOCATION"] = location
    os.environ["GOOGLE_API_KEY"] = api_key


def check_configuration(shopper: genai.Consumer) -> None:
    service = "Vertex AI" if shopper.vertexai else "Google AI Studio"
    print(f"Utilizing the {service} API", finish="")

    if shopper._api_client.undertaking:
        print(f' with undertaking "{shopper._api_client.undertaking[:7]}…"', finish="")
        print(f' in location "{shopper._api_client.location}"')
    elif shopper._api_client.api_key:
        api_key = shopper._api_client.api_key
        print(f' with API key "{api_key[:5]}…{api_key[-5:]}"', finish="")
        print(f" (in case of error, make certain it was created for {service})")

import enum
from dataclasses import dataclass
from datetime import timedelta

import IPython.show
import tenacity
from google.genai.errors import ClientError
from google.genai.sorts import (
    FileData,
    FinishReason,
    GenerateContentConfig,
    GenerateContentResponse,
    Half,
    VideoMetadata,
)


class Mannequin(enum.Enum):
    # Typically Out there (GA)
    GEMINI_2_0_FLASH = "gemini-2.0-flash"
    GEMINI_2_5_FLASH = "gemini-2.5-flash"
    GEMINI_2_5_PRO = "gemini-2.5-pro"
    # Default mannequin
    DEFAULT = GEMINI_2_0_FLASH


# Default configuration for extra deterministic outputs
DEFAULT_CONFIG = GenerateContentConfig(
    temperature=0.0,
    top_p=0.0,
    seed=42,  # Arbitrary fastened worth
)

YOUTUBE_URL_PREFIX = "https://www.youtube.com/watch?v="
CLOUD_STORAGE_URI_PREFIX = "gs://"


def url_for_youtube_id(youtube_id: str) -> str:
    return f"{YOUTUBE_URL_PREFIX}{youtube_id}"


class Video(enum.Enum):
    go


class TestVideo(Video):
    # For testing functions, video period is statically specified within the enum title
    # Suffix (ISO 8601 primarily based): _PT[H][M][S]

    # Google DeepMind | The Podcast | Season 3 Trailer | 59s
    GDM_PODCAST_TRAILER_PT59S = url_for_youtube_id("0pJn3g8dfwk")
    # Google Maps | Stroll within the footsteps of Jane Goodall | 2min 42s
    JANE_GOODALL_PT2M42S = "gs://cloud-samples-data/video/JaneGoodall.mp4"
    # Google DeepMind | AlphaFold | The making of a scientific breakthrough | 7min 54s
    GDM_ALPHAFOLD_PT7M54S = url_for_youtube_id("gg7WjuFs8F4")
    # Brut | French reportage | 8min 28s
    BRUT_FR_DOGS_WATER_LEAK_PT8M28S = url_for_youtube_id("U_yYkb-ureI")
    # Google DeepMind | The Podcast | AI for science | 54min 23s
    GDM_AI_FOR_SCIENCE_FRONTIER_PT54M23S = url_for_youtube_id("nQKmVhLIGcs")
    # Google I/O 2025 | Developer Keynote | 1h 10min 03s
    GOOGLE_IO_DEV_KEYNOTE_PT1H10M03S = url_for_youtube_id("GjvgtwSOCao")
    # Google Cloud | Subsequent 2025 | Opening Keynote | 1h 40min 03s
    GOOGLE_CLOUD_NEXT_PT1H40M03S = url_for_youtube_id("Md4Fs-Zc3tg")
    # Google I/O 2025 | Keynote | 1h 56min 35s
    GOOGLE_IO_KEYNOTE_PT1H56M35S = url_for_youtube_id("o8NiE3XMPrM")


class ShowAs(enum.Enum):
    DONT_SHOW = enum.auto()
    TEXT = enum.auto()
    MARKDOWN = enum.auto()


@dataclass
class VideoSegment:
    begin: timedelta
    finish: timedelta


def generate_content(
    immediate: str,
    video: Video | None = None,
    video_segment: VideoSegment | None = None,
    mannequin: Mannequin | None = None,
    config: GenerateContentConfig | None = None,
    show_as: ShowAs = ShowAs.TEXT,
) -> None:
    immediate = immediate.strip()
    mannequin = mannequin or Mannequin.DEFAULT
    config = config or DEFAULT_CONFIG

    model_id = mannequin.worth
    if video:
        if not (video_part := get_video_part(video, video_segment)):
            return
        contents = [video_part, prompt]
        caption = f"{video.title} / {model_id}"
    else:
        contents = immediate
        caption = f"{model_id}"
    print(f" {caption} ".heart(80, "-"))

    for try in get_retrier():
        with try:
            response = shopper.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)
            display_response(response, show_as)


def get_video_part(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> Half | None:
    video_uri: str = video.worth

    if not shopper.vertexai:
        video_uri = convert_to_https_url_if_cloud_storage_uri(video_uri)
        if not video_uri.startswith(YOUTUBE_URL_PREFIX):
            print("Google AI Studio API: Solely YouTube URLs are presently supported")
            return None

    file_data = FileData(file_uri=video_uri, mime_type="video/*")
    video_metadata = get_video_part_metadata(video_segment, fps)

    return Half(file_data=file_data, video_metadata=video_metadata)


def get_video_part_metadata(
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> VideoMetadata:
    def offset_as_str(offset: timedelta) -> str:
        return f"{offset.total_seconds()}s"

    if video_segment:
        start_offset = offset_as_str(video_segment.begin)
        end_offset = offset_as_str(video_segment.finish)
    else:
        start_offset = None
        end_offset = None

    return VideoMetadata(start_offset=start_offset, end_offset=end_offset, fps=fps)


def convert_to_https_url_if_cloud_storage_uri(uri: str) -> str:
    if uri.startswith(CLOUD_STORAGE_URI_PREFIX):
        return f"https://storage.googleapis.com/{uri.removeprefix(CLOUD_STORAGE_URI_PREFIX)}"
    else:
        return uri


def get_retrier() -> tenacity.Retrying:
    return tenacity.Retrying(
        cease=tenacity.stop_after_attempt(7),
        wait=tenacity.wait_incrementing(begin=10, increment=1),
        retry=should_retry_request,
        reraise=True,
    )


def should_retry_request(retry_state: tenacity.RetryCallState) -> bool:
    if not retry_state.consequence:
        return False
    err = retry_state.consequence.exception()
    if not isinstance(err, ClientError):
        return False
    print(f"❌ ClientError {err.code}: {err.message}")

    retry = False
    match err.code:
        case 400 if err.message will not be None and " strive once more " in err.message:
            # Workshop: undertaking accessing Cloud Storage for the primary time (service agent provisioning)
            retry = True
        case 429:
            # Workshop: momentary undertaking with 1 QPM quota
            retry = True
    print(f"🔄 Retry: {retry}")

    return retry


def display_response_info(response: GenerateContentResponse) -> None:
    if usage_metadata := response.usage_metadata:
        if usage_metadata.prompt_token_count:
            print(f"Enter tokens   : {usage_metadata.prompt_token_count:9,d}")
        if usage_metadata.candidates_token_count:
            print(f"Output tokens  : {usage_metadata.candidates_token_count:9,d}")
        if usage_metadata.thoughts_token_count:
            print(f"Ideas tokens: {usage_metadata.thoughts_token_count:9,d}")
    if not response.candidates:
        print("❌ No `response.candidates`")
        return
    if (finish_reason := response.candidates[0].finish_reason) != FinishReason.STOP:
        print(f"❌ {finish_reason = }")
    if not response.textual content:
        print("❌ No `response.textual content`")
        return


def display_response(
    response: GenerateContentResponse,
    show_as: ShowAs,
) -> None:
    if show_as == ShowAs.DONT_SHOW:
        return
    if not (response_text := response.textual content):
        return
    response_text = response.textual content.strip()

    print(" begin of response ".heart(80, "-"))
    match show_as:
        case ShowAs.TEXT:
            print(response_text)
        case ShowAs.MARKDOWN:
            display_markdown(response_text)
    print(" finish of response ".heart(80, "-"))


def display_markdown(markdown: str) -> None:
    IPython.show.show(IPython.show.Markdown(markdown))


def display_video(video: Video) -> None:
    video_url = convert_to_https_url_if_cloud_storage_uri(video.worth)
    assert video_url.startswith("https://")

    video_width = 600
    if video_url.startswith(YOUTUBE_URL_PREFIX):
        youtube_id = video_url.removeprefix(YOUTUBE_URL_PREFIX)
        ipython_video = IPython.show.YouTubeVideo(youtube_id, width=video_width)
    else:
        ipython_video = IPython.show.Video(video_url, width=video_width)

    display_markdown(f"### Video ([source]({video_url}))")
    IPython.show.show(ipython_video)

------------------- GDM_ALPHAFOLD_PT7M54S / gemini-2.0-flash -------------------
Enter tokens   :   134,177
Output tokens  :     2,689
------------------------------ begin of response -------------------------------
[00:02] ? - We have found extra in regards to the world than some other civilization earlier than us.
[00:08] ? - However we have now been caught on this one downside.
[00:11] ? - How do proteins fold up?
[00:13] ? - How do proteins go from a string of amino acids to a compact form that acts as a machine and drives life?
[00:22] ? - Whenever you discover out about proteins, it is extremely thrilling.
[00:25] ? - You possibly can consider them as little organic nano machines.
[00:28] ? - They're primarily the elemental constructing blocks that energy every thing residing on this planet.
[00:34] ? - If we will reliably predict protein buildings utilizing AI, that might change the best way we perceive the pure world.
[00:46] ? - Protein folding is considered one of these holy grail kind issues in biology.
[00:50] Demis Hassabis - We have at all times hypothesized that AI ought to be useful to make these sorts of huge scientific breakthroughs extra rapidly.
[00:58] ? - After which I will most likely be  little tunings that may make a distinction.
[01:02] ? - It ought to be making a histogram on and a background ability.
[01:04] ? - We have been engaged on our system AlphaFold actually exhausting now for over two years.
[01:08] ? - Reasonably than having to do painstaking experiments, sooner or later biologists may be capable to as a substitute depend on AI strategies to straight predict buildings rapidly and effectively.
[01:17] Kathryn Tunyasuvunakool - Typically talking, biologists are typically fairly skeptical of computational work, and I believe that skepticism is wholesome and I respect it, however I really feel very enthusiastic about what AlphaFold can obtain.
[01:28] Andrew Senior - CASP is after we, we are saying, look, DeepMind is doing protein folding.
[01:31] Andrew Senior - That is how good we're, and possibly it is higher than all people else, possibly it is not.
[01:37] ? - We determined to enter CASP competitors as a result of it represented the Olympics of protein folding.
[01:44] John Moult - CASP, we began to try to pace up the answer to the protein folding downside.
[01:50] John Moult - After we began CASP in 1994, I definitely was naive about how exhausting this was going to be.
[01:58] ? - It was very cumbersome to try this as a result of it took a very long time.
[02:01] ? - Let's examine what, what, what are we doing nonetheless to enhance?
[02:03] ? - Sometimes 100 totally different teams from all over the world take part in CASP, and we take a set of 100 proteins and we ask the teams to ship us what they assume the buildings seem like.
[02:15] ? - We are able to attain 57.9 GDT on CASP 12 floor reality.
[02:19] John Jumper - CASP has a metric on which you can be scored, which is that this GDT metric.
[02:25] John Jumper - On a scale of zero to 100, you'll count on a GDT over 90 to be an answer to the issue.
[02:33] ? - If we do obtain this, this has unimaginable medical relevance.
[02:37] Pushmeet Kohli - The implications are immense, from how ailments progress, how one can uncover new medicine.
[02:45] Pushmeet Kohli - It is limitless.
[02:46] ? - I wished to make a, a very easy system and the outcomes have been surprisingly good.
[02:50] ? - The crew received some outcomes with a brand new method, not solely is it extra correct, nevertheless it's a lot sooner than the outdated system.
[02:56] ? - I believe we'll considerably exceed what we're doing proper now.
[02:59] ? - This can be a recreation, recreation changer, I believe.
[03:01] John Moult - In CASP 13, one thing very vital had occurred.
[03:06] John Moult - For the primary time, we noticed the efficient software of synthetic intelligence.
[03:11] ? - We have superior the state-of-the-art within the subject, in order that's unbelievable, however we nonetheless received a protracted method to go earlier than we have solved it.
[03:18] ? - The shapes have been now roughly appropriate for lots of the proteins, however the particulars, precisely the place every atom sits, which is de facto what we might name an answer, we're not but there.
[03:29] ? - It does not assist if in case you have the tallest ladder when you are going to the moon.
[03:33] ? - We hit somewhat little bit of a brick wall, um, since we received CASP, then it was again to the drafting board and like what are our new concepts?
[03:41] ? - Um, after which it is taken a short while, I'd say, for them to get again to the place they have been, however with the brand new concepts.
[03:51] ? - They'll go additional, proper?
[03:52] ? - So, um, in order that's a very necessary second.
[03:55] ? - I've seen that second so many instances now, however I do know what meaning now, and I do know that is the time now to press.
[04:02] ? - We have to double down and go as quick as attainable from right here.
[04:05] ? - I believe we have got no time to lose.
[04:07] ? - So the intention is to enter CASP once more.
[04:09] ? - CASP is deeply worrying.
[04:12] ? - There's one thing bizarre happening with, um, the educational as a result of it's studying one thing that is correlated with GDT, nevertheless it's not calibrated.
[04:18] ? - I really feel barely uncomfortable.
[04:20] ? - We ought to be studying this, you already know, within the blink of an eye fixed.
[04:23] ? - The know-how advancing exterior DeepMind can also be doing unimaginable work.
[04:27] Richard Evans - And there is at all times the likelihood one other crew has come someplace on the market subject that we do not even find out about.
[04:32] ? - Somebody requested me, nicely, ought to we panic now?
[04:33] ? - In fact, we must always have been panicking earlier than.
[04:35] ? - It does appear to do higher, however nonetheless does not do fairly in addition to the perfect mannequin.
[04:39] ? - Um, so it appears to be like like there's room for enchancment.
[04:42] ? - There's at all times a threat that you've got missed one thing, and that is why blind assessments like CASP are so necessary to validate whether or not our outcomes are actual.
[04:49] ? - Clearly, I am excited to see how CASP 14 goes.
[04:51] ? - My expectation is we get our heads down, we give attention to the complete objective, which is to resolve the entire downside.
[05:14] ? - We have been ready for CASP to begin on April fifteenth as a result of that is when it was initially scheduled to begin, and it has been delayed by a month as a result of coronavirus.
[05:24] ? - I actually miss everybody.
[05:25] ? - No, I struggled somewhat bit simply form of getting right into a routine, particularly, uh, my spouse, she got here down with the, the virus.
[05:32] ? - I imply, fortunately it did not end up too critical.
[05:34] ? - CASP began on Monday.
[05:37] Demis Hassabis - Can I simply verify this diagram you have received right here, John, this one the place we ask floor reality.
[05:40] Demis Hassabis - Is that this one we have finished badly on?
[05:42] ? - We're really fairly good on this area.
[05:43] ? - Should you think about that we hadn't have mentioned it got here round this manner, however had put it in.
[05:47] ? - Yeah, and that as a substitute.
[05:48] ? - Yeah.
[05:49] ? - One of many hardest proteins we have gotten in CASP to this point is a SARS-CoV-2 protein, uh, referred to as Orf8.
[05:55] ? - Orf8 is a coronavirus protein.
[05:57] ? - We tried actually exhausting to enhance our prediction, like actually, actually exhausting, most likely essentially the most time that we have now ever spent on a single goal.
[06:05] ? - So we're about two-thirds of the best way by CASP, and we have gotten three solutions again.
[06:11] ? - We now have a floor reality for Orf8, which is among the coronavirus proteins.
[06:17] ? - And it seems we did rather well in predicting that.
[06:20] Demis Hassabis - Superb job, everybody, the entire crew.
[06:23] Demis Hassabis - It has been an unimaginable effort.
[06:24] John Moult - Right here what we noticed in CASP 14 was a gaggle delivering atomic accuracy off the bat, primarily fixing what in our world is 2 issues.
[06:34] John Moult - How do you look to search out the suitable resolution, after which how do you acknowledge you have received the suitable resolution whenever you're there?
[06:41] ? - All proper, are we, are we principally right here?
[06:46] ? - I will learn an e mail.
[06:48] ? - Uh, I received this from John Moult.
[06:50] ? - Now I will simply learn it.
[06:51] ? - It says, John, as I count on you already know, your group has carried out amazingly nicely in CASP 14, each relative to different teams and in absolute mannequin accuracy.
[07:02] ? - Congratulations on this work.
[07:05] ? - It's actually excellent.
[07:07] Demis Hassabis - AlphaFold represents an enormous leap ahead that I hope will actually speed up drug discovery and assist us to higher perceive illness.
[07:13] John Moult - It is fairly mind-blowing.
[07:16] John Moult - , these outcomes have been, for me, having labored on this downside so lengthy, after many, many stops and begins and can this ever get there, instantly this can be a resolution.
[07:28] John Moult - We have solved the issue.
[07:29] John Moult - This provides you such pleasure about the best way science works, about how one can by no means see precisely and even roughly what is going on to occur subsequent.
[07:37] John Moult - There are at all times these surprises, and that basically, as a scientist, is what retains you going.
[07:41] John Moult - What is going on to be the subsequent shock?
------------------------------- finish of response --------------------------------

------------------------------- gemini-2.0-flash -------------------------------
Enter tokens   :        18
Output tokens  :     1,814
------------------------------ begin of response -------------------------------

## Terminology Used for Video Transcriptions

This is a breakdown of widespread terminology used when discussing video transcriptions:

*   **Transcription:** The final time period for changing audio or video content material into textual content.
*   **Video Transcription:** Particularly refers to transcribing the audio portion of a video.
*   **Verbatim Transcription:** A transcription that features each spoken phrase, together with filler phrases ("um," "ah"), false begins, stutters, and background noises (if related).  It goals for a extremely correct illustration of the spoken phrase.
*   **Clear Verbatim Transcription (or Edited Transcription):** A transcription that removes filler phrases, false begins, and stutters to create a extra readable and polished textual content. It focuses on conveying the which means clearly.
*   **Clever Verbatim Transcription:** A center floor between verbatim and clear verbatim. It removes some filler phrases and stutters however retains parts that contribute to the speaker's type or the context of the dialog.
*   **Abstract Transcription:** A quick overview of the video's content material, highlighting the details.
*   **Timecoding (or Timestamping):** Including timestamps to the transcription to point when every part of textual content was spoken within the video.  That is essential for accessibility and navigation.
*   **Speaker Identification:** Figuring out and labeling totally different audio system within the video.
*   **Captioning (or Subtitling):** Creating textual content variations of the video's audio which can be displayed on the display screen, normally synchronized with the video.  Captions are sometimes used for accessibility (for deaf or hard-of-hearing viewers) and for viewers preferring to learn alongside.
*   **Closed Captions:** Captions that may be turned on or off by the viewer.
*   **Open Captions:** Captions which can be completely embedded within the video and can't be turned off.
*   **SRT (SubRip Subtitle):** A standard file format for storing captions and subtitles.
*   **VTT (Video Textual content Tracks):** A extra trendy file format for captions and subtitles, typically used with HTML5 video.
*   **Burned-in Captions:** Captions which can be completely embedded into the video file.
*   **Transcriber:** The particular person or software program that performs the transcription.
*   **Accuracy Fee:** The share of phrases transcribed accurately.
*   **Turnaround Time:** The time it takes to finish the transcription.

## Typical Output Instance

For instance we have now a brief video clip of an interview. This is how various kinds of transcriptions may look:

**State of affairs:** A brief clip of an interview with a chef discussing their new restaurant.

**1. Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is, uh, nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us somewhat bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, um, it is, it has been a dream of mine for, like, years, you already know?  To, like, open a spot that basically focuses on, um, seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can folks count on?
00:00:25 --> 00:00:32 Chef: We're doing, uh, trendy American, however with, like, an actual emphasis on, uh, native farms and, you already know, sustainable practices.
```

**2. Clear Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us somewhat bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, it has been a dream of mine for years to open a spot that basically focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can folks count on?
00:00:25 --> 00:00:32 Chef: We're doing trendy American, however with an actual emphasis on native farms and sustainable practices.
```

**3. Clever Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us somewhat bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, it has been a dream of mine for, like, years, you already know? To open a spot that basically focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can folks count on?
00:00:25 --> 00:00:32 Chef: We're doing trendy American, however with, like, an actual emphasis on native farms and, you already know, sustainable practices.
```

**4. Abstract Transcription:**

```
Chef discusses their new restaurant, "The Golden Spoon," which focuses on seasonal elements and trendy American delicacies with an emphasis on native farms and sustainable practices.
```

**5. SRT File (for Captions):**

```
1
00:00:00,000 --> 00:00:03,000
Interviewer: So, Chef, welcome to the present!

2
00:00:03,000 --> 00:00:06,000
Chef: Thanks for having me. It is nice to be right here.

3
00:00:06,000 --> 00:00:12,000
Interviewer: So, inform us somewhat bit about your new restaurant, "The Golden Spoon."

4
00:00:12,000 --> 00:00:20,000
Chef: Properly, it has been a dream of mine for years to open a spot that basically focuses on seasonal elements.

5
00:00:20,000 --> 00:00:25,000
Interviewer: And what sort of delicacies can folks count on?

6
00:00:25,000 --> 00:00:32,000
Chef: We're doing trendy American, however with an actual emphasis on native farms and sustainable practices.
```

**Key Variations:**

*   **Verbatim:** Captures each element, together with imperfections in speech.
*   **Clear Verbatim:**  Focuses on readability and readability by eradicating pointless parts.
*   **Clever Verbatim:** A steadiness between accuracy and readability.
*   **Abstract:** Gives a concise overview.
*   **SRT:**  A structured file format for displaying captions synchronized with the video.

The very best kind of transcription is determined by the particular wants of the undertaking.  For authorized or analysis functions, a verbatim transcription could be obligatory. For basic accessibility or advertising and marketing functions, a clear verbatim or clever verbatim transcription is usually most well-liked.  Abstract transcriptions are helpful for rapidly understanding the content material of a video.  SRT information are important for including captions to movies.

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,800
Output tokens  :       635
------------------------------ begin of response -------------------------------

[
  {
    "start": "00:00",
    "text": "Do I have to call you Sir Demis now?",
    "voice": 1
  },
  {
    "start": "00:01",
    "text": "Oh, you don't. Absolutely not.",
    "voice": 2
  },
  {
    "start": "00:03",
    "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
    "voice": 1
  },
  {
    "start": "00:06",
    "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
    "voice": 1
  },
  {
    "start": "00:19",
    "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
    "voice": 3
  },
  {
    "start": "00:26",
    "text": "I want a bridge that will not collapse.",
    "voice": 3
  },
  {
    "start": "00:30",
    "text": "just give these scientists a superpower that they had not imagined earlier.",
    "voice": 4
  },
  {
    "start": "00:34",
    "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
    "voice": 5
  },
  {
    "start": "00:38",
    "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
    "voice": 6
  },
  {
    "start": "00:48",
    "text": "You really want AGI to be able to peer into the mysteries of the universe.",
    "voice": 1
  },
  {
    "start": "00:51",
    "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
    "voice": 2
  },
  {
    "start": "00:55",
    "text": "Ow.",
    "voice": 1
  },
  {
    "start": "00:56",
    "text": "the magic of AI.",
    "voice": 6
  }
]

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,920
Output tokens  :       806
------------------------------ begin of response -------------------------------

{
  "task1_transcripts": [
    {
      "start": "00:00",
      "text": "Do I have to call you Sir Demis now?",
      "voice": 1
    },
    {
      "start": "00:01",
      "text": "Oh, you don't. Absolutely not.",
      "voice": 2
    },
    {
      "start": "00:04",
      "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
      "voice": 1
    },
    {
      "start": "00:06",
      "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
      "voice": 1
    },
    {
      "start": "00:19",
      "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
      "voice": 3
    },
    {
      "start": "00:26",
      "text": "I want a bridge that will not collapse.",
      "voice": 3
    },
    {
      "start": "00:30",
      "text": "That just give these scientists a superpower that they had not imagined earlier.",
      "voice": 4
    },
    {
      "start": "00:34",
      "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
      "voice": 5
    },
    {
      "start": "00:38",
      "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
      "voice": 6
    },
    {
      "start": "00:48",
      "text": "You really want AGI to be able to peer into the mysteries of the universe.",
      "voice": 1
    },
    {
      "start": "00:51",
      "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
      "voice": 2
    },
    {
      "start": "00:55",
      "text": "Ow.",
      "voice": 1
    },
    {
      "start": "00:56",
      "text": "the magic of AI.",
      "voice": 6
    }
  ],
  "task2_speakers": [
    {
      "voice": 1,
      "name": "Professor Hannah Fry"
    },
    {
      "voice": 2,
      "name": "Demis Hassabis"
    },
    {
      "voice": 3,
      "name": "Anca Dragan"
    },
    {
      "voice": 4,
      "name": "Pushmeet Kohli"
    },
    {
      "voice": 5,
      "name": "Jeff Dean"
    },
    {
      "voice": 6,
      "name": "Douglas Eck"
    }
  ]
}

------------------------------- finish of response --------------------------------

import re

import pydantic
from google.genai.sorts import MediaResolution, ThinkingConfig

SamplingFrameRate = float

VIDEO_TRANSCRIPTION_PROMPT = """
**Activity 1 - Transcripts**

- Watch the video and pay attention rigorously to the audio.
- Establish every distinctive voice utilizing a `voice` ID (1, 2, 3, and so on.).
- Transcribe the video's audio verbatim with voice diarization.
- Embrace the `begin` timecode ({timecode_spec}) for every speech section.

**Activity 2 - Audio system**

- For every `voice` ID from Activity 1, extract details about the corresponding speaker.
- Use visible and audio cues.
- If a chunk of data can't be discovered, use a query mark (`?`) as the worth.
"""
NOT_FOUND = "?"


class Transcript(pydantic.BaseModel):
    begin: str
    textual content: str
    voice: int


class Speaker(pydantic.BaseModel):
    voice: int
    title: str
    firm: str
    place: str
    role_in_video: str


class VideoTranscription(pydantic.BaseModel):
    task1_transcripts: listing[Transcript] = pydantic.Subject(default_factory=listing)
    task2_speakers: listing[Speaker] = pydantic.Subject(default_factory=listing)


def get_generate_content_config(mannequin: Mannequin, video: Video) -> GenerateContentConfig:
    media_resolution = get_media_resolution_for_video(video)
    thinking_config = get_thinking_config(mannequin)

    return GenerateContentConfig(
        temperature=DEFAULT_CONFIG.temperature,
        top_p=DEFAULT_CONFIG.top_p,
        seed=DEFAULT_CONFIG.seed,
        response_mime_type="software/json",
        response_schema=VideoTranscription,
        media_resolution=media_resolution,
        thinking_config=thinking_config,
    )


def get_video_duration(video: Video) -> timedelta | None:
    # For testing functions, video period is statically specified within the enum title
    # Suffix (ISO 8601 primarily based): _PT[H][M][S]
    # For manufacturing,
    # - fetch durations dynamically or retailer them individually
    # - have in mind video VideoMetadata.start_offset & VideoMetadata.end_offset
    regex = r"_PT(?:(d+)H)?(?:(d+)M)?(?:(d+)S)?$"
    if not (match := re.search(regex, video.title)):
        print(f"⚠️ No period information in {video.title}. Will use defaults.")
        return None

    h_str, m_str, s_str = match.teams()
    return timedelta(
        hours=int(h_str) if h_str will not be None else 0,
        minutes=int(m_str) if m_str will not be None else 0,
        seconds=int(s_str) if s_str will not be None else 0,
    )


def get_media_resolution_for_video(video: Video) -> MediaResolution | None:
    if not (video_duration := get_video_duration(video)):
        return None  # Default

    # For testing functions, that is primarily based on video period, as our brief movies are typically extra detailed
    less_than_five_minutes = video_duration < timedelta(minutes=5)
    if less_than_five_minutes:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_MEDIUM
    else:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_LOW

    return media_resolution


def get_sampling_frame_rate_for_video(video: Video) -> SamplingFrameRate | None:
    sampling_frame_rate = None  # Default (1 FPS for present fashions)

    # [Optional] Outline a customized FPS: 0.0 < sampling_frame_rate <= 24.0

    return sampling_frame_rate


def get_timecode_spec_for_model_and_video(mannequin: Mannequin, video: Video) -> str:
    timecode_spec = "MM:SS"  # Default

    match mannequin:
        case Mannequin.GEMINI_2_0_FLASH:  # Helps MM:SS
            go
        case Mannequin.GEMINI_2_5_FLASH | Mannequin.GEMINI_2_5_PRO:  # Help MM:SS and H:MM:SS
            period = get_video_duration(video)
            one_hour_or_more = period will not be None and timedelta(hours=1) <= period
            if one_hour_or_more:
                timecode_spec = "MM:SS or H:MM:SS"
        case _:
            assert False, "Add timecode format for brand new mannequin"

    return timecode_spec


def get_thinking_config(mannequin: Mannequin) -> ThinkingConfig | None:
    # Examples of pondering configurations (Gemini 2.5 fashions)
    match mannequin:
        case Mannequin.GEMINI_2_5_FLASH:  # Considering disabled
            return ThinkingConfig(thinking_budget=0, include_thoughts=False)
        case Mannequin.GEMINI_2_5_PRO:  # Minimal pondering funds and no summarized ideas
            return ThinkingConfig(thinking_budget=128, include_thoughts=False)
        case _:
            return None  # Default


def get_video_transcription_from_response(
    response: GenerateContentResponse,
) -> VideoTranscription:
    if not isinstance(response.parsed, VideoTranscription):
        print("❌ Couldn't parse the JSON response")
        return VideoTranscription()  # Empty transcription

    return response.parsed


def get_video_transcription(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> VideoTranscription:
    mannequin = mannequin or Mannequin.DEFAULT
    model_id = mannequin.worth

    fps = fps or get_sampling_frame_rate_for_video(video)
    video_part = get_video_part(video, video_segment, fps)
    if not video_part:  # Unsupported supply, return an empty transcription
        return VideoTranscription()
    if immediate is None:
        timecode_spec = get_timecode_spec_for_model_and_video(mannequin, video)
        immediate = VIDEO_TRANSCRIPTION_PROMPT.format(timecode_spec=timecode_spec)
    contents = [video_part, prompt.strip()]

    config = get_generate_content_config(mannequin, video)

    print(f" {video.title} / {model_id} ".heart(80, "-"))
    response = None
    for try in get_retrier():
        with try:
            response = shopper.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)

    assert isinstance(response, GenerateContentResponse)
    return get_video_transcription_from_response(response)

import itertools
from collections.abc import Callable, Iterator

from pandas import DataFrame, Collection
from pandas.io.codecs.type import Styler
from pandas.io.codecs.style_render import CSSDict

BGCOLOR_COLUMN = "bg_color"  # Hidden column to retailer row background colours


def yield_known_speaker_color() -> Iterator[str]:
    PAL_40 = ("#669DF6", "#EE675C", "#FCC934", "#5BB974")
    PAL_30 = ("#8AB4F8", "#F28B82", "#FDD663", "#81C995")
    PAL_20 = ("#AECBFA", "#F6AEA9", "#FDE293", "#A8DAB5")
    PAL_10 = ("#D2E3FC", "#FAD2CF", "#FEEFC3", "#CEEAD6")
    PAL_05 = ("#E8F0FE", "#FCE8E6", "#FEF7E0", "#E6F4EA")
    return itertools.cycle([*PAL_40, *PAL_30, *PAL_20, *PAL_10, *PAL_05])


def yield_unknown_speaker_color() -> Iterator[str]:
    GRAYS = ["#80868B", "#9AA0A6", "#BDC1C6", "#DADCE0", "#E8EAED", "#F1F3F4"]
    return itertools.cycle(GRAYS)


def get_color_for_voice_mapping(audio system: listing[Speaker]) -> dict[int, str]:
    known_speaker_color = yield_known_speaker_color()
    unknown_speaker_color = yield_unknown_speaker_color()

    mapping: dict[int, str] = {}
    for speaker in audio system:
        if speaker.title != NOT_FOUND:
            coloration = subsequent(known_speaker_color)
        else:
            coloration = subsequent(unknown_speaker_color)
        mapping[speaker.voice] = coloration

    return mapping


def get_table_styler(df: DataFrame) -> Styler:
    def join_styles(kinds: listing[str]) -> str:
        return ";".be part of(kinds)

    table_css = [
        "color: #202124",
        "background-color: #BDC1C6",
        "border: 0",
        "border-radius: 0.5rem",
        "border-spacing: 0px",
        "outline: 0.5rem solid #BDC1C6",
        "margin: 1rem 0.5rem",
    ]
    th_css = ["background-color: #E8EAED"]
    th_td_css = ["text-align:left", "padding: 0.25rem 1rem"]
    table_styles = [
        CSSDict(selector="", props=join_styles(table_css)),
        CSSDict(selector="th", props=join_styles(th_css)),
        CSSDict(selector="th,td", props=join_styles(th_td_css)),
    ]

    return df.type.set_table_styles(table_styles).disguise()


def change_row_bgcolor(row: Collection) -> listing[str]:
    type = f"background-color:{row[BGCOLOR_COLUMN]}"
    return [style] * len(row)


def display_table(yield_rows: Callable[[], Iterator[list[str]]]) -> None:
    information = yield_rows()
    df = DataFrame(columns=subsequent(information), information=information)
    styler = get_table_styler(df)
    styler.apply(change_row_bgcolor, axis=1)
    styler.disguise([BGCOLOR_COLUMN], axis="columns")

    html = styler.to_html()
    IPython.show.show(IPython.show.HTML(html))


def display_speakers(transcription: VideoTranscription) -> None:
    def sanitize_field(s: str, symbol_if_unknown: str) -> str:
        return symbol_if_unknown if s == NOT_FOUND else s

    def yield_rows() -> Iterator[list[str]]:
        yield ["voice", "name", "company", "position", "role_in_video", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        for speaker in transcription.task2_speakers:
            yield [
                str(speaker.voice),
                sanitize_field(speaker.name, NOT_FOUND),
                sanitize_field(speaker.company, NOT_FOUND),
                sanitize_field(speaker.position, NOT_FOUND),
                sanitize_field(speaker.role_in_video, NOT_FOUND),
                color_for_voice.get(speaker.voice, "red"),
            ]

    display_markdown(f"### Audio system ({len(transcription.task2_speakers)})")
    display_table(yield_rows)


def display_transcripts(transcription: VideoTranscription) -> None:
    def yield_rows() -> Iterator[list[str]]:
        yield ["start", "speaker", "transcript", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        speaker_for_voice = {
            speaker.voice: speaker for speaker in transcription.task2_speakers
        }
        previous_voice = None
        for transcript in transcription.task1_transcripts:
            current_voice = transcript.voice
            speaker_label = ""
            if speaker := speaker_for_voice.get(current_voice, None):
                if speaker.title != NOT_FOUND:
                    speaker_label = speaker.title
                elif speaker.place != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.position}]"
                elif speaker.role_in_video != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.role_in_video}]"
            if not speaker_label:
                speaker_label = f"[voice {current_voice}]"
            yield [
                transcript.start,
                speaker_label if current_voice != previous_voice else '"',
                transcript.text,
                color_for_voice.get(current_voice, "red"),
            ]
            previous_voice = current_voice

    display_markdown(f"### Transcripts ({len(transcription.task1_transcripts)})")
    display_table(yield_rows)


def transcribe_video(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> None:
    display_video(video)
    transcription = get_video_transcription(video, video_segment, fps, immediate, mannequin)
    display_speakers(transcription)
    display_transcripts(transcription)