diff --git a/CHANGELOG.md b/CHANGELOG.md index c9a622b8..5d598b2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### New features + +* Added video input for `ChatGoogle()`/`ChatVertex()` (Gemini is the only provider that accepts video): `content_video_file()` for small inline clips (mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp), and `content_video_youtube()` to reference a public YouTube URL directly, with no upload and no MIME type. Passing either to a provider other than Gemini raises a clear `NotImplementedError`. For larger local video files, use `chat.files.upload()` instead. + ### Improvements * Reasoning is now visible when echoing. Previously, thinking content was wrapped in literal `` tags that a markdown renderer treated as an HTML block and dropped, so reasoning never appeared at all — even with `echo="all"`. It now renders in a "Thinking" panel in the console, and in a `
` block in notebooks that stays expanded while reasoning streams in and collapses once it's done. `echo="text"` continues to show only the assistant's answer. (#361) diff --git a/chatlas/__init__.py b/chatlas/__init__.py index 7b3c0c50..197b3a10 100644 --- a/chatlas/__init__.py +++ b/chatlas/__init__.py @@ -14,6 +14,7 @@ from ._content_document import content_document_file, content_document_url from ._content_image import content_image_file, content_image_plot, content_image_url from ._content_pdf import content_pdf_file, content_pdf_url +from ._content_video import content_video_file, content_video_youtube from ._files import FileManager from ._interpolate import interpolate, interpolate_file from ._parallel import parallel_chat, parallel_chat_structured, parallel_chat_text @@ -87,6 +88,8 @@ "content_image_url", "content_pdf_file", "content_pdf_url", + "content_video_file", + "content_video_youtube", "ContentToolRequest", "ContentToolResult", "FileManager", diff --git a/chatlas/_content.py b/chatlas/_content.py index 9de75566..bf5871ef 100644 --- a/chatlas/_content.py +++ b/chatlas/_content.py @@ -159,6 +159,22 @@ def is_image_content_type(content_type: str) -> TypeIs[ImageContentTypes]: return content_type in IMAGE_CONTENT_TYPES +VideoContentTypes = Literal[ + "video/mp4", + "video/mpeg", + "video/mov", + "video/avi", + "video/x-flv", + "video/mpg", + "video/webm", + "video/wmv", + "video/3gpp", +] +""" +Allowable content types for inline video. Only Gemini accepts video input. +""" + + class ToolInfo(BaseModel): """ Serializable tool information @@ -207,6 +223,8 @@ def from_tool(cls, tool: "Tool | ToolBuiltIn") -> "ToolInfo": "text", "image_remote", "image_inline", + "video_inline", + "video_url", "tool_request", "tool_result", "tool_result_image", @@ -358,6 +376,77 @@ def __str__(self): return f"![](data:{self.image_content_type};base64,{self.data})" +class ContentVideo(Content): + """ + Base class for video content. + + This class is not meant to be used directly. Instead, use + [](`~chatlas.content_video_file`) or [](`~chatlas.content_video_youtube`). + """ + + pass + + +class ContentVideoInline(ContentVideo): + """ + Inline video content, for small clips. + + This is the return type for [](`~chatlas.content_video_file`). + It's not meant to be used directly. + + Only Gemini accepts video input, and only for requests that stay under + roughly 100 MB once base64-encoded; use `chat.files.upload()` for larger + files instead. + + Parameters + ---------- + video_content_type + The content type of the video. + data + The base64-encoded video data. + filename + The name of the video file, if known. + """ + + video_content_type: VideoContentTypes + data: str + filename: Optional[str] = None + + content_type: ContentTypeEnum = "video_inline" + + def __str__(self): + name = f" file={self.filename}" if self.filename else "" + return f"" + + +class ContentVideoUrl(ContentVideo): + """ + A video referenced by URL, with no upload involved. + + This is the return type for [](`~chatlas.content_video_youtube`). + It's not meant to be used directly. + + Unlike [](`~chatlas.types.ContentUploaded`), this isn't a file a provider + is hosting on your behalf -- there's nothing to list, download, or + delete, and it doesn't expire. It's sent with no MIME type, since Gemini + determines the video format itself. As of this writing, Gemini only + accepts public YouTube URLs this way (up to 10 per request on Gemini + 2.5+), and only Gemini accepts video URLs at all. + + Parameters + ---------- + url + The URL of the video. + """ + + url: str + + content_type: ContentTypeEnum = "video_url" + + def __str__(self): + return f"