diff --git a/python/README.md b/python/README.md index 51f4291c..f3c62b91 100644 --- a/python/README.md +++ b/python/README.md @@ -68,6 +68,77 @@ Existing API-key and environment-based calls are unchanged. See the and [configured-client example](../site/src/docs/python/basic-usage.md#reusing-an-already-configured-client) for ownership, retry, and error behavior. +All four model factories accept a keyword-only `temperature` option. It defaults +to `0.0`, preserving existing behavior. Pass `temperature=None` to omit the +parameter entirely and use the configured client's or service's default, for +example with reasoning models that reject explicit sampling parameters. This +does not disable reasoning, change authentication, or select a different API. + +## Text and image prompts + +Both built-in Chat Completions models and the configured-client adapter accept +typed text/image parts in `PromptSection.content`. Existing whole-string prompts +remain supported without migration. + +The following example demonstrates both `model.complete` and translator context; +applications can use either call independently. Supply your own hosted image URL +and already-base64-encoded image data: + +```python +from typing_extensions import TypedDict + +from typechat import ( + PromptSection, + Result, + TypeChatJsonTranslator, + TypeChatLanguageModel, + TypeChatValidator, +) + + +class ImageDescription(TypedDict): + description: str + + +async def inspect_images( + model: TypeChatLanguageModel, image_base64: str +) -> tuple[Result[str], Result[ImageDescription]]: + context: list[PromptSection] = [ + {"role": "user", "content": [ + {"type": "text", "text": "Use these images as context."}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.png"}}, + {"type": "image_url", "image_url": { + "url": f"data:image/png;base64,{image_base64}", + "detail": "low", + }}, + ]}, + ] + text = await model.complete(context) + translator = TypeChatJsonTranslator(model, TypeChatValidator(ImageDescription), ImageDescription) + data = await translator.translate("Describe the images.", prompt_preamble=context) + return text, data +``` + +The public dictionary types are `TextPromptContent` (`{"type": "text", "text": str}`), +`ImagePromptContent` (`{"type": "image_url", "image_url": ImageUrl}`), and `ImageUrl` +with a required nonempty `url: str` and optional `detail: "auto" | "low" | "high"`. +`PromptContent` is a string or an ordered list of these tagged parts, not a list of +bare strings. Text works in system, user, and assistant messages; images are +supported only in user messages. Malformed input produces a content-free `Failure` +locating the section or part before authentication or dispatch. + +`translate` still takes a string request. Its preamble stays before the generated +schema/request message, and image context is preserved during validation and repair. +Successful model completions remain text-only. Authentication, HTTP controls, and +configured-client ownership remain unchanged. + +TypeChat only forwards image references: it does not fetch, decode, resize, or +validate image contents. Supported formats (such as PNG, JPEG, or WebP), URL +accessibility, image counts, detail support, and size/resolution limits depend on +your provider and model. No image-processing or provider SDK dependency is added. +See [text and image usage](../site/src/docs/python/basic-usage.md#text-and-image-prompts) +for the complete prompt type definitions and limitations. + ## Documenting schema fields Trailing Python comments on directly declared `TypedDict` and dataclass fields become diff --git a/python/src/typechat/__init__.py b/python/src/typechat/__init__.py index 241f1cfc..13ace520 100644 --- a/python/src/typechat/__init__.py +++ b/python/src/typechat/__init__.py @@ -4,7 +4,11 @@ from typechat._internal.model import ( ChatCompletionClient, + ImagePromptContent, + ImageUrl, + PromptContent, PromptSection, + TextPromptContent, TokenProvider, TypeChatLanguageModel, create_language_model, @@ -26,7 +30,11 @@ "Failure", "Result", "python_type_to_typescript_schema", + "PromptContent", "PromptSection", + "TextPromptContent", + "ImagePromptContent", + "ImageUrl", "TokenProvider", "ChatCompletionClient", "create_language_model", diff --git a/python/src/typechat/_internal/model.py b/python/src/typechat/_internal/model.py index 7b92ed1c..44f87d15 100644 --- a/python/src/typechat/_internal/model.py +++ b/python/src/typechat/_internal/model.py @@ -2,14 +2,32 @@ import json import re from collections.abc import Awaitable, Callable, Sequence +from copy import deepcopy from inspect import isawaitable from types import TracebackType -from typing_extensions import AsyncContextManager, Literal, Protocol, Self, TypedDict, cast, override +from typing_extensions import ( + AsyncContextManager, Literal, NotRequired, Protocol, Self, TypeAlias, TypedDict, cast, override, +) from typechat._internal.result import Failure, Result, Success import httpx +class TextPromptContent(TypedDict): + type: Literal["text"] + text: str + +class ImageUrl(TypedDict): + """A hosted image or data URL; image contents and provider limits are caller-owned.""" + url: str + detail: NotRequired[Literal["auto", "low", "high"]] + +class ImagePromptContent(TypedDict): + type: Literal["image_url"] + image_url: ImageUrl + +PromptContent: TypeAlias = str | list[TextPromptContent | ImagePromptContent] + class PromptSection(TypedDict): """ Represents a section of an LLM prompt with an associated role. TypeChat uses the "user" role for @@ -17,7 +35,57 @@ class PromptSection(TypedDict): the prompt in repair attempts). TypeChat currently doesn't use the "system" role. """ role: Literal["system", "user", "assistant"] - content: str + content: PromptContent + +def _normalize_prompt(prompt: object) -> Result[list[PromptSection]]: + if isinstance(prompt, str): + return Success([{"role": "user", "content": prompt}]) + if not isinstance(prompt, list): + return Failure("Prompt must be a string or a list of prompt sections.") + + for section_index, section in enumerate(cast(list[object], prompt)): + location = f"Prompt section {section_index}" + if not isinstance(section, dict): + return Failure(f"{location} must be a prompt-section dictionary.") + fields = cast(dict[object, object], section) + role = fields.get("role") + if not isinstance(role, str) or role not in ("system", "user", "assistant"): + return Failure(f"{location} must have a supported role.") + content = fields.get("content") + if isinstance(content, str): + continue + if not isinstance(content, list): + return Failure(f"{location} content must be a string or a list of tagged parts.") + + for part_index, part in enumerate(cast(list[object], content)): + part_location = f"{location}, part {part_index}" + if not isinstance(part, dict): + return Failure(f"{part_location} must be a tagged text or image_url dictionary.") + part_fields = cast(dict[object, object], part) + part_type = part_fields.get("type") + if not isinstance(part_type, str): + return Failure(f"{part_location} must have a supported type tag.") + if part_type == "text": + if not isinstance(part_fields.get("text"), str): + return Failure(f"{part_location} text must be a string.") + elif part_type == "image_url": + if role != "user": + return Failure(f"{part_location}: images are only supported in user messages.") + image = part_fields.get("image_url") + if not isinstance(image, dict): + return Failure(f"{part_location} image_url must be a dictionary.") + image_fields = cast(dict[object, object], image) + url = image_fields.get("url") + if not isinstance(url, str) or not url: + return Failure(f"{part_location} image_url.url must be a nonempty string.") + if "detail" in image_fields: + detail = image_fields["detail"] + if not isinstance(detail, str) or detail not in ("auto", "low", "high"): + return Failure(f"{part_location} image_url.detail must be auto, low, or high.") + else: + return Failure(f"{part_location} must have a supported type tag.") + + return Success(deepcopy(cast(list[PromptSection], prompt))) TokenProvider = Callable[[], str | Awaitable[str]] @@ -63,19 +131,24 @@ def chat(self) -> _ClientChat: ... class _ClientLanguageModel(TypeChatLanguageModel): _client: ChatCompletionClient _model: str + _temperature: float | None - def __init__(self, client: ChatCompletionClient, model: str): + def __init__(self, client: ChatCompletionClient, model: str, temperature: float | None): super().__init__() self._client = client self._model = model + self._temperature = temperature @override async def complete(self, prompt: str | list[PromptSection]) -> Result[str]: - messages = [{"role": "user", "content": prompt}] if isinstance(prompt, str) else prompt + normalized = _normalize_prompt(prompt) + if isinstance(normalized, Failure): + return normalized + temperature_options = {"temperature": self._temperature} if self._temperature is not None else {} completion = await self._client.chat.completions.create( model=self._model, - messages=messages, - temperature=0.0, + messages=normalized.value, + **temperature_options, n=1, stream=False, ) @@ -86,17 +159,23 @@ async def complete(self, prompt: str | list[PromptSection]) -> Result[str]: return Failure("Client completion returned no text content.") return Success(content) -def create_openai_language_model_from_client(client: ChatCompletionClient, model: str) -> TypeChatLanguageModel: +def create_openai_language_model_from_client( + client: ChatCompletionClient, + model: str, + *, + temperature: float | None = 0.0, +) -> TypeChatLanguageModel: """ Adapts an already configured asynchronous OpenAI-compatible Chat Completions client. TypeChat does not import an SDK, reconfigure or close the client, or add retries or timeouts. Exceptions raised by the client propagate unchanged. The caller supplies the model name, or the deployment name for Azure OpenAI. + Temperature defaults to 0.0; pass None to omit it and use the client's or service's default. """ if not model.strip(): raise ValueError("The client model or deployment name must not be empty.") - return _ClientLanguageModel(client, model) + return _ClientLanguageModel(client, model, temperature) _TRANSIENT_ERROR_CODES = [ 429, @@ -111,10 +190,14 @@ class _ResponseTooLargeError(Exception): def __init__(self, max_bytes: int): super().__init__(f"REST API response exceeded the maximum allowed size of {max_bytes} bytes") +class _HttpResponseMessage(TypedDict): + content: object + class HttpxLanguageModel(TypeChatLanguageModel, AsyncContextManager): url: str headers: dict[str, str] default_params: dict[str, str] + temperature: float | None # Specifies the maximum number of retry attempts. max_retry_attempts: int = 3 # Specifies the delay before retrying in milliseconds. @@ -137,30 +220,34 @@ def __init__( default_params: dict[str, str], *, token_provider: TokenProvider | None = None, + temperature: float | None = 0.0, ): super().__init__() self.url = url self.headers = headers self.default_params = default_params self._token_provider = token_provider + self.temperature = temperature self._async_client = httpx.AsyncClient() @override async def complete(self, prompt: str | list[PromptSection]) -> Success[str] | Failure: + normalized = _normalize_prompt(prompt) + if isinstance(normalized, Failure): + return normalized headers = { "Content-Type": "application/json", **self.headers, } - if isinstance(prompt, str): - prompt = [{"role": "user", "content": prompt}] - body = { **self.default_params, - "messages": prompt, - "temperature": 0.0, + "messages": normalized.value, + "temperature": self.temperature, "n": 1, } + if self.temperature is None: + del body["temperature"] retry_count = 0 while True: try: @@ -182,10 +269,15 @@ async def complete(self, prompt: str | list[PromptSection]) -> Success[str] | Fa if response.is_success: raw = await self._read_capped(response) json_result = cast( - dict[Literal["choices"], list[dict[Literal["message"], PromptSection]]], + dict[Literal["choices"], list[dict[Literal["message"], _HttpResponseMessage]]], json.loads(raw) ) - return Success(json_result["choices"][0]["message"]["content"] or "") + content = json_result["choices"][0]["message"]["content"] + if content is None: + return Success("") + if not isinstance(content, str): + return Failure("REST API completion returned non-text content.") + return Success(content) if response.status_code not in _TRANSIENT_ERROR_CODES or retry_count >= self.max_retry_attempts: return Failure(f"REST API error {response.status_code}: {response.reason_phrase}") @@ -239,7 +331,11 @@ def __del__(self): except Exception: pass -def create_language_model(vals: dict[str, str | None]) -> HttpxLanguageModel: +def create_language_model( + vals: dict[str, str | None], + *, + temperature: float | None = 0.0, +) -> HttpxLanguageModel: """ Creates a language model encapsulation of an OpenAI or Azure OpenAI REST API endpoint chosen by a dictionary of variables (typically just `os.environ`). @@ -255,6 +351,7 @@ def create_language_model(vals: dict[str, str | None]) -> HttpxLanguageModel: Args: vals: A dictionary of variables. Typically just `os.environ`. + temperature: Sampling temperature, defaulting to 0.0. None omits the request parameter. """ def required_var(name: str) -> str: @@ -268,16 +365,23 @@ def required_var(name: str) -> str: model = required_var("OPENAI_MODEL") endpoint = vals.get("OPENAI_ENDPOINT", None) or "https://api.openai.com/v1/chat/completions" org = vals.get("OPENAI_ORGANIZATION", None) or vals.get("OPENAI_ORG", None) or "" - return create_openai_language_model(api_key, model, endpoint, org) + return create_openai_language_model(api_key, model, endpoint, org, temperature=temperature) elif "AZURE_OPENAI_API_KEY" in vals: api_key=required_var("AZURE_OPENAI_API_KEY") endpoint=required_var("AZURE_OPENAI_ENDPOINT") - return create_azure_openai_language_model(api_key, endpoint) + return create_azure_openai_language_model(api_key, endpoint, temperature=temperature) else: raise ValueError("Missing environment variables for OPENAI_API_KEY or AZURE_OPENAI_API_KEY.") -def create_openai_language_model(api_key: str, model: str, endpoint: str = "https://api.openai.com/v1/chat/completions", org: str = "") -> HttpxLanguageModel: +def create_openai_language_model( + api_key: str, + model: str, + endpoint: str = "https://api.openai.com/v1/chat/completions", + org: str = "", + *, + temperature: float | None = 0.0, +) -> HttpxLanguageModel: """ Creates a language model encapsulation of an OpenAI REST API endpoint. @@ -286,6 +390,7 @@ def create_openai_language_model(api_key: str, model: str, endpoint: str = "http model: The OpenAI model name. endpoint: The OpenAI REST API endpoint. org: The OpenAI organization. + temperature: Sampling temperature, defaulting to 0.0. None omits the request parameter. """ headers = { "Authorization": f"Bearer {api_key}", @@ -294,13 +399,16 @@ def create_openai_language_model(api_key: str, model: str, endpoint: str = "http default_params = { "model": model, } - return HttpxLanguageModel(url=endpoint, headers=headers, default_params=default_params) + return HttpxLanguageModel( + url=endpoint, headers=headers, default_params=default_params, temperature=temperature, + ) def create_azure_openai_language_model( api_key: str | None, endpoint: str, *, token_provider: TokenProvider | None = None, + temperature: float | None = 0.0, ) -> HttpxLanguageModel: """ Creates a language model encapsulation of an Azure OpenAI REST API endpoint. @@ -310,11 +418,14 @@ def create_azure_openai_language_model( endpoint: The Azure OpenAI REST API endpoint. token_provider: An optional synchronous or asynchronous bearer-token provider. Invoked for each request attempt; the provider is responsible for caching and renewal. + temperature: Sampling temperature, defaulting to 0.0. None omits the request parameter. """ if token_provider is not None: if api_key is not None: raise ValueError("Specify either an API key or a token provider, not both.") - return HttpxLanguageModel(url=endpoint, headers={}, default_params={}, token_provider=token_provider) + return HttpxLanguageModel( + url=endpoint, headers={}, default_params={}, token_provider=token_provider, temperature=temperature, + ) if api_key is None: raise ValueError("Specify an API key or a token provider.") headers = { @@ -323,4 +434,4 @@ def create_azure_openai_language_model( # Needed when using regular API key "api-key": api_key, } - return HttpxLanguageModel(url=endpoint, headers=headers, default_params={}) + return HttpxLanguageModel(url=endpoint, headers=headers, default_params={}, temperature=temperature) diff --git a/python/tests/test_create_language_model.py b/python/tests/test_create_language_model.py index 8c25dc12..2e82d76d 100644 --- a/python/tests/test_create_language_model.py +++ b/python/tests/test_create_language_model.py @@ -4,8 +4,11 @@ These only inspect the constructed model, so no request is sent. """ +import asyncio + import pytest import typechat +from typechat._internal.model import HttpxLanguageModel def test_openai_organization_is_sent_as_organization_header(): model = typechat.create_language_model( @@ -93,3 +96,31 @@ def test_azure_rejects_ambiguous_authentication(): def test_azure_requires_authentication(): with pytest.raises(ValueError, match="API key or a token provider"): typechat.create_azure_openai_language_model(None, "https://example.invalid/chat/completions") + + +@pytest.mark.parametrize("kind", ["direct", "openai", "azure-key", "azure-token", "env-openai", "env-azure"]) +@pytest.mark.parametrize("temperature", [0.0, 1.0, None]) +def test_http_factories_forward_optional_temperature(kind: str, temperature: float | None): + endpoint = "https://example.invalid/chat/completions" + if kind == "direct": + model = HttpxLanguageModel(endpoint, {}, {}, temperature=temperature) + elif kind == "openai": + model = typechat.create_openai_language_model("test-key", "test-model", temperature=temperature) + elif kind == "azure-key": + model = typechat.create_azure_openai_language_model("test-key", endpoint, temperature=temperature) + elif kind == "azure-token": + model = typechat.create_azure_openai_language_model( + None, endpoint, token_provider=lambda: "test-token", temperature=temperature, + ) + elif kind == "env-openai": + model = typechat.create_language_model( + {"OPENAI_API_KEY": "test-key", "OPENAI_MODEL": "test-model"}, temperature=temperature, + ) + elif kind == "env-azure": + model = typechat.create_language_model( + {"AZURE_OPENAI_API_KEY": "test-key", "AZURE_OPENAI_ENDPOINT": endpoint}, temperature=temperature, + ) + else: + pytest.fail("Unexpected factory case.") + assert model.temperature == temperature + asyncio.run(model.__aexit__(None, None, None)) diff --git a/python/tests/test_multimodal_model.py b/python/tests/test_multimodal_model.py new file mode 100644 index 00000000..340f2ba3 --- /dev/null +++ b/python/tests/test_multimodal_model.py @@ -0,0 +1,513 @@ +import asyncio +import json +from collections.abc import Iterator, Sequence +from copy import deepcopy +from dataclasses import dataclass +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import httpx +import pytest +from typing_extensions import Literal, Protocol, cast + +import typechat +from typechat._internal.model import HttpxLanguageModel + +from .test_prompt_types import inspect_images + + +_ENDPOINT = "https://example.invalid/chat/completions" +_HOSTED_IMAGE = "https://images.example.invalid/private-image.png" +_DATA_IMAGE = "data:image/png;base64,cHJpdmF0ZS1pbWFnZQ==" +_PRIVATE_TEXT = "prompt-data-must-not-leak" + + +@dataclass +class _ModelHarness: + model: typechat.TypeChatLanguageModel + prompts: list[list[typechat.PromptSection]] + provider: Mock + client: Mock | None + http_model: HttpxLanguageModel | None + http_requests: list[httpx.Request] + mode: str + + +class ModelFactory(Protocol): + def __call__( + self, responses: Sequence[object], *, temperature: float | None = 0.0, + ) -> _ModelHarness: ... + + +@pytest.fixture(params=["http-key", "http-token", "client"]) +def model_factory( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[ModelFactory]: + async_client = httpx.AsyncClient + models_to_close: list[HttpxLanguageModel] = [] + mode = cast(str, request.param) + + def create(responses: Sequence[object], *, temperature: float | None = 0.0) -> _ModelHarness: + remaining = iter(responses) + prompts: list[list[typechat.PromptSection]] = [] + http_requests: list[httpx.Request] = [] + provider = Mock(return_value="test-token") + client: Mock | None = None + http_model: HttpxLanguageModel | None = None + model: typechat.TypeChatLanguageModel + + if mode == "client": + configured_temperature = temperature + + async def complete( + *, + model: str, + messages: list[typechat.PromptSection], + n: int, + stream: Literal[False], + temperature: float | None = None, + ) -> SimpleNamespace: + assert (model, temperature, n, stream) == ("test-deployment", configured_temperature, 1, False) + prompts.append(deepcopy(messages)) + response = next(remaining) + if isinstance(response, BaseException): + raise response + choices = [] if isinstance(response, typechat.Failure) else [ + SimpleNamespace(message=SimpleNamespace(content=response)) + ] + return SimpleNamespace(choices=choices) + + client = Mock(timeout=42.0) + client.chat.completions.create = AsyncMock(side_effect=complete) + model = typechat.create_openai_language_model_from_client( + client, "test-deployment", temperature=temperature, + ) + else: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == _ENDPOINT + http_requests.append(request) + prompts.append(cast(list[typechat.PromptSection], json.loads(request.content)["messages"])) + response = next(remaining) + if isinstance(response, BaseException): + raise response + if isinstance(response, httpx.Response): + return response + if isinstance(response, typechat.Failure): + return httpx.Response(400) + return httpx.Response(200, json={"choices": [{"message": {"content": response}}]}) + + transport_client = async_client(transport=httpx.MockTransport(handler)) + monkeypatch.setattr(httpx, "AsyncClient", lambda: transport_client) + if mode == "http-token": + http_model = typechat.create_azure_openai_language_model( + None, _ENDPOINT, token_provider=provider, temperature=temperature, + ) + else: + http_model = typechat.create_azure_openai_language_model( + "test-api-key", _ENDPOINT, temperature=temperature, + ) + http_model.retry_pause_seconds = 0 + models_to_close.append(http_model) + model = http_model + + return _ModelHarness(model, prompts, provider, client, http_model, http_requests, mode) + + yield create + for model in models_to_close: + asyncio.run(model.__aexit__(None, None, None)) + + +def _image_context() -> list[typechat.PromptSection]: + return [ + {"role": "system", "content": [{"type": "text", "text": "Follow the schema."}]}, + {"role": "user", "content": [ + {"type": "text", "text": "First image:"}, + {"type": "image_url", "image_url": {"url": _HOSTED_IMAGE}}, + {"type": "text", "text": "Second image:"}, + {"type": "image_url", "image_url": {"url": _DATA_IMAGE, "detail": "high"}}, + ]}, + {"role": "assistant", "content": [{"type": "text", "text": "Ready."}]}, + ] + + +@pytest.mark.parametrize("prompt", [ + "existing string prompt", + "", + [], + [ + {"role": "system", "content": "Instructions"}, + {"role": "user", "content": "Question"}, + {"role": "assistant", "content": "Previous answer"}, + ], + [ + {"role": role, "content": [{"type": "text", "text": ""}, {"type": "text", "text": "Text"}]} + for role in ("system", "user", "assistant") + ], +]) +def test_preserves_existing_strings_and_tagged_text( + model_factory: ModelFactory, + prompt: str | list[typechat.PromptSection], +): + harness = model_factory(["reply"]) + original = deepcopy(prompt) + assert asyncio.run(harness.model.complete(prompt)) == typechat.Success("reply") + expected = [{"role": "user", "content": prompt}] if isinstance(prompt, str) else prompt + assert harness.prompts == [expected] + assert prompt == original + + +@pytest.mark.parametrize("url", [_HOSTED_IMAGE, _DATA_IMAGE]) +@pytest.mark.parametrize("detail", [None, "auto", "low", "high"]) +def test_preserves_image_references_details_and_order( + model_factory: ModelFactory, + url: str, + detail: Literal["auto", "low", "high"] | None, +): + image: typechat.ImageUrl = {"url": url} + if detail is not None: + image["detail"] = detail + prompt = _image_context() + prompt.append({"role": "user", "content": [ + {"type": "text", "text": "Another image:"}, + {"type": "image_url", "image_url": image}, + {"type": "text", "text": "Describe it."}, + ]}) + original = deepcopy(prompt) + harness = model_factory(["reply"]) + + assert asyncio.run(harness.model.complete(prompt)) == typechat.Success("reply") + assert harness.prompts == [original] + assert prompt == original + if harness.mode == "http-token": + harness.provider.assert_called_once_with() + assert "api-key" not in harness.http_requests[0].headers + elif harness.mode == "http-key": + harness.provider.assert_not_called() + assert harness.http_requests[0].headers["api-key"] == "test-api-key" + else: + assert harness.client is not None + assert harness.client.timeout == 42.0 + harness.client.close.assert_not_called() + harness.client.chat.completions.create.assert_awaited_once() + forwarded = cast( + list[typechat.PromptSection], + harness.client.chat.completions.create.call_args.kwargs["messages"], + ) + assert forwarded is not prompt + content = forwarded[1]["content"] + assert isinstance(content, list) + part = content[1] + assert part["type"] == "image_url" + part["image_url"]["url"] = "changed by the configured client" + assert prompt == original + + +@pytest.mark.parametrize("temperature", [0.0, 1.0, None]) +def test_temperature_is_configurable_and_none_omits_the_field( + model_factory: ModelFactory, + temperature: float | None, +): + harness = model_factory(["reply"], temperature=temperature) + if harness.http_model is not None: + harness.http_model.default_params["temperature"] = "must-not-override-the-option" + assert asyncio.run(harness.model.complete(_image_context())) == typechat.Success("reply") + if harness.client is not None: + params = harness.client.chat.completions.create.call_args.kwargs + else: + params = json.loads(harness.http_requests[0].content) + if temperature is None: + assert "temperature" not in params + else: + assert params["temperature"] == temperature + + +def _assert_invalid_prompt( + harness: _ModelHarness, + result: typechat.Result[object], + location: str, +) -> None: + assert isinstance(result, typechat.Failure) + assert location in result.message + assert _PRIVATE_TEXT not in result.message + assert _HOSTED_IMAGE not in result.message + assert _DATA_IMAGE not in result.message + assert not result.has_value + assert result.response_text is None + assert harness.prompts == [] + harness.provider.assert_not_called() + if harness.client is not None: + harness.client.chat.completions.create.assert_not_called() + + +_MALFORMED_PROMPTS: list[tuple[object, str]] = [ + (None, "Prompt"), + (42, "Prompt"), + (b"text", "Prompt"), + ({"role": "user", "content": _PRIVATE_TEXT}, "Prompt"), + (({"role": "user", "content": _PRIVATE_TEXT},), "Prompt"), + ([None], "section 0"), + ([_PRIVATE_TEXT], "section 0"), + ([{}], "section 0"), + ([{"content": _PRIVATE_TEXT}], "section 0"), + ([{"role": _PRIVATE_TEXT, "content": _PRIVATE_TEXT}], "section 0"), + ([{"role": None, "content": _PRIVATE_TEXT}], "section 0"), + ([{"role": [], "content": _PRIVATE_TEXT}], "section 0"), + ([{"role": "user"}], "section 0"), + ([{"role": "user", "content": None}], "section 0"), + ([{"role": "user", "content": 42}], "section 0"), + ([{"role": "user", "content": {"type": "text", "text": _PRIVATE_TEXT}}], "section 0"), + ([{"role": "user", "content": ({"type": "text", "text": _PRIVATE_TEXT},)}], "section 0"), +] + + +@pytest.mark.parametrize("prompt, location", _MALFORMED_PROMPTS) +def test_rejects_malformed_sections_before_authentication_or_dispatch( + model_factory: ModelFactory, + prompt: object, + location: str, +): + harness = model_factory([]) + result = asyncio.run(harness.model.complete(cast(str | list[typechat.PromptSection], prompt))) + _assert_invalid_prompt(harness, result, location) + + +@pytest.mark.parametrize("part", [ + _PRIVATE_TEXT, + None, + 42, + [], + {}, + {"text": _PRIVATE_TEXT}, + {"type": "audio", "audio": _PRIVATE_TEXT}, + {"type": 42, "text": _PRIVATE_TEXT}, + {"type": "text"}, + {"type": "text", "text": None}, + {"type": "text", "text": 42}, + {"type": "text", "text": []}, + {"type": "image_url"}, + {"type": "image_url", "image_url": _HOSTED_IMAGE}, + {"type": "image_url", "image_url": None}, + {"type": "image_url", "image_url": []}, + {"type": "image_url", "image_url": {}}, + {"type": "image_url", "image_url": {"url": ""}}, + {"type": "image_url", "image_url": {"url": None}}, + {"type": "image_url", "image_url": {"url": 42}}, + {"type": "image_url", "image_url": {"url": []}}, + {"type": "image_url", "image_url": {"url": _HOSTED_IMAGE, "detail": _PRIVATE_TEXT}}, + {"type": "image_url", "image_url": {"url": _DATA_IMAGE, "detail": None}}, + {"type": "image_url", "image_url": {"url": _HOSTED_IMAGE, "detail": []}}, + {"type": "image_url", "image_url": {"url": _HOSTED_IMAGE, "detail": 42}}, +]) +def test_rejects_malformed_parts_without_exposing_content(model_factory: ModelFactory, part: object): + prompt = [ + {"role": "system", "content": "Instructions"}, + {"role": "user", "content": [{"type": "text", "text": _PRIVATE_TEXT}, part]}, + ] + harness = model_factory([]) + result = asyncio.run(harness.model.complete(cast(list[typechat.PromptSection], prompt))) + _assert_invalid_prompt(harness, result, "section 1, part 1") + + +@pytest.mark.parametrize("role", ["system", "assistant"]) +def test_images_are_only_supported_in_user_messages(model_factory: ModelFactory, role: str): + prompt = [{"role": role, "content": [{"type": "image_url", "image_url": {"url": _HOSTED_IMAGE}}]}] + harness = model_factory([]) + result = asyncio.run(harness.model.complete(cast(list[typechat.PromptSection], prompt))) + _assert_invalid_prompt(harness, result, "section 0, part 0") + assert isinstance(result, typechat.Failure) + assert "only supported in user messages" in result.message + + +@pytest.mark.parametrize("content", ["reply", "", None, 0, False, [], {}, [{"type": "text", "text": "reply"}]]) +def test_completions_remain_text_only(model_factory: ModelFactory, content: object): + harness = model_factory([content]) + result = asyncio.run(harness.model.complete(_image_context())) + if isinstance(content, str): + assert result == typechat.Success(content) + elif content is None and harness.http_model is not None: + assert result == typechat.Success("") + else: + assert isinstance(result, typechat.Failure) + assert "content" in result.message + assert not result.has_value + assert result.response_text is None + assert len(harness.prompts) == 1 + + +@dataclass +class _ImageResult: + value: int + + +@pytest.mark.parametrize("responses, repair_limit, expected", [ + (['{"value":1}'], 1, typechat.Success(_ImageResult(1))), + (["not JSON", '{"value":2}'], 1, typechat.Success(_ImageResult(2))), + (['{"value":}', '{"value":2}'], 1, typechat.Success(_ImageResult(2))), + (['{"value":"wrong"}', '{"value":3}'], 1, typechat.Success(_ImageResult(3))), + ( + ['{"value":"wrong"}', '{"value":"still wrong"}'], + 1, + typechat.Failure("", value={"value": "still wrong"}, response_text='{"value":"still wrong"}'), + ), + ( + ['{"value":"wrong"}', "not JSON"], + 1, + typechat.Failure("", value={"value": "wrong"}, response_text="not JSON"), + ), + ( + ['{"value":"wrong"}', typechat.Failure("model failure")], + 1, + typechat.Failure("", value={"value": "wrong"}, response_text='{"value":"wrong"}'), + ), + ( + ['{"value":"wrong"}'], + 0, + typechat.Failure("", value={"value": "wrong"}, response_text='{"value":"wrong"}'), + ), + ( + ['{"value":"wrong"}', '{"value":"still wrong"}', '{"value":4}'], + 2, + typechat.Success(_ImageResult(4)), + ), + ([typechat.Failure("initial model failure")], 1, typechat.Failure("")), +]) +def test_translator_retains_image_context_layout_repair_limits_and_failure_payloads( + model_factory: ModelFactory, + responses: list[object], + repair_limit: int, + expected: typechat.Result[_ImageResult], +): + harness = model_factory(responses) + class LimitedTranslator(typechat.TypeChatJsonTranslator[_ImageResult]): + _max_repair_attempts = repair_limit + + translator = LimitedTranslator( + harness.model, typechat.TypeChatValidator(_ImageResult), _ImageResult, + ) + preamble = _image_context() + original = deepcopy(preamble) + result = asyncio.run(translator.translate("Describe the images.", prompt_preamble=preamble)) + + if isinstance(expected, typechat.Success): + assert result == expected + else: + assert isinstance(result, typechat.Failure) + assert result.message + assert result.has_value == expected.has_value + assert result.value == expected.value + assert result.response_text == expected.response_text + assert preamble == original + assert len(harness.prompts) == len(responses) + expected_request: typechat.PromptSection = {"role": "user", "content": f""" +You are a service that translates user requests into JSON objects of type "{translator.type_name}" according to the following TypeScript definitions: +``` +{translator.schema_str} +``` +The following is a user request: +''' +Describe the images. +''' +The following is the user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined: +"""} + first_prompt = [*original, expected_request] + for attempt, prompt in enumerate(harness.prompts): + assert prompt[:len(first_prompt)] == first_prompt + assert len(prompt) == len(first_prompt) + 2 * attempt + if attempt: + assert prompt[:-2] == harness.prompts[attempt - 1] + assert prompt[-2] == {"role": "assistant", "content": responses[attempt - 1]} + assert prompt[-1]["role"] == "user" + repair = prompt[-1]["content"] + assert isinstance(repair, str) + assert repair.startswith("\nThe above JSON object is invalid for the following reason:\n'''\n") + assert repair.endswith("\nThe following is a revised JSON object:\n") + + +def test_malformed_preamble_does_not_trigger_translator_repair( + model_factory: ModelFactory, + monkeypatch: pytest.MonkeyPatch, +): + harness = model_factory([]) + complete = AsyncMock(wraps=harness.model.complete) + monkeypatch.setattr(harness.model, "complete", complete) + translator = typechat.TypeChatJsonTranslator( + harness.model, typechat.TypeChatValidator(_ImageResult), _ImageResult, + ) + preamble = cast(list[typechat.PromptSection], [{"role": "user", "content": [_PRIVATE_TEXT]}]) + result = asyncio.run(translator.translate("Describe the images.", prompt_preamble=preamble)) + _assert_invalid_prompt(harness, result, "section 0, part 0") + complete.assert_awaited_once() + + +def test_documented_model_and_translator_examples_work_offline(model_factory: ModelFactory): + harness = model_factory(["image description", '{"description":"two images"}']) + text, data = asyncio.run(inspect_images(harness.model, "aW1hZ2U=")) + assert text == typechat.Success("image description") + assert data == typechat.Success({"description": "two images"}) + assert len(harness.prompts) == 2 + assert harness.prompts[1][:-1] == harness.prompts[0] + + +@pytest.mark.parametrize("model_factory", ["http-key", "http-token"], indirect=True) +def test_image_prompts_preserve_http_retries_and_per_attempt_authentication(model_factory: ModelFactory): + harness = model_factory([httpx.Response(429), httpx.Response(500), "reply"]) + assert harness.http_model is not None + harness.http_model.max_retry_attempts = 2 + harness.provider.side_effect = ["first-token", "second-token", "third-token"] + prompt = _image_context() + original = deepcopy(prompt) + + assert asyncio.run(harness.model.complete(prompt)) == typechat.Success("reply") + assert harness.prompts == [original, original, original] + assert prompt == original + if harness.mode == "http-token": + assert harness.provider.call_count == 3 + assert [request.headers["Authorization"] for request in harness.http_requests] == [ + "Bearer first-token", "Bearer second-token", "Bearer third-token", + ] + else: + harness.provider.assert_not_called() + assert all(request.headers["api-key"] == "test-api-key" for request in harness.http_requests) + + +@pytest.mark.parametrize("model_factory", ["http-key", "http-token"], indirect=True) +def test_image_prompts_preserve_http_timeout_and_response_size_controls(model_factory: ModelFactory): + harness = model_factory(["x" * 1000]) + assert harness.http_model is not None + harness.http_model.timeout_seconds = 7 + harness.http_model.max_response_bytes = 100 + result = asyncio.run(harness.model.complete(_image_context())) + assert isinstance(result, typechat.Failure) + assert "maximum allowed size" in result.message + assert len(harness.http_requests) == 1 + assert harness.http_requests[0].extensions["timeout"] == { + "connect": 7, "read": 7, "write": 7, "pool": 7, + } + + +@pytest.mark.parametrize("model_factory", ["client"], indirect=True) +@pytest.mark.parametrize("exception", [RuntimeError("Client failure."), asyncio.CancelledError()]) +def test_image_prompts_preserve_client_exceptions_and_ownership( + model_factory: ModelFactory, + exception: BaseException, +): + harness = model_factory([exception]) + with pytest.raises(type(exception)) as raised: + asyncio.run(harness.model.complete(_image_context())) + assert raised.value is exception + assert len(harness.prompts) == 1 + assert harness.client is not None + assert harness.client.timeout == 42.0 + harness.client.close.assert_not_called() + harness.client.chat.completions.create.assert_awaited_once() + + +@pytest.mark.parametrize("model_factory", ["http-token"], indirect=True) +def test_image_prompts_preserve_token_provider_cancellation(model_factory: ModelFactory): + harness = model_factory([]) + harness.provider.side_effect = asyncio.CancelledError() + with pytest.raises(asyncio.CancelledError): + asyncio.run(harness.model.complete(_image_context())) + harness.provider.assert_called_once_with() + assert harness.http_requests == [] diff --git a/python/tests/test_prompt_types.py b/python/tests/test_prompt_types.py new file mode 100644 index 00000000..5a427a2f --- /dev/null +++ b/python/tests/test_prompt_types.py @@ -0,0 +1,104 @@ +from inspect import getsource +from pathlib import Path +from typing import TYPE_CHECKING + +from typing_extensions import Literal, TypedDict, assert_type + +from typechat import ( + ChatCompletionClient, + ImagePromptContent, + ImageUrl, + PromptContent, + PromptSection, + Result, + TextPromptContent, + TypeChatJsonTranslator, + TypeChatLanguageModel, + TypeChatValidator, + create_azure_openai_language_model, + create_language_model, + create_openai_language_model, + create_openai_language_model_from_client, +) + + +class ImageDescription(TypedDict): + description: str + + +async def inspect_images( + model: TypeChatLanguageModel, image_base64: str +) -> tuple[Result[str], Result[ImageDescription]]: + context: list[PromptSection] = [ + {"role": "user", "content": [ + {"type": "text", "text": "Use these images as context."}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.png"}}, + {"type": "image_url", "image_url": { + "url": f"data:image/png;base64,{image_base64}", + "detail": "low", + }}, + ]}, + ] + text = await model.complete(context) + translator = TypeChatJsonTranslator(model, TypeChatValidator(ImageDescription), ImageDescription) + data = await translator.translate("Describe the images.", prompt_preamble=context) + return text, data + + +def test_public_prompt_types_retain_dictionary_construction(): + text: TextPromptContent = {"type": "text", "text": "Describe this image."} + image_url: ImageUrl = {"url": "https://example.com/photo.png"} + image: ImagePromptContent = {"type": "image_url", "image_url": image_url} + content: PromptContent = [text, image] + section = PromptSection(role="user", content=content) + assert_type(section, PromptSection) + assert_type(text["type"], Literal["text"]) + assert_type(text["text"], str) + assert_type(image["type"], Literal["image_url"]) + assert_type(image["image_url"], ImageUrl) + assert section == {"role": "user", "content": [text, image]} + assert "detail" not in image_url + + +def test_documentation_examples_match_typechecked_functions(): + root = Path(__file__).resolve().parents[2] + example = getsource(ImageDescription).rstrip() + "\n\n\n" + getsource(inspect_images).rstrip() + for path in (root / "python" / "README.md", root / "site" / "src" / "docs" / "python" / "basic-usage.md"): + assert example in path.read_text(encoding="utf-8") + + +if TYPE_CHECKING: + missing_text_tag: TextPromptContent = {"text": "text"} # type: ignore + missing_text: TextPromptContent = {"type": "text"} # type: ignore + wrong_text: TextPromptContent = {"type": "text", "text": 42} # type: ignore + unknown_tag: TextPromptContent = {"type": "audio", "text": "text"} # type: ignore + missing_image: ImagePromptContent = {"type": "image_url"} # type: ignore + wrong_image: ImagePromptContent = {"type": "image_url", "image_url": "https://example.com"} # type: ignore + missing_url: ImageUrl = {"detail": "auto"} # type: ignore + wrong_url: ImageUrl = {"url": 42} # type: ignore + wrong_detail: ImageUrl = {"url": "https://example.com", "detail": "medium"} # type: ignore + bare_part: PromptContent = ["text"] # type: ignore + wrong_role: PromptSection = {"role": "tool", "content": "text"} # type: ignore + + async def check_completion_and_request_types( + model: TypeChatLanguageModel, + translator: TypeChatJsonTranslator[ImageDescription], + client: ChatCompletionClient, + ) -> None: + sections: list[PromptSection] = [{"role": "user", "content": "existing text"}] + assert_type(await model.complete("existing text"), Result[str]) + assert_type(await model.complete(sections), Result[str]) + assert_type(await translator.translate("existing text"), Result[ImageDescription]) + assert_type( + await translator.translate("existing text", prompt_preamble="context"), + Result[ImageDescription], + ) + assert_type( + await translator.translate("existing text", prompt_preamble=sections), + Result[ImageDescription], + ) + await translator.translate(sections) # type: ignore + create_language_model({}, temperature="default") # type: ignore + create_openai_language_model("key", "model", temperature="default") # type: ignore + create_azure_openai_language_model("key", "https://example.com", temperature="default") # type: ignore + create_openai_language_model_from_client(client, "model", temperature="default") # type: ignore diff --git a/site/src/docs/python/basic-usage.md b/site/src/docs/python/basic-usage.md index 1ba85f41..59c63b1c 100644 --- a/site/src/docs/python/basic-usage.md +++ b/site/src/docs/python/basic-usage.md @@ -76,7 +76,9 @@ def create_openai_language_model( api_key: str, model: str, endpoint: str = "https://api.openai.com/v1/chat/completions", - org: str = "" + org: str = "", + *, + temperature: float | None = 0.0, ): ... @@ -84,7 +86,8 @@ def create_azure_openai_language_model( api_key: str | None, endpoint: str, *, - token_provider: TokenProvider | None = None + token_provider: TokenProvider | None = None, + temperature: float | None = 0.0, ): ... ``` @@ -93,7 +96,9 @@ you're using OpenAI or Azure OpenAI. ```py def create_language_model( - vals: dict[str, str | None] + vals: dict[str, str | None], + *, + temperature: float | None = 0.0, ) -> TypeChatLanguageModel: ... ``` @@ -171,8 +176,8 @@ model = create_openai_language_model_from_client( The adapter uses `client.chat.completions.create` and reads `choices[0].message.content`. It requires no environment variables and adds no SDK dependency to TypeChat. It sends TypeChat's prompt sections with -`temperature=0.0`, `n=1`, and `stream=False`; the caller supplies the model or -Azure deployment name. +`n=1` and `stream=False`; the caller supplies the model or Azure deployment name. +Temperature defaults to `0.0` and can be configured or omitted as described below. Your client's endpoint, API version, authentication, retries, and timeouts remain configured by your application. TypeChat does not reconfigure or close @@ -183,6 +188,33 @@ The adapter is for Chat Completions, not the Responses API. Pass either model to `TypeChatJsonTranslator` as usual. Schema generation, validation, and repair behavior remain unchanged. +#### Configuring temperature + +All four model factories accept a keyword-only `temperature: float | None` +option. Leaving it unspecified retains the existing `0.0` setting. Pass a number +to configure it, or `None` to omit the field entirely (not send JSON `null`) and +use the configured client's or service's default: + +```py +model = create_azure_openai_language_model( + api_key=None, + endpoint=chat_completions_endpoint, + token_provider=token_provider, + temperature=None, +) + +model = create_openai_language_model_from_client( + openai_client, + model="my-model-or-azure-deployment", + temperature=None, +) +``` + +Omission supports reasoning models that reject explicit sampling parameters, +including models that do not support disabling reasoning. It does not set a +reasoning effort, infer model capabilities, change authentication, or switch APIs. +Other client settings, ownership, retries, and timeouts are unchanged. + #### Using OpenAI-compatible gateways TypeChat works with any service that exposes the OpenAI Chat Completions API. @@ -230,6 +262,104 @@ import typechat model = typechat.create_language_model(os.environ) ``` +## Text and image prompts + +The HTTP Chat Completions models and configured-client adapter accept image +references alongside text. Use `model.complete` for text output, or supply image +context through the translator's existing `prompt_preamble` for validated data. +The calls below demonstrate both interfaces; an application can use either +independently. Replace the hosted URL with your own and pass already-base64-encoded +image data: + +```py +from typing_extensions import TypedDict + +from typechat import ( + PromptSection, + Result, + TypeChatJsonTranslator, + TypeChatLanguageModel, + TypeChatValidator, +) + + +class ImageDescription(TypedDict): + description: str + + +async def inspect_images( + model: TypeChatLanguageModel, image_base64: str +) -> tuple[Result[str], Result[ImageDescription]]: + context: list[PromptSection] = [ + {"role": "user", "content": [ + {"type": "text", "text": "Use these images as context."}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.png"}}, + {"type": "image_url", "image_url": { + "url": f"data:image/png;base64,{image_base64}", + "detail": "low", + }}, + ]}, + ] + text = await model.complete(context) + translator = TypeChatJsonTranslator(model, TypeChatValidator(ImageDescription), ImageDescription) + data = await translator.translate("Describe the images.", prompt_preamble=context) + return text, data +``` + +These are ordinary dictionaries, with the following public types exported by +`typechat`: + +```py +from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict + +class TextPromptContent(TypedDict): + type: Literal["text"] + text: str + +class ImageUrl(TypedDict): + url: str + detail: NotRequired[Literal["auto", "low", "high"]] + +class ImagePromptContent(TypedDict): + type: Literal["image_url"] + image_url: ImageUrl + +PromptContent: TypeAlias = str | list[TextPromptContent | ImagePromptContent] + +class PromptSection(TypedDict): + role: Literal["system", "user", "assistant"] + content: PromptContent +``` + +Whole-string content still works in every role. Lists must contain tagged +`text` or `image_url` dictionaries, not bare strings. Tagged text is supported +in system, user, and assistant messages; images are supported only in user +messages. Image references must be nonempty strings. Omit `detail` to leave it +unspecified, or use `auto`, `low`, or `high`. The models preserve message and part +ordering, exact URL/data-URL strings, and caller-owned dictionaries. Malformed +or unsupported input returns a `Failure` identifying the section or part without +echoing its content, before resolving a token or dispatching a request. + +The translator's current request remains a string, and ordinary `translate(str)` +calls need no changes. Preamble messages remain before the combined schema/request +message; validation and repair keep the original context and append rejected +assistant text followed by user diagnostics. This does not adopt the alternative +prompt layout proposed in [#241](https://github.com/microsoft/TypeChat/pull/241). +Model completions remain text-only, and schema generation and strict result +validation are unchanged. + +Supply a provider/model that supports images and follow its supported file formats +(for example PNG, JPEG, or WebP), accessible-URL requirements, image-count limits, +and encoded-payload, resolution, and size limits. TypeChat validates reference +structure only. It never fetches, decodes, resizes, or validates image bytes and +does not add image generation, audio/video, streaming, or Responses API support. +No new dependency is required. Existing API-key/token-provider authentication, +HTTP retries/timeouts/response-size limits, and caller-owned SDK configuration and +lifecycle remain unchanged. Custom models must handle the prompt parts they accept. + +For a separate application example, see the +[contributed TypeScript image example](https://github.com/microsoft/TypeChat/pull/258). + ## Defining and Loading the Schema TypeChat describes types to language models to help guide their responses.