From 3ddd4580e3f37bc1ff9588dc5a04a51911c4328b Mon Sep 17 00:00:00 2001 From: PSA-Team-source <155616440+PSA-Team-source@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:21:56 +0700 Subject: [PATCH] Add UberLLM as a model gateway backend --- docs/cookbook/02-models.md | 17 ++- docs/deep-dive.md | 2 +- grapharc/cli/main.py | 4 + grapharc/cli/probe.py | 21 +++ grapharc/gateway/__init__.py | 10 +- grapharc/gateway/config.py | 15 ++- grapharc/gateway/registry.py | 15 ++- grapharc/gateway/uberllm.py | 62 +++++++++ pyproject.toml | 8 +- tests/test_cli.py | 4 +- tests/test_cookbook_models.py | 9 +- tests/test_gateway_openai_ollama.py | 2 +- tests/test_gateway_openrouter.py | 2 +- tests/test_gateway_uberllm.py | 201 ++++++++++++++++++++++++++++ uv.lock | 8 +- 15 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 grapharc/gateway/uberllm.py create mode 100644 tests/test_gateway_uberllm.py diff --git a/docs/cookbook/02-models.md b/docs/cookbook/02-models.md index 573f64f..dceb972 100644 --- a/docs/cookbook/02-models.md +++ b/docs/cookbook/02-models.md @@ -25,6 +25,7 @@ for spec in ( "openrouter/anthropic/claude-haiku-4.5", "openai/gpt-4o-mini", "novita/moonshotai/kimi-k3", + "uberllm/deepseek/deepseek-v4.1-flash", "ollama/llama3.1", "mock/anything", ): @@ -40,12 +41,13 @@ print(model.invoke("say hi").content) {'spec': 'openrouter/anthropic/claude-haiku-4.5', 'backend': 'openrouter', 'model': 'anthropic/claude-haiku-4.5'} {'spec': 'openai/gpt-4o-mini', 'backend': 'openai', 'model': 'gpt-4o-mini'} {'spec': 'novita/moonshotai/kimi-k3', 'backend': 'novita', 'model': 'moonshotai/kimi-k3'} +{'spec': 'uberllm/deepseek/deepseek-v4.1-flash', 'backend': 'uberllm', 'model': 'deepseek/deepseek-v4.1-flash'} {'spec': 'ollama/llama3.1', 'backend': 'ollama', 'model': 'llama3.1'} {'spec': 'mock/anything', 'backend': 'mock', 'model': 'anything'} hello from a scripted model ``` -There are six backends: +There are seven backends: | backend | credential | what it is for | | --- | --- | --- | @@ -53,6 +55,7 @@ There are six backends: | `openrouter` | `OPENROUTER_API_KEY` | one key, most vendors, per-call cost in the response | | `openai` | `OPENAI_API_KEY` | the OpenAI API directly, or any endpoint via `OPENAI_BASE_URL` | | `novita` | `NOVITA_API_KEY` | Novita's own endpoint, token counts but no per-call cost | +| `uberllm` | `UBERLLM_API_KEY` | one key, many vendors, per-call cost in the response | | `ollama` | none — a local server | models on your own machine, free and offline | | `mock` | none | a scripted test double | @@ -66,8 +69,8 @@ needs it. Asking for `claude-cli` therefore does not require `langchain-openai`, for `openrouter` does not require the Claude CLI to be installed. A missing optional dependency fails for the backend that wanted it and nothing else. -`openrouter`, `openai`, `novita` and `ollama` all speak the OpenAI wire format and share one -base class, so they behave identically on everything except money and routing: same +`openrouter`, `openai`, `novita`, `uberllm` and `ollama` all speak the OpenAI wire format and +share one base class, so they behave identically on everything except money and routing: same `bind_tools`, same `with_structured_output`, same streaming and async, same retry policy, same usage envelope. @@ -96,7 +99,7 @@ except UnknownBackendError as exc: ('openrouter', 'openai/gpt-4o-mini:floor') ('claude-cli', 'anthropic/claude-haiku-4.5') ('openai', 'gpt-4o-mini') -UnknownBackendError: unknown backend 'opnerouter' in spec 'opnerouter/openai/gpt-4o-mini'; expected one of: claude-cli, openrouter, openai, novita, ollama, mock — or a bare model name for the claude-cli default +UnknownBackendError: unknown backend 'opnerouter' in spec 'opnerouter/openai/gpt-4o-mini'; expected one of: claude-cli, openrouter, openai, novita, uberllm, ollama, mock — or a bare model name for the claude-cli default ``` Only the first segment is a backend, because OpenRouter model ids are themselves @@ -125,10 +128,11 @@ model through the broker is still `openrouter/openai/gpt-4o-mini`. ```console $ grapharc models -backends: claude-cli, openrouter, openai, novita, ollama, mock +backends: claude-cli, openrouter, openai, novita, uberllm, ollama, mock openrouter key: openai key: novita key: +uberllm key: ollama url: http://localhost:11434/v1 examples: @@ -137,6 +141,7 @@ examples: openrouter/openai/gpt-4o-mini:floor cheapest provider for that model openai/gpt-4o-mini the OpenAI API directly, your key novita/moonshotai/kimi-k3 Novita's own endpoint, your key + uberllm/deepseek/deepseek-v4.1-flash one key, many vendors, cost per call ollama/llama3.1 a local server, no key and no bill grapharc models --check probes which of these this machine can use @@ -172,6 +177,8 @@ openai unusable no API key (set OPENAI_API_KEY, or add one to .env) credential: novita unusable no API key (set NOVITA_API_KEY, or add one to .env) credential: +uberllm unusable no API key (set UBERLLM_API_KEY, or add one to .env) + credential: ollama usable local server at http://localhost:11434/v1 credential: none needed (local server) mock usable scripted test double; never reaches a provider diff --git a/docs/deep-dive.md b/docs/deep-dive.md index 06dff29..9ed38d0 100644 --- a/docs/deep-dive.md +++ b/docs/deep-dive.md @@ -256,7 +256,7 @@ A stable system is not one that claims to have no edges — it is one whose edge - **`.env` and `grapharc.toml` follow the same discovery rule: the working directory, and nowhere else.** Neither searches parent directories — a run must not be governed by a file you did not know about, and must not be *billed* to one either. **This is a behaviour change:** the credential loader used to walk up to `/`, so a `.env` in an ancestor directory (a `$HOME` one on a shared box, a client project one above a demo checkout) was picked up silently. If you relied on that, move the file into the directory you run from, `export` the variable, or pass `env_file=` to name it explicitly. A real environment variable still beats any file. - **`grapharc run` has no budget unless you give it one.** Set any of `--max-tokens`, `--max-iterations`, `--max-seconds`, or `--max-concurrency`; without them each dimension is unlimited and the gate admits a topology of any worst-case cost. -**Verified this pass:** `pytest` → green, 2,408 selected and 13 deselected (the live ones); `ruff check .` clean; all eight `grapharc demo` stages green, plus the `trace` / `metrics` / `viz` / `replay` tour against a freshly recorded demo trace; the wheel builds and imports all submodules in a clean virtualenv with `[all]`, and `0.1.8` on PyPI is that wheel. The counts are a snapshot, not a property of the project — `pytest` re-derives them in one command, which is the only reason they are quoted, and `tests/test_deep_dive.py` fails this line rather than letting it drift. +**Verified this pass:** `pytest` → green, 2,425 selected and 14 deselected (the live ones); `ruff check .` clean; all eight `grapharc demo` stages green, plus the `trace` / `metrics` / `viz` / `replay` tour against a freshly recorded demo trace; the wheel builds and imports all submodules in a clean virtualenv with `[all]`, and `0.1.8` on PyPI is that wheel. The counts are a snapshot, not a property of the project — `pytest` re-derives them in one command, which is the only reason they are quoted, and `tests/test_deep_dive.py` fails this line rather than letting it drift. [ROADMAP.md](../ROADMAP.md) tracks what is built and what is not, item by item. diff --git a/grapharc/cli/main.py b/grapharc/cli/main.py index c815f92..f2a82fe 100644 --- a/grapharc/cli/main.py +++ b/grapharc/cli/main.py @@ -470,6 +470,7 @@ def _cmd_models(args: argparse.Namespace) -> int: openai_api_key, openrouter_api_key, redact, + uberllm_api_key, ) from grapharc.gateway.registry import BACKENDS @@ -515,6 +516,7 @@ def _cmd_models(args: argparse.Namespace) -> int: "openrouter/openai/gpt-4o-mini:floor": "cheapest provider for that model", "openai/gpt-4o-mini": "the OpenAI API directly, your key", "novita/moonshotai/kimi-k3": "Novita's own endpoint, your key", + "uberllm/deepseek/deepseek-v4.1-flash": "one key, many vendors, cost per call", "ollama/llama3.1": "a local server, no key and no bill", } payload = { @@ -524,6 +526,7 @@ def _cmd_models(args: argparse.Namespace) -> int: "openrouter_key": redact(openrouter_api_key()), "openai_key": redact(openai_api_key()), "novita_key": redact(novita_api_key()), + "uberllm_key": redact(uberllm_api_key()), # An address, not a secret: it is printed whole, and it is where a # request would go rather than proof that anything is listening. "ollama_base_url": ollama_base_url(), @@ -536,6 +539,7 @@ def _cmd_models(args: argparse.Namespace) -> int: style.kv("openrouter key", redact(openrouter_api_key())), style.kv("openai key", redact(openai_api_key())), style.kv("novita key", redact(novita_api_key())), + style.kv("uberllm key", redact(uberllm_api_key())), style.kv("ollama url", ollama_base_url(), tint=style.accent), "", style.heading("examples:"), diff --git a/grapharc/cli/probe.py b/grapharc/cli/probe.py index 703c3b7..6f5ce8e 100644 --- a/grapharc/cli/probe.py +++ b/grapharc/cli/probe.py @@ -115,6 +115,26 @@ def _probe_novita() -> dict[str, Any]: } +def _probe_uberllm() -> dict[str, Any]: + from grapharc.gateway import redact, uberllm_api_key + + key = uberllm_api_key() + has_dependency = importlib.util.find_spec("langchain_openai") is not None + missing = [] + if not key: + missing.append("no API key (set UBERLLM_API_KEY, or add one to .env)") + if not has_dependency: + missing.append("langchain-openai not installed (uv sync --extra uberllm)") + return { + "backend": "uberllm", + "kind": KIND_PROVIDER, + "usable": bool(key) and has_dependency, + "credential": redact(key), + "detail": "; ".join(missing) or "api key configured and langchain-openai installed", + "checked": "credential presence only; no request was sent to api.uberllm.dev", + } + + def _probe_ollama() -> dict[str, Any]: from grapharc.gateway import ollama_base_url @@ -173,6 +193,7 @@ def probe_backends(*, claude_path: str = "claude") -> list[dict[str, Any]]: "openrouter": _probe_openrouter, "openai": _probe_openai, "novita": _probe_novita, + "uberllm": _probe_uberllm, "ollama": _probe_ollama, "mock": _probe_mock, } diff --git a/grapharc/gateway/__init__.py b/grapharc/gateway/__init__.py index 47a9c14..77ea0f5 100644 --- a/grapharc/gateway/__init__.py +++ b/grapharc/gateway/__init__.py @@ -7,6 +7,7 @@ get_model("openrouter/anthropic/claude-sonnet-4.5") # many providers, one key get_model("openai/gpt-4o-mini") # OPENAI_API_KEY get_model("novita/moonshotai/kimi-k3") # NOVITA_API_KEY + get_model("uberllm/deepseek/deepseek-v4.1-flash") # UBERLLM_API_KEY get_model("ollama/llama3.1") # local server, no key get_model("mock/x", responses=[...]) # deterministic tests @@ -17,8 +18,9 @@ get_model(spec, cost_ceiling_usd=0.25) # raises when passed get_model(spec, spend=shared_meter) # one ceiling, many models -The four OpenAI-wire backends (`openrouter`, `openai`, `novita`, `ollama`) are -imported lazily — they need `langchain-openai`, which is an optional extra. +The five OpenAI-wire backends (`openrouter`, `openai`, `novita`, `uberllm`, +`ollama`) are imported lazily — they need `langchain-openai`, which is an +optional extra. """ from grapharc.gateway.claude_cli import ClaudeCodeCLIChatModel @@ -30,6 +32,7 @@ openai_base_url, openrouter_api_key, redact, + uberllm_api_key, ) from grapharc.gateway.errors import ( CostCeilingExceeded, @@ -76,6 +79,7 @@ "openrouter_api_key", "redact", "split_spec", + "uberllm_api_key", "vendor", ] @@ -90,6 +94,8 @@ "OpenAIError": "openai", "NovitaChatModel": "novita", "NovitaError": "novita", + "UberLLMChatModel": "uberllm", + "UberLLMError": "uberllm", "OllamaChatModel": "ollama", "OllamaError": "ollama", } diff --git a/grapharc/gateway/config.py b/grapharc/gateway/config.py index b74d6e2..bca5017 100644 --- a/grapharc/gateway/config.py +++ b/grapharc/gateway/config.py @@ -20,7 +20,7 @@ Secrets are returned, never logged. Anything that renders a config for humans goes through `redact`. -Four key-holding backends, plus one that usually holds none: +Five key-holding backends, plus one that usually holds none: - **OpenRouter** — `OPENROUTER_API_KEY`. - **OpenAI** — `OPENAI_API_KEY`, optionally with `OPENAI_BASE_URL` for an @@ -31,6 +31,8 @@ - **Novita** — `NOVITA_API_KEY`. The endpoint is fixed (`grapharc.gateway.novita.NOVITA_BASE_URL`), so unlike OpenAI there is no base-url override to resolve here. +- **UberLLM** — `UBERLLM_API_KEY`. The endpoint is fixed too + (`grapharc.gateway.uberllm.UBERLLM_BASE_URL`). - **Ollama** — no credential by default: it is a server on your own machine. What it needs is an address, so `ollama_base_url()` always returns one (`OLLAMA_HOST` / `OLLAMA_BASE_URL`, else localhost). `OLLAMA_API_KEY` exists @@ -66,6 +68,13 @@ "novita_api_key", ) +UBERLLM_KEYS = ( + "UBERLLM_API_KEY", + "UBERLLM_KEY", + "uberllm-api-key", + "uberllm_api_key", +) + # `OPENAI_API_BASE` is the older spelling and is still what a lot of tooling # sets; both are accepted, the current one first. OPENAI_BASE_URL_KEYS = ( @@ -147,6 +156,10 @@ def novita_api_key(*, env_file: Path | None = None) -> str | None: return get_secret(NOVITA_KEYS, env_file=env_file) +def uberllm_api_key(*, env_file: Path | None = None) -> str | None: + return get_secret(UBERLLM_KEYS, env_file=env_file) + + def openai_base_url(*, env_file: Path | None = None) -> str | None: """An endpoint override, or None for api.openai.com. diff --git a/grapharc/gateway/registry.py b/grapharc/gateway/registry.py index 3338669..18b814e 100644 --- a/grapharc/gateway/registry.py +++ b/grapharc/gateway/registry.py @@ -8,6 +8,7 @@ openrouter/openai/gpt-4o:floor -> OpenRouter, cheapest provider openai/gpt-4o-mini -> OpenAI directly (OPENAI_API_KEY) novita/moonshotai/kimi-k3 -> Novita (NOVITA_API_KEY) + uberllm/deepseek/deepseek-v4.1-flash -> UberLLM (UBERLLM_API_KEY) ollama/llama3.1 -> a local Ollama server, no key mock/anything -> scripted test double @@ -29,7 +30,7 @@ class UnknownBackendError(Exception): """The spec named a backend that is not registered.""" -BACKENDS = ("claude-cli", "openrouter", "openai", "novita", "ollama", "mock") +BACKENDS = ("claude-cli", "openrouter", "openai", "novita", "uberllm", "ollama", "mock") # Authors that appear in OpenRouter model ids. A spec starting with one of # these is a model name, not a mistyped backend — `anthropic/claude-haiku-4.5` @@ -73,6 +74,7 @@ class UnknownBackendError(Exception): "openrouter": "openrouter/anthropic/claude-sonnet-4.5", "openai": "openai/gpt-4o-mini", "novita": "novita/moonshotai/kimi-k3", + "uberllm": "uberllm/deepseek/deepseek-v4.1-flash", "ollama": "ollama/llama3.1", } @@ -173,6 +175,17 @@ def get_model(spec: str, **kwargs: Any) -> BaseChatModel: return NovitaChatModel(model, **kwargs) + if backend == "uberllm": + try: + from grapharc.gateway.uberllm import UberLLMChatModel + except ImportError as exc: # pragma: no cover - depends on install extras + raise UnknownBackendError( + "The UberLLM backend needs langchain-openai. " + "Install it with: uv sync --extra uberllm" + ) from exc + + return UberLLMChatModel(model, **kwargs) + if backend == "ollama": try: from grapharc.gateway.ollama import OllamaChatModel diff --git a/grapharc/gateway/uberllm.py b/grapharc/gateway/uberllm.py new file mode 100644 index 0000000..26c8ce0 --- /dev/null +++ b/grapharc/gateway/uberllm.py @@ -0,0 +1,62 @@ +"""UberLLM backend — one key for models from many vendors, priced per call. + +`uberllm/deepseek/deepseek-v4.1-flash` reaches UberLLM's own OpenAI-compatible +endpoint (`https://api.uberllm.dev/v1`), not api.openai.com, so this builds on +`OpenAICompatChatModel` the same way `openrouter.py` and `novita.py` do rather +than on `openai.py`: the endpoint is fixed, not an override of OpenAI's own. + +Model ids on UberLLM are themselves `vendor/model` — `deepseek/deepseek-v4.1-flash`, +`anthropic/claude-sonnet-4.6` — the same shape OpenRouter uses, so `vendor()` in +`registry.py` already reads the right author off an UberLLM spec with no +backend-specific handling: `BACKEND_VENDOR` stays absent for `uberllm`. + +**UberLLM prices its own calls.** The chat-completions usage block carries +`cost_usd`, what the call was billed, so like OpenRouter a `cost_ceiling_usd` +on this backend is an enforcement rather than a counter — except on streamed +turns, where the usage block arrives in the final SSE chunk and does not survive +chunk conversion. +""" + +from __future__ import annotations + +from typing import Any + +from grapharc.gateway.config import uberllm_api_key +from grapharc.gateway.openai_compat import OpenAICompatChatModel + +UBERLLM_BASE_URL = "https://api.uberllm.dev/v1" + + +class UberLLMError(Exception): + """The UberLLM backend could not be constructed or used.""" + + +class UberLLMChatModel(OpenAICompatChatModel): + """A LangChain chat model over UberLLM's OpenAI-compatible endpoint.""" + + def __init__(self, model: str, /, **kwargs: Any) -> None: + api_key = kwargs.pop("api_key", None) or uberllm_api_key() + if not api_key: + raise UberLLMError( + "No UberLLM API key found. Set UBERLLM_API_KEY in the environment, " + "or add one of UBERLLM_API_KEY / uberllm-api-key to a .env file." + ) + # One retry layer, not two — same reasoning as the OpenRouter backend. + kwargs.setdefault("max_retries", 0) + super().__init__( + model=model, + api_key=api_key, + base_url=kwargs.pop("base_url", None) or UBERLLM_BASE_URL, + **kwargs, + ) + + @property + def _llm_type(self) -> str: + return "grapharc-uberllm" + + def _provider_cost(self, usage: dict[str, Any]) -> float | None: + """UberLLM reports what the call was billed in its usage block.""" + return usage.get("cost_usd") + + +__all__ = ["UBERLLM_BASE_URL", "UberLLMChatModel", "UberLLMError"] diff --git a/pyproject.toml b/pyproject.toml index 9cfd5a7..234dee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,12 @@ openai = [ novita = [ "langchain-openai>=0.2", ] +# Imported by grapharc/gateway/uberllm.py. UberLLM speaks the OpenAI wire format +# against its own endpoint, so this needs the same client and no UberLLM-specific +# package. +uberllm = [ + "langchain-openai>=0.2", +] # Imported by grapharc/gateway/ollama.py. Ollama speaks the OpenAI wire format, # so the local backend needs the same client and no ollama-specific package. ollama = [ @@ -121,7 +127,7 @@ slack = [ ] # Everything above. Self-referential so it cannot drift out of sync. all = [ - "grapharc[api,ladybug,mcp,novita,ollama,openai,openrouter,otel,server,slack]", + "grapharc[api,ladybug,mcp,novita,ollama,openai,openrouter,otel,server,slack,uberllm]", ] [dependency-groups] diff --git a/tests/test_cli.py b/tests/test_cli.py index 4c3a2b4..2d2a15b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -446,6 +446,7 @@ def test_models_check_exits_one_when_nothing_is_configured(monkeypatch, capsys): "OPENROUTER_API_KEY", "OPENROUTER_KEY", "open-router-api-key", "OPENAI_API_KEY", "OPENAI_KEY", "openai-api-key", "NOVITA_API_KEY", "NOVITA_KEY", "novita-api-key", + "UBERLLM_API_KEY", "UBERLLM_KEY", "uberllm-api-key", "OLLAMA_HOST", "OLLAMA_BASE_URL", ): monkeypatch.delenv(name, raising=False) @@ -461,10 +462,11 @@ def test_models_check_exits_one_when_nothing_is_configured(monkeypatch, capsys): "openrouter": False, "openai": False, "novita": False, + "uberllm": False, "ollama": False, "mock": True, } - for backend in ("openrouter", "openai", "novita"): + for backend in ("openrouter", "openai", "novita", "uberllm"): assert next(b for b in payload["backends"] if b["backend"] == backend)[ "credential" ] == "" diff --git a/tests/test_cookbook_models.py b/tests/test_cookbook_models.py index 5ed5551..d7f99ae 100644 --- a/tests/test_cookbook_models.py +++ b/tests/test_cookbook_models.py @@ -47,6 +47,7 @@ OPENAI_BASE_URL_KEYS, OPENAI_KEYS, OPENROUTER_KEYS, + UBERLLM_KEYS, ) from grapharc.gateway.registry import BACKENDS, DEFAULT_BACKEND from grapharc.gateway.resilience import RETRYABLE_STATUS @@ -78,6 +79,7 @@ *OPENAI_KEYS, *OPENAI_BASE_URL_KEYS, *NOVITA_KEYS, + *UBERLLM_KEYS, *OLLAMA_KEYS, *OLLAMA_BASE_URL_KEYS, ) @@ -198,6 +200,7 @@ def _needs_openrouter(body: str) -> bool: "langchain_openai", '"openai/', '"novita/', + '"uberllm/', '"ollama/', ) ) @@ -292,9 +295,11 @@ def test_cli_command_with_machine_dependent_output_runs(position, tmp_path): def test_the_backend_list_the_page_prints_is_the_real_one(): page = DOC.read_text(encoding="utf-8") - assert BACKENDS == ("claude-cli", "openrouter", "openai", "novita", "ollama", "mock") + assert BACKENDS == ( + "claude-cli", "openrouter", "openai", "novita", "uberllm", "ollama", "mock" + ) assert DEFAULT_BACKEND == "claude-cli" - assert "There are six backends:" in page + assert "There are seven backends:" in page # Every backend has a row in the page's table. A backend added to the # gateway and not to the page fails here rather than going undocumented. for backend in BACKENDS: diff --git a/tests/test_gateway_openai_ollama.py b/tests/test_gateway_openai_ollama.py index 82fd5fb..7d3fc4b 100644 --- a/tests/test_gateway_openai_ollama.py +++ b/tests/test_gateway_openai_ollama.py @@ -175,7 +175,7 @@ def test_a_missing_openai_key_surfaces_through_the_registry(no_credentials): def test_an_unknown_backend_still_names_every_real_one(): - with pytest.raises(UnknownBackendError, match="openai, novita, ollama"): + with pytest.raises(UnknownBackendError, match="openai, novita, uberllm, ollama"): get_model("opnai/gpt-4o-mini") diff --git a/tests/test_gateway_openrouter.py b/tests/test_gateway_openrouter.py index 5d4a54c..5f6defe 100644 --- a/tests/test_gateway_openrouter.py +++ b/tests/test_gateway_openrouter.py @@ -149,7 +149,7 @@ def test_a_typoed_backend_is_caught_early_not_folded_into_a_model_name(): """`opnerouter/...` must fail here, not become a Claude-CLI call with a nonsense model that errors confusingly much later.""" with pytest.raises( - UnknownBackendError, match="claude-cli, openrouter, openai, novita, ollama, mock" + UnknownBackendError, match="claude-cli, openrouter, openai, novita, uberllm, ollama, mock" ): split_spec("opnerouter/anthropic/claude-haiku-4.5") with pytest.raises(UnknownBackendError): diff --git a/tests/test_gateway_uberllm.py b/tests/test_gateway_uberllm.py new file mode 100644 index 0000000..1196710 --- /dev/null +++ b/tests/test_gateway_uberllm.py @@ -0,0 +1,201 @@ +"""The UberLLM backend. + +Everything here is offline except the test marked `live`: constructing a +`ChatOpenAI` opens no socket, so the wiring — credentials, endpoint, cost +accounting — is checkable without spending anything. +""" + +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage +from langchain_core.outputs import ChatGeneration, ChatResult + +from grapharc.gateway import config, describe, get_model, split_spec, vendor +from grapharc.gateway.registry import UnknownBackendError + +pytest.importorskip("langchain_openai", reason="UberLLM needs the uberllm extra") + +from grapharc.gateway.uberllm import ( # noqa: E402 + UBERLLM_BASE_URL, + UberLLMChatModel, + UberLLMError, +) + + +@pytest.fixture +def no_credentials(monkeypatch, tmp_path): + """No key in the environment, and a working directory holding no .env.""" + for name in config.UBERLLM_KEYS: + monkeypatch.delenv(name, raising=False) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _result(**token_usage) -> ChatResult: + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="x"))], + llm_output={"model_name": "test-model", "token_usage": token_usage}, + ) + + +# ---------------------------------------------------------------- credentials + + +def test_uberllm_key_is_read_from_a_dotenv_file(tmp_path, monkeypatch): + """`uberllm-api-key` cannot be a shell variable, so the file is parsed.""" + for name in config.UBERLLM_KEYS: + monkeypatch.delenv(name, raising=False) + env = tmp_path / ".env" + env.write_text('uberllm-api-key="ull-fromfile"\n', encoding="utf-8") + assert config.uberllm_api_key(env_file=env) == "ull-fromfile" + + +def test_uberllm_process_env_beats_the_file(tmp_path, monkeypatch): + env = tmp_path / ".env" + env.write_text("uberllm-api-key=from-file\n", encoding="utf-8") + monkeypatch.setenv("UBERLLM_API_KEY", "from-env") + assert config.uberllm_api_key(env_file=env) == "from-env" + + +def test_constructing_uberllm_without_a_key_explains_how_to_fix_it(no_credentials): + with pytest.raises(UberLLMError, match="UBERLLM_API_KEY"): + UberLLMChatModel("deepseek/deepseek-v4.1-flash") + + +def test_uberllm_key_never_appears_in_a_description_or_a_redaction(monkeypatch): + secret = "ull_0123456789abcdef0123456789abcdef0123456" + monkeypatch.setenv("UBERLLM_API_KEY", secret) + assert secret not in str(describe("uberllm/deepseek/deepseek-v4.1-flash")) + assert secret not in config.redact(secret) + + +def test_uberllm_always_points_at_its_own_endpoint(monkeypatch): + """Unlike OpenAI, there is no override — the endpoint is fixed.""" + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel("deepseek/deepseek-v4.1-flash") + assert str(model.openai_api_base) == UBERLLM_BASE_URL + assert UBERLLM_BASE_URL == "https://api.uberllm.dev/v1" + + +# ------------------------------------------------------------------- registry + + +@pytest.mark.parametrize( + ("spec", "backend", "model"), + [ + ("uberllm/deepseek/deepseek-v4.1-flash", "uberllm", "deepseek/deepseek-v4.1-flash"), + ("uberllm/anthropic/claude-sonnet-4.6", "uberllm", "anthropic/claude-sonnet-4.6"), + ], +) +def test_uberllm_specs_split_the_way_the_docs_say(spec, backend, model): + assert split_spec(spec) == (backend, model) + assert describe(spec) == {"spec": spec, "backend": backend, "model": model} + + +def test_the_registry_builds_the_uberllm_backend(monkeypatch): + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + assert get_model("uberllm/deepseek/deepseek-v4.1-flash")._llm_type == "grapharc-uberllm" + + +def test_a_missing_uberllm_key_surfaces_through_the_registry(no_credentials): + with pytest.raises(UberLLMError): + get_model("uberllm/deepseek/deepseek-v4.1-flash") + + +def test_an_unknown_backend_still_names_uberllm(): + with pytest.raises(UnknownBackendError, match="uberllm"): + get_model("uberlm/deepseek/deepseek-v4.1-flash") + + +def test_vendor_reads_the_model_authors_uberllm_ids_carry(): + """UberLLM ids are themselves `vendor/model`, the same shape OpenRouter uses, + so `vendor()` needs no UberLLM-specific entry in `BACKEND_VENDOR`.""" + assert vendor("uberllm/deepseek/deepseek-v4.1-flash") == "deepseek" + assert vendor("uberllm/anthropic/claude-sonnet-4.6") == "anthropic" + + +# ------------------------------------------------- capabilities and accounting + + +def test_uberllm_can_bind_tools_and_structure_output(monkeypatch): + """The capability the Claude-CLI backend cannot offer.""" + from langchain_core.tools import tool + from pydantic import BaseModel + + @tool + def get_weather(city: str) -> str: + """Get the current weather for a city.""" + return f"sunny in {city}" + + class Verdict(BaseModel): + supported: bool + + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel("deepseek/deepseek-v4.1-flash") + model.bind_tools([get_weather]) + model.with_structured_output(Verdict) + + +def test_the_usage_envelope_matches_every_other_backend(monkeypatch): + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel("deepseek/deepseek-v4.1-flash") + model._record_usage( + _result( + prompt_tokens=1000, + completion_tokens=50, + prompt_tokens_details={"cached_tokens": 800}, + ) + ) + usage = model.last_usage + assert usage["input_tokens"] == 1000 # cached input still counts + assert usage["total_tokens"] == 1050 + assert usage["input_token_details"]["cache_read"] == 800 + assert usage["uncached_input_tokens"] == 200 + + +def test_uberllm_charges_the_cost_it_reports(monkeypatch): + """UberLLM puts what the call was billed in `usage.cost_usd`, so the meter + is charged that figure rather than counting the call as unpriced.""" + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel("deepseek/deepseek-v4.1-flash") + model._settle(_result(prompt_tokens=35, completion_tokens=10, cost_usd=0.00042)) + assert model.last_usage["cost_usd"] == 0.00042 + assert model.spend.spent_usd == pytest.approx(0.00042) + assert model.spend.unpriced_calls == 0 + + +def test_the_reported_cost_wins_over_a_rate_card(monkeypatch): + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel( + "deepseek/deepseek-v4.1-flash", + price_per_million={"input": 100.0, "output": 100.0}, + ) + model._settle(_result(prompt_tokens=1000, completion_tokens=50, cost_usd=0.00042)) + assert model.last_usage["cost_usd"] == 0.00042 + + +def test_without_a_reported_cost_the_call_is_counted_as_unpriced(monkeypatch): + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + model = UberLLMChatModel("deepseek/deepseek-v4.1-flash") + model._settle(_result(prompt_tokens=1000, completion_tokens=50)) + assert model.last_usage["cost_usd"] is None + assert model.spend.unpriced_calls == 1 + + +def test_uberllm_sets_no_max_tokens_ceiling_of_its_own(monkeypatch): + monkeypatch.setenv("UBERLLM_API_KEY", "ull-test") + assert UberLLMChatModel("deepseek/deepseek-v4.1-flash").max_tokens is None + + +# ------------------------------------------------------------------ live ---- + + +@pytest.mark.live +@pytest.mark.skipif(not config.uberllm_api_key(), reason="no UberLLM API key configured") +def test_live_uberllm_completion_and_cost_accounting(): + model = get_model("uberllm/deepseek/deepseek-v4.1-flash", temperature=0, max_tokens=512) + reply = model.invoke("Reply with exactly: ok") + assert reply.content + assert model.last_usage["cost_usd"] is not None + assert model.spend.unpriced_calls == 0 diff --git a/uv.lock b/uv.lock index bf61f30..270b8b5 100644 --- a/uv.lock +++ b/uv.lock @@ -405,6 +405,9 @@ server = [ slack = [ { name = "slack-bolt" }, ] +uberllm = [ + { name = "langchain-openai" }, +] [package.dev-dependencies] dev = [ @@ -420,12 +423,13 @@ dev = [ requires-dist = [ { name = "anthropic", marker = "extra == 'api'", specifier = ">=0.40" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115" }, - { name = "grapharc", extras = ["api", "ladybug", "mcp", "novita", "ollama", "openai", "openrouter", "otel", "server", "slack"], marker = "extra == 'all'" }, + { name = "grapharc", extras = ["api", "ladybug", "mcp", "novita", "ollama", "openai", "openrouter", "otel", "server", "slack", "uberllm"], marker = "extra == 'all'" }, { name = "langchain-core", specifier = ">=0.3" }, { name = "langchain-openai", marker = "extra == 'novita'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ollama'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'openai'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'openrouter'", specifier = ">=0.2" }, + { name = "langchain-openai", marker = "extra == 'uberllm'", specifier = ">=0.2" }, { name = "langgraph", specifier = ">=0.4" }, { name = "langgraph-checkpoint-sqlite", specifier = ">=2.0" }, { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.2,<2" }, @@ -437,7 +441,7 @@ requires-dist = [ { name = "slack-bolt", marker = "extra == 'slack'", specifier = ">=1.20" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.32" }, ] -provides-extras = ["openrouter", "openai", "novita", "ollama", "server", "mcp", "ladybug", "otel", "api", "slack", "all"] +provides-extras = ["openrouter", "openai", "novita", "uberllm", "ollama", "server", "mcp", "ladybug", "otel", "api", "slack", "all"] [package.metadata.requires-dev] dev = [