{ "generated_at": "2026-09-26T04:21:13.726Z", "catalogue_last_changed": "2026-09-26T04:17:00.525715+00:00", "source": "supabase", "provider_count": 31, "license": "Catalog values are facts published by each vendor. Reuse freely; please link back and re-verify before relying on any figure.", "scope": "Curated catalogue, not the whole market. generated_at and catalogue_last_changed are not verification dates. Each field has its own verified_at entry. Save this file with the edition you cite.", "providers": [ { "slug": "eden-ai", "name": "Eden AI", "company": "Eden AI", "tagline": "European multi-modality AI gateway: LLM chat behind an OpenAI-compatible endpoint plus OCR, speech, translation and vision APIs, billed at provider cost with a 5.5% fee on credit purchases.", "category": "managed_marketplace", "deployment": "managed_only", "license": "Proprietary", "founded_year": null, "hq_country": "France", "url_home": "https://www.edenai.co/", "url_docs": "https://www.edenai.co/docs/v3/overview/ai-gateway", "url_pricing": "https://www.edenai.co/pricing", "url_status": "https://app-edenai.instatus.com/", "url_github": "https://github.com/edenai", "url_changelog": "https://changelog.edenai.co/", "url_models_api": "https://api.edenai.run/v3/models", "github_stars": null, "model_count": 1104, "model_count_note": "Count of entries returned by the models API on 2026-09-25. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 50, "token_markup_pct": 0, "credit_fee_pct": 5.5, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No BYOK surcharge is published: the BYOK page states provider billing goes direct and that with BYOK the returned `cost` becomes an estimate rather than a charge; the only fee named anywhere on the pricing pages is the 5.5% credit-purchase fee ([BYOK](https://www.edenai.co/docs/v3/general/byok), [Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices)).", "seat_fee_usd_month": 0, "free_tier": "No free inference allowance is documented. What is free is the sandbox: sandbox tokens \"let you test your integration without real provider calls and at no cost\", returning mock data ([Sandbox](https://www.edenai.co/docs/v3/general/sandbox)). Real calls need credits - a depleted balance returns `402 Payment Required` ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits)). Seats are free: the AI API Gateway plan lists \"Unlimited seats\" ([Eden AI pricing](https://www.edenai.co/pricing)).", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Eden AI platform fee on credit purchases", "amount": "5.5%, applied at checkout when buying credits" }, { "label": "Advanced AI Platform plan", "amount": "Custom price (quote): high rate limits, bulk discounts, private deployments, dedicated support & SLA, professional services" } ], "self_host_cost_note": "n.a. - no self-hosted distribution is offered, so there is no self-host cost model. The Advanced plan mentions \"private deployments\" for specific compliance needs without describing customer-run infrastructure, pricing or artifacts ([Eden AI pricing](https://www.edenai.co/pricing)), and no Docker image, Helm chart, binary or package appears anywhere in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": false, "portability_note": "Programmatic export exists for usage and cost data (`/v2/cost_management/`) and uploaded files can be listed and deleted, but there is no documented bulk export of logs, prompts or configuration, and no import/export format for routing settings ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [File upload](https://www.edenai.co/docs/v3/llms/file-upload), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Responses", "OpenAI Embeddings", "OpenAI Moderations", "Anthropic Messages", "Eden AI Universal AI (expert models, sync + async)", "Eden AI cost-monitoring and token-management API" ], "sdk_note": "No first-party SDK is documented - integration is a base-URL swap on the official OpenAI SDKs (`base_url=\"https://api.edenai.run/v3\"`) for Python and TypeScript/JavaScript, or raw HTTP ([OpenAI Python SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-python), [OpenAI TS/JS SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-typescript), [First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call)).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": false, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": null, "cap_virtual_keys": true, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No latency figure of any kind is published: no gateway-overhead number, no percentile, no time-to-first-token claim on the home, pricing, docs-overview, routing, streaming or status pages ([Eden AI home](https://www.edenai.co/), [Eden AI pricing](https://www.edenai.co/pricing), [AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), [Streaming](https://www.edenai.co/docs/v3/llms/streaming), [Eden AI status](https://app-edenai.instatus.com/)). Latency appears only as a routing preference (`routing.sort: latency` or `speed`, and a `model:latency` suffix) with no measurements attached ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)). Adjacent availability claims exist but are not latency: \"99.99% uptime\" on the home page, and the status page displaying \"All systems operational\" with \"100% - uptime\" and a \"99.98% uptime\" figure on 2026-09-02 ([Eden AI home](https://www.edenai.co/), [Eden AI status](https://app-edenai.instatus.com/)). Independent coverage notes the uptime and developer-count claims are unaudited ([Founderland analysis, 2026-04-27](https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy)). Nothing here was estimated or benchmarked.", "throughput_rps": null, "soc2": true, "hipaa_baa": null, "gdpr": true, "eu_residency": true, "eu_residency_note": "True and self-service: a dedicated EU gateway host provides EU data residency, restricted to EU-compatible providers, alongside EU-hosted infrastructure ([EU AI gateway](https://www.edenai.co/eu), [Servers location](https://www.edenai.co/docs/v3/data-governance/servers-location)). Scope limit: the DPA states customer data may still be transferred outside the EEA depending on selected providers and features, with standard contractual clauses applied ([Data Processing Agreement](https://www.edenai.co/dpa)); independent coverage flagged the same gap ([Founderland analysis, 2026-04-27](https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy)).", "zdr": true, "zdr_note": "Zero data retention is the documented default rather than an on-request enterprise mode: the home page badges \"ZERO DATA RETENTION\", the security page states prompts and outputs are not retained, and the docs confirm content is not stored unless you opt in ([Eden AI home](https://www.edenai.co/), [Security](https://www.edenai.co/security), [Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)). Caveat worth stating: ZDR covers Eden AI's own storage; what each upstream provider retains is governed by that provider ([Data Processing Agreement](https://www.edenai.co/dpa)).", "air_gapped": null, "best_for": "Teams that need OCR, speech, translation or document parsing alongside LLM calls under one contract, one prepaid balance and EU data residency, and who are happy to build routing policy per request.", "strengths": [ "No token markup: provider list prices pass through, and 115 of 1,010 catalog endpoints are priced below list via a `discount` field", "Multi-modality in one API: 36 expert-model subfeatures (OCR, document parsing, STT/TTS, translation, vision, video generation) alongside 488 routable LLM names", "Public, unauthenticated model catalog with per-endpoint pricing, capabilities and region tags", "EU-first data path: EU-hosted infrastructure, a dedicated EU endpoint that refuses out-of-EU routing, and per-request region pinning", "Zero data retention by default, no training on customer data, with content logging as an opt-in per-project toggle", "Per-key budgets, expiry and free sandbox tokens make cost and access control concrete", "Ordered per-request fallbacks and cost/latency routing preferences, with failed attempts costing nothing" ], "weaknesses": [ "Not LLM-routing-first: half the product is non-LLM expert models on a separate, non-OpenAI-compatible endpoint that does not stream", "5.5% fee is charged on credit purchases at checkout, so small top-ups pay the fee before a single token is spent", "Guardrails are asserted on the marketing security page but have no documented feature, parameter or example anywhere in the docs", "No timeout, retry or circuit-breaker controls at all; reliability tuning is limited to a fallback list capped at 3 entries", "Rate limits contradict across vendor pages (7, 10 and 15 req/s), and model counts range from 300+ to 500+ depending on the page", "No self-hosted or hybrid deployment, no Terraform or Kubernetes path, and no config-as-code surface", "Projects, RBAC and per-environment spend limits are Advanced-plan only, on a custom quote with no published price", "SOC 2 and ISO 27001 are asserted without a report, portal or audit scope; the subprocessor list is available only on request", "No SLA figure, latency claim or throughput claim is published; uptime and developer-count claims are unaudited" ], "avoid_if": "You want an LLM-routing-first gateway with configurable timeouts, retries, enforced guardrails, per-key rate limits or a self-hosted data plane - none of those are documented, and account-level features like projects and RBAC sit behind the quote-only Advanced plan.", "reviews": [ { "url": "https://techpoint.africa/guide/eden-ai-review/", "why": "Hands-on sign-up-to-first-call walkthrough of the dashboard and unified API across text, speech, image and document features, from a reviewer who came to it as a working developer rather than an evaluator of gateways.", "date": "2025-06-10", "type": "written", "title": "Eden AI review: full-stack AI platform tested & rated", "publisher": "Techpoint Africa" }, { "url": "https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy", "why": "The most sceptical independent read available: it checks the EU-residency positioning against Eden AI's own DPA (transfers outside the EEA are permitted under SCCs) and flags the 99.99% uptime and 200k-developer claims as unaudited.", "date": "2026-04-27", "type": "written", "title": "Eden AI Launches European AI Gateway to Challenge OpenRouter", "publisher": "Founderland" }, { "url": "https://www.firstaimovers.com/p/edenai-vs-openrouter-ai-api-comparison", "why": "Practitioner comparison drawn from hands-on use and an interview with Eden AI's CEO; useful because it lands on the same scope split this entry records - Eden AI for multi-modal workflows, OpenRouter when the stack is LLM-only - and warns that the wide provider catalog can overwhelm small teams.", "date": "2025-11-06", "type": "written", "title": "EdenAI vs OpenRouter: Which AI Aggregator Fits Your Stack?", "publisher": "First AI Movers" }, { "url": "https://qveris.ai/guides/openrouter-vs-eden-ai/", "why": "Implementation-check guide that makes the structural cost of a cross-modality universal API explicit: OCR pages, speech timestamps, image dimensions, moderation labels, video jobs and LLM streams cannot share one response schema, so cross-modal fallback is usually unsafe.", "date": "2026-07-21", "type": "written", "title": "OpenRouter vs Eden AI: LLM or Multimodal API?", "publisher": "QVeris" } ], "sources": { "zdr": "https://www.edenai.co/security", "gdpr": "https://www.edenai.co/dpa", "soc2": "https://www.edenai.co/security", "license": "https://api.github.com/orgs/edenai/repos", "reviews": "https://techpoint.africa/guide/eden-ai-review/", "sdk_note": "https://www.edenai.co/docs/v3/integrations/openai-sdk-python", "url_docs": "https://www.edenai.co/docs/v3/overview/ai-gateway", "url_home": "https://www.edenai.co/", "zdr_note": "https://www.edenai.co/security", "free_tier": "https://www.edenai.co/docs/v3/general/sandbox", "deployment": "https://www.edenai.co/docs/v3/overview/ai-gateway", "extra_fees": "https://www.edenai.co/pricing", "hq_country": "https://www.edenai.co/about", "url_github": "https://api.github.com/orgs/edenai", "url_status": "https://app-edenai.instatus.com/", "cap_budgets": "https://www.edenai.co/docs/v3/general/custom-api-keys", "cost_export": "https://www.edenai.co/docs/v3/general/monitoring", "eden_ai_dpa": "https://www.edenai.co/dpa", "model_count": "https://api.edenai.run/v3/models", "url_pricing": "https://www.edenai.co/pricing", "api_surfaces": "https://www.edenai.co/docs/v3/overview/ai-gateway", "cap_failover": "https://www.edenai.co/docs/v3/general/fallback", "eu_residency": "https://www.edenai.co/eu", "product_fate": "https://www.edenai.co/about", "byok_fee_note": "https://www.edenai.co/docs/v3/general/byok", "cache_support": "https://www.edenai.co/docs/v3/llms/prompt-caching", "cost_controls": "https://www.edenai.co/docs/v3/general/custom-api-keys", "eden_ai_terms": "https://www.edenai.co/terms", "pricing_model": "https://www.edenai.co/docs/v3/overview/plans-prices", "url_changelog": "https://changelog.edenai.co/", "vendor_status": "https://www.edenai.co/about", "byok_supported": "https://www.edenai.co/docs/v3/general/byok", "cap_guardrails": "https://www.edenai.co/security", "certifications": "https://www.edenai.co/security", "credit_fee_pct": "https://www.edenai.co/pricing", "provider_count": "https://www.edenai.co/docs/v3/overview/ai-gateway", "retention_days": "https://www.edenai.co/docs/v3/data-governance/data-retention", "url_models_api": "https://api.edenai.run/v3/models", "guardrails_note": "https://www.edenai.co/security", "logs_by_default": "https://www.edenai.co/docs/v3/data-governance/data-retention", "model_count_low": "https://api.edenai.run/v3/models", "cap_cache_simple": "https://www.edenai.co/docs/v3/general/caching", "cap_virtual_keys": "https://www.edenai.co/docs/v3/general/custom-api-keys", "eden_ai_info_api": "https://api.edenai.run/v3/info", "model_count_high": "https://api.edenai.run/v3/models", "model_count_note": "https://api.edenai.run/v3/models", "token_markup_pct": "https://www.edenai.co/docs/v3/overview/plans-prices", "cap_observability": "https://www.edenai.co/docs/v3/general/monitoring", "openai_compatible": "https://www.edenai.co/docs/v3/llms/chat-completions", "rel_fallback_note": "https://www.edenai.co/docs/v3/general/fallback", "streaming_support": "https://www.edenai.co/docs/v3/overview/llms-vs-expert-models", "architecture_class": "https://www.edenai.co/docs/api-reference/anthropic-messages/create-anthropic-message", "cap_cache_semantic": "https://www.edenai.co/docs/v3/general/caching", "cap_load_balancing": "https://www.edenai.co/docs/v3/llms/provider-routing", "eden_ai_seed_round": "https://www.edenai.co/post/eden-ai-raises-eu3m-to-bridge-the-gap-between-ai-models-and-business-needs", "fit_quickstart_url": "https://www.edenai.co/docs/v3/quickstart/first-llm-call", "merchant_of_record": "https://www.edenai.co/docs/v3/general/buying-credits", "pricing_model_note": "https://www.edenai.co/docs/v3/overview/plans-prices", "guardrails_location": "https://www.edenai.co/security", "latency_source_note": "https://app-edenai.instatus.com/", "rel_fallback_config": "https://www.edenai.co/docs/v3/general/fallback", "self_host_cost_note": "https://www.edenai.co/docs/llms.txt", "self_host_data_plane": "https://www.edenai.co/docs/llms.txt", "prompt_transits_vendor": "https://www.edenai.co/docs/v3/overview/ai-gateway", "cap_conditional_routing": "https://www.edenai.co/docs/v3/llms/provider-routing", "independent_founderland": "https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy", "model_count_source_note": "https://api.edenai.run/v3/models", "provider_count_source_note": "https://api.edenai.run/v3/info" }, "verified_at": { "zdr": "2026-09-02", "gdpr": "2026-09-02", "name": "2026-09-02", "soc2": "2026-09-02", "company": "2026-09-02", "license": "2026-09-02", "category": "2026-09-02", "url_docs": "2026-09-02", "url_home": "2026-09-02", "byok_mode": "2026-09-02", "free_tier": "2026-09-02", "published": "2026-09-02", "deployment": "2026-09-02", "extra_fees": "2026-09-02", "hq_country": "2026-09-02", "url_github": "2026-09-02", "url_status": "2026-09-02", "zdr_status": "2026-09-02", "cap_budgets": "2026-09-02", "model_count": "2026-09-25", "url_pricing": "2026-09-02", "api_surfaces": "2026-09-02", "cap_failover": "2026-09-02", "eu_residency": "2026-09-02", "product_fate": "2026-09-02", "cache_support": "2026-09-02", "pricing_model": "2026-09-02", "vendor_status": "2026-09-02", "byok_supported": "2026-09-02", "certifications": "2026-09-02", "credit_fee_pct": "2026-09-02", "provider_count": "2026-09-02", "retention_days": "2026-09-02", "logs_by_default": "2026-09-02", "model_count_low": "2026-09-25", "cap_cache_simple": "2026-09-02", "cap_virtual_keys": "2026-09-02", "model_count_high": "2026-09-25", "model_count_note": "2026-09-25", "token_markup_pct": "2026-09-02", "cap_observability": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "architecture_class": "2026-09-02", "cap_cache_semantic": "2026-09-02", "cap_load_balancing": "2026-09-02", "merchant_of_record": "2026-09-02", "guardrails_location": "2026-09-02", "requires_vendor_sdk": "2026-09-02", "self_host_data_plane": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "cap_conditional_routing": "2026-09-02", "model_count_source_note": "2026-09-25", "trains_on_customer_data": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-25T16:55:57.814125+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed and independent. V3 is the current platform with the pre-2026 app \"supported until the end of 2026\", so legacy users have a dated migration ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), page dateModified 2026-05-07). The company is French, \"founded and headquartered in Lyon\", with \"25+\" employees and \"$4.5M raised\" ([About Eden AI](https://www.edenai.co/about)); the latest round is a EUR 3M seed led by Galion.exe with 50 Partners and angels including Datadog's Olivier Pomel and Docker's Sebastien Pahl ([Eden AI seed announcement](https://www.edenai.co/post/eden-ai-raises-eu3m-to-bridge-the-gap-between-ai-models-and-business-needs)), which independent coverage dates to late 2024 and reads as early-stage ([Founderland analysis, 2026-04-27](https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy)). Docs shipping is visible: the public `edenai/docs` repository was last pushed 2026-09-02 ([GitHub repos API](https://api.github.com/orgs/edenai/repos)). Two caveats for buyers - marketing claims of 99.99% uptime and 200k+ developers are unaudited ([Founderland analysis, 2026-04-27](https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy)), and the vendor's own pages disagree on rate limits and model counts ([Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices), [Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits)).", "last_release_date": null, "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "No dated release artifact could be confirmed. There is no versioned product changelog in the documentation index and the hosted changelog at changelog.edenai.co renders its entries client-side, so no entry date was extractable from either the cleaned text or the raw HTML on 2026-09-02; its visible content still references v2 endpoints such as `https://api.edenai.run/v2/llm/chat` ([Eden AI changelog](https://changelog.edenai.co/)). The freshest dated vendor artifacts observed are the docs-overview `dateModified` of 2026-05-07 ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)) and a 2026-09-02 push to the public docs repository ([GitHub repos API](https://api.github.com/orgs/edenai/repos)).", "prompt_transits_vendor": "yes", "prompt_path_note": "Always: Eden AI is hosted-only, so every request passes through `api.edenai.run` (or `api.eu.edenai.run`) before reaching the provider, including in BYOK mode where only the billing relationship moves ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), [BYOK](https://www.edenai.co/docs/v3/general/byok)). Eden AI's own infrastructure \"is hosted in secure European data centers\", while the provider that ultimately serves the request may be elsewhere - provider data-centre locations vary per provider and model and must be checked in the catalog ([Servers location](https://www.edenai.co/docs/v3/data-governance/servers-location)).", "logs_by_default": "metadata_only", "logs_note": "Content logging is off by default: \"By default, Eden AI does not store the content of your requests or responses\", and what is kept is metadata - timestamps, token counts and costs - for billing and operations ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)). The terms repeat it: \"prompts and outputs are not stored\", \"Customer Data is processed only as needed to route requests\" ([Terms of service](https://www.edenai.co/terms)).", "retention_days": 0, "retention_note": "Zero for request and response content by default - it is not stored at all ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)). Other clocks do exist: async job results are \"retained for 7 days\", and uploaded files default to 7 days with an `expires_at` timestamp after which \"expired files cannot be recovered\" ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention), [File upload](https://www.edenai.co/docs/v3/llms/file-upload)). Retention periods for the metadata Eden AI does keep, and for the opt-in Log Retention mode, are not published.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Explicitly no, in three places: \"Customer Data is not used to train models\" and \"Customer Data is not sold\" ([Terms of service](https://www.edenai.co/terms)), \"we do not use your data to train models\" ([Privacy policy](https://www.edenai.co/privacy)), and the same statement on the security page ([Security](https://www.edenai.co/security)).", "subprocessors_url": null, "regions_note": "EU-first with per-request granularity. Eden AI's infrastructure sits in European data centres ([Servers location](https://www.edenai.co/docs/v3/data-governance/servers-location)); a separate EU endpoint `https://api.eu.edenai.run/v3/` enforces EU residency, exposes only EU-compatible providers and errors rather than routing outside the EU ([EU AI gateway](https://www.edenai.co/eu)); and individual fallback entries can be pinned with `model@eu` or `model@us` ([Fallback](https://www.edenai.co/docs/v3/general/fallback)). Counted in the catalog on 2026-09-02: 275 endpoints tagged `eu`, 378 `us`, 323 `global`, 36 `ap`, 1 `ch` ([GET /v3/models](https://api.edenai.run/v3/models)). The DPA is the counterweight - transfers outside the EEA can occur depending on the providers and features chosen, covered by standard contractual clauses ([Data Processing Agreement](https://www.edenai.co/dpa)).", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "\"Eden AI is SOC 2 and ISO 27001 certified\" on the security page and \"SOC 2 & ISO 27001 certified\" on the home page; no type, audit period, report or trust portal is named", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "public DPA last updated May 2026: customer is controller, Eden AI is processor as a \"French company operating under European Union data protection laws\", with SCCs for outside-EEA transfers and breach notification without undue delay", "status": "published" }, "iso27001": { "detail": "asserted on the same two pages; no certificate number, scope statement or certification body named", "status": "claimed" }, "hipaa_baa": { "status": "not_published" }, "subprocessor_list": { "detail": "the DPA describes subprocessor categories (AI providers plus hosting, analytics, support and payment providers) and says the list is \"available upon request\"; no public subprocessors page exists", "status": "on_request" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Structurally vendor-side, because Eden AI runs only as hosted SaaS - anything it inspects, it inspects on its own infrastructure ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). But treat the capability itself as unproven: the only guardrail statement found is a marketing FAQ answer, \"Does Eden AI provide guardrails? Yes... including input protection, output moderation, policy-based routing, and fallback rules\" ([Security](https://www.edenai.co/security)), and there is no guardrails page, parameter or example anywhere in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "platform_fee", "pricing_model_note": "The 5.5% is a platform fee on credit purchases, not a per-token markup: \"Eden AI charges no markup on provider pricing. You pay exactly what the underlying provider charges, plus a 5.5% Eden AI platform fee applied at checkout\" ([Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices)) and the pricing page repeats \"We do not mark up provider pricing... The only additional cost is a 5.5% Eden AI platform fee, applied at checkout when purchasing credits\" ([Eden AI pricing](https://www.edenai.co/pricing)). So per-request `cost` values returned by the API are provider list prices; the fee is incurred once per credit top-up. Corroborated in the live catalog, where `pricing` equals `list_pricing` for 895 of 1,010 endpoints and is *below* list for the remaining 115 (a `discount` field, e.g. 0.35 on `qwen/qwen3.8-max-0902`), i.e. no endpoint is priced above provider list ([GET /v3/models](https://api.edenai.run/v3/models), 2026-09-02). No subscription: \"No subscription, No hidden costs, No API call limit\" ([Eden AI pricing](https://www.edenai.co/pricing)).", "min_commitment": "None stated for the self-serve plan: \"No subscription, No hidden costs, No API call limit\" ([Eden AI pricing](https://www.edenai.co/pricing)). Credits are prepaid; Advanced/Enterprise accounts can switch to postpaid monthly invoicing with 30-day terms ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits)). No credit-expiry rule appears in the terms, which do state \"Unless otherwise stated, payments are non-refundable\" ([Terms of service](https://www.edenai.co/terms)).", "overage_note": "No request-volume tiers or overage rates exist - billing is pure pay-per-use against prepaid credits, and each response carries a `cost` field in USD ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). Running out of credit is a hard stop (`402 Payment Required`), mitigated by auto-refill with a trigger threshold and a monthly refill cap, or by a monthly spend limit on postpaid accounts ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits)). The Advanced plan's \"bulk discounts\" and \"high rate limits\" are unpriced ([Eden AI pricing](https://www.edenai.co/pricing)).", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Two distinct mechanisms. (1) Response caching: exact-match on same model plus same input, \"enabled by default\", toggled per project in the dashboard, and a hit is \"returned at no additional cost\" - the vendor recommends it for embeddings, moderation, OCR and NER and against chat and image generation ([Caching](https://www.edenai.co/docs/v3/general/caching)). No semantic or similarity cache exists on any page or in the documentation index, which is why the semantic capability is recorded false rather than not-published ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). (2) Prompt caching: provider-side prefix reuse passed through, with `prompt_cache_key`, `prompt_cache_retention` (`in_memory`, `24h`) and preserved-but-untranslated `cache_control` breakpoints; Eden AI adds default ephemeral boundaries for cache-capable Anthropic and Bedrock Claude models ([Prompt caching](https://www.edenai.co/docs/v3/llms/prompt-caching)). Cached-token prices are per model, exposed as `cache_read_input_token_cost` and `cache_creation_input_token_cost` in the catalog, so no single discount percentage exists ([GET /v3/models](https://api.edenai.run/v3/models)).", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated. Auto-refill has a trigger threshold and a monthly cap, and postpaid accounts can set a monthly spend limit, but no alert or notification feature is documented ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits)).", "tier": null, "supported": null }, "virtual_keys": { "note": "Custom API keys created through `/v2/user/custom_token/`, each with type (`api_token` or `sandbox_api_token`), expiry and its own budget ([Custom API keys](https://www.edenai.co/docs/v3/general/custom-api-keys)).", "tier": null, "supported": true }, "rate_limiting": { "note": "Not a customer control: the account-level limit (10 req/s by default, shared across keys) is set by Eden AI and raised on request, with no user-configurable per-key limit documented ([Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits)).", "tier": null, "supported": null }, "spend_webhooks": { "note": "Not stated; the only webhooks documented deliver async job results ([Webhooks](https://www.edenai.co/docs/v3/expert-models/webhooks)).", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated as governance. `routing.allowed_providers` restricts which providers a single request may use, which is a routing preference rather than an account policy ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)).", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Per-token `balance` and `active_balance`: the key stops working when its budget reaches $0 ([Custom API keys](https://www.edenai.co/docs/v3/general/custom-api-keys)).", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Per-project separation with per-environment keys and spend limits is documented as an Advanced-plan feature only ([Users & Organisation](https://www.edenai.co/docs/v3/general/users-organisation)).", "tier": "advanced", "supported": true } }, "cost_attribution": "Per provider, per feature/subfeature, per API key and per workflow, via the cost-monitoring API: `GET https://api.edenai.run/v2/cost_management/` takes `begin`/`end` plus a `step` (daily to yearly) and filters on provider, subfeature, token, `workflow_id` and `rag_project_id`, returning total cost, call counts and cost per provider for keys such as `text__chat` and `ocr__ocr`; `/v2/cost_management/credits/` returns the balance ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring)). Every response also carries its own `cost` in USD ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). Per-user, per-team and per-customer splits are not stated.", "cost_export": "API only (`/v2/cost_management/` and `/v2/cost_management/credits/`), plus the dashboard's cost and performance monitoring ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [Eden AI pricing](https://www.edenai.co/pricing)). No CSV download, webhook, S3 or warehouse destination is documented on any fetched page or in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "product_kind_note": "Aggregator gateway, and deliberately broader than LLMs. The vendor calls it \"a unified AI gateway that gives you access to 500+ AI models from 50+ providers through a single API\" with \"fallback, routing, and cost tracking built in\" ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). The scope caveat matters for this catalogue: half the product is non-LLM. Alongside `/v3/chat/completions` sits `/v3/universal-ai` for \"expert models\" - OCR and document parsing (invoices, IDs, resumes, tables), speech-to-text and text-to-speech, translation, image analysis and generation, video generation, moderation and NER ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)). The live feature catalog returned 7 families and 36 subfeatures on 2026-09-02, including a `web` family (search, map, research) that is not even mentioned in the docs ([GET /v3/info](https://api.edenai.run/v3/info)). Read Eden AI as a multi-modality AI-API marketplace that also routes LLMs, not as an LLM-routing-first gateway.", "deployment_modes": [ "saas" ], "deployment_note": "Nothing to install: the vendor operates the gateway and its infrastructure \"is hosted in secure European data centers\" ([Servers location](https://www.edenai.co/docs/v3/data-governance/servers-location)). Legacy note for existing users - v3 is the current platform and pre-2026 accounts on `old-app.edenai.run` are \"supported until the end of 2026\", so migration is a dated obligation ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), page dateModified 2026-05-07).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "partial", "api_audio": "partial", "api_batch": "not_documented", "api_surface_note": "Base URL `https://api.edenai.run/v3` with `Authorization: Bearer`, split across two request shapes: `POST /v3/chat/completions` for LLMs (model string `provider/model`, OpenAI-compatible, SSE streaming) and `POST /v3/universal-ai` for expert models (model string `feature/subfeature/provider[/model]`, unified `status`/`cost`/`output` envelope, no streaming, not OpenAI-SDK compatible) ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), [LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)). Other surfaces: `POST /v3/responses` with GET/DELETE by id, `POST /v3/embeddings` plus an unauthenticated `GET /v3/embeddings/models`, `POST /v3/moderations`, `POST /v3/upload` for file storage, and `POST https://api.edenai.run/v3/v1/messages` documented as the \"Anthropic Messages API - native pass-through via litellm\" ([Responses](https://www.edenai.co/docs/v3/llms/responses), [Embeddings](https://www.edenai.co/docs/v3/llms/embeddings), [Create Moderation](https://www.edenai.co/docs/api-reference/moderations/create-moderation), [Create Anthropic Message](https://www.edenai.co/docs/api-reference/anthropic-messages/create-anthropic-message)). An EU-only host `https://api.eu.edenai.run/v3/` mirrors the same paths ([EU AI gateway](https://www.edenai.co/eu)). All observed 2026-09-02.", "model_count_low": 1104, "model_count_high": 1104, "model_count_source_note": "Count of entries returned by the models API on 2026-09-25. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": 33, "provider_count_high": 68, "provider_count_source_note": "Vendor figures disagree: \"50+ providers\" in the docs ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)) versus \"60+ AI providers\" on the company page ([About Eden AI](https://www.edenai.co/about)), both observed 2026-09-02. Counted from the live catalog the same day: 33 distinct `owned_by` LLM providers ([GET /v3/models](https://api.edenai.run/v3/models)) and 42 distinct expert-model providers ([GET /v3/info](https://api.edenai.run/v3/info)), a union of 68 once overlaps such as OpenAI, Google, Microsoft, Amazon, Mistral and MiniMax are removed.", "first_party_vs_partner": "All third-party: Eden AI hosts no models of its own and routes to provider APIs, returning the provider's `owned_by` and per-endpoint `regions` in its catalog. Counted 2026-09-02: 33 LLM serving providers (amazon, anthropic, azure, bytedance, cerebras, cloudflare, cohere, databricks, deepinfra, deepseek, fireworks_ai, flexai, google, groq, infomaniak, ionos, lilac, microsoft, minimax, mistral, moonshot, nebius, openai, ovhcloud, perplexityai, qwen, scaleway, tensorx, together_ai, vertex, xai, xiaomi, zai) ([GET /v3/models](https://api.edenai.run/v3/models)) and 42 expert-model providers including affinda, api4ai, assembly, deepgram, deepl, elevenlabs, klippa, mindee, sightengine, stabilityai and veryfi ([GET /v3/info](https://api.edenai.run/v3/info)).", "custom_endpoint_support": "No: no custom provider, custom base URL, self-hosted-model or vLLM/Ollama registration appears on any fetched page or in the complete documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). Model selection is limited to catalog entries or a provider-less model name handed to Eden AI's router ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)). Checked 2026-09-02.", "byok_mode": "both", "merchant_of_record": "Differs by mode. Credits mode: Eden AI invoices - you prepay credits by card, PayPal or bank transfer through Stripe and each request deducts its `cost` from the balance ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits), [Eden AI pricing](https://www.edenai.co/pricing)). BYOK mode: the upstream provider bills you directly for requests routed to your own key ([BYOK](https://www.edenai.co/docs/v3/general/byok)).", "key_management_note": "Two key types. Your Eden AI tokens: a main account key plus \"Custom API keys\" created via `/v2/user/custom_token/`, each with its own type (`api_token` or `sandbox_api_token`), expiry date and `balance`/`active_balance` budget that stops the token once exhausted ([Custom API keys](https://www.edenai.co/docs/v3/general/custom-api-keys)). Upstream provider keys: BYOK credentials are entered in the dashboard per provider and used for requests routed there ([BYOK](https://www.edenai.co/docs/v3/general/byok)). The DPA states encryption in transit and at rest ([Data Processing Agreement](https://www.edenai.co/dpa)); no KMS, secret-manager or key-rotation mechanics are documented.", "api_openai_chat_completions_note": "Yes: \"Eden AI V3 provides full OpenAI API compatibility with multi-provider support. The endpoint follows OpenAI's exact format, making it a drop-in replacement\" - `POST https://api.edenai.run/v3/chat/completions`, documented `temperature`, `top_p`, `max_tokens`, `frequency_penalty`, `presence_penalty`, `stream`, plus Eden-specific `fallbacks` and Claude `thinking` ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions), observed 2026-09-02).", "api_anthropic_messages_note": "Yes: a dedicated \"Anthropic Messages API - native pass-through via litellm\" is documented at `POST https://api.edenai.run/v3/v1/messages`, accepting Anthropic-shaped bodies plus Eden AI's `fallbacks`, `router_candidates` and `cache_control`, alongside a token-counting endpoint ([Create Anthropic Message](https://www.edenai.co/docs/api-reference/anthropic-messages/create-anthropic-message), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "api_openai_responses_note": "Yes: \"Eden AI supports the OpenAI Responses API - a stateful alternative to chat completions that stores conversation history server-side\", with `POST /v3/responses`, `GET`/`DELETE /v3/responses/{response_id}`, `store` defaulting to true and `previous_response_id` chaining; it is also the documented path for web search on regular OpenAI models ([Responses](https://www.edenai.co/docs/v3/llms/responses), [Web search](https://www.edenai.co/docs/v3/llms/web-search)).", "api_embeddings_note": "Yes: OpenAI-compatible `POST /v3/embeddings` accepting arrays of inputs, with a companion catalog endpoint `GET /v3/embeddings/models` that needs no authentication and returned 29 embedding models from 9 providers on 2026-09-02 ([Embeddings](https://www.edenai.co/docs/v3/llms/embeddings), live response from `https://api.edenai.run/v3/embeddings/models`).", "api_images_note": "partial: there is no OpenAI `/v1/images` surface. Two other paths exist - image-capable LLMs return a base64 data URL through `POST /v3/chat/completions` with an `image_config` block (documented for Google Gemini Flash Image models only), and dedicated image APIs run through Universal AI as `image/generation/...` ([Image generation](https://www.edenai.co/docs/v3/llms/image-generation), [Create Async Job](https://www.edenai.co/docs/api-reference/universal-ai/create-async-job), [LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)). The live catalog lists `image` subfeatures for generation, background removal, anonymization, object/face/logo detection, explicit-content, deepfake and AI-image detection ([GET /v3/info](https://api.edenai.run/v3/info)).", "api_audio_note": "partial: no OpenAI `/v1/audio` endpoint. Audio is an expert-model feature on Universal AI - text-to-speech (sync) and speech-to-text (async job with polling or webhook) - and the Universal AI endpoint explicitly does not stream ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models), [Webhooks](https://www.edenai.co/docs/v3/expert-models/webhooks), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The live feature catalog confirms the `audio` family with `tts` and `speech_to_text_async` only ([GET /v3/info](https://api.edenai.run/v3/info), 2026-09-02).", "api_batch_note": "not_documented as a batch surface. What exists is async single-job processing: `POST /v3/universal-ai/async` returns a `public_id` for polling or webhook delivery, one job per request ([Create Async Job](https://www.edenai.co/docs/api-reference/universal-ai/create-async-job), [Webhooks](https://www.edenai.co/docs/v3/expert-models/webhooks)). No OpenAI-style `/batches` endpoint or bulk-file submission appears on the fetched pages or in the full documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)), even though the live catalog exposes a per-model `supports_batch_inference` capability flag ([GET /v3/models](https://api.edenai.run/v3/models)).", "deployment_modes_note": "hosted SaaS only: `https://api.edenai.run/v3` with an EU-resident variant `https://api.eu.edenai.run/v3/` ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), [EU AI gateway](https://www.edenai.co/eu)). Self-host / hybrid VPC / on-prem / air-gapped: n.a. - not documented anywhere in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The Advanced plan lists \"Private deployments\" for \"specific compliance needs\" with no further detail, which is not a self-hosting statement ([Eden AI pricing](https://www.edenai.co/pricing)). Observed 2026-09-02.", "byok_mode_note": "both: you can spend Eden AI credits, or register your own provider keys in the dashboard so \"requests routed to that provider will use your key and be billed directly by the provider\", and the two can be mixed per provider ([BYOK](https://www.edenai.co/docs/v3/general/byok), observed 2026-09-02).", "access_meta": { "qualified": [ "api_images", "api_audio", "streaming_support" ], "model_count_kind": "enumerated", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "not_documented for inference: no request-timeout parameter, header or default appears on the chat-completions, fallback, provider-routing or streaming pages ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions), [Fallback](https://www.edenai.co/docs/v3/general/fallback), [Streaming](https://www.edenai.co/docs/v3/llms/streaming)). The only timeout published anywhere is on the webhook receiver side - 30 s per delivery attempt with up to 3 retries and exponential backoff capped at 30 s ([Webhooks](https://www.edenai.co/docs/v3/expert-models/webhooks)).", "rel_retry_config": "not_documented", "rel_retry_note": "not_documented: no retry count, backoff strategy or retry-on-status configuration appears on the fallback, provider-routing, chat-completions or rate-limit pages, and there is no retry page in the documentation index ([Fallback](https://www.edenai.co/docs/v3/general/fallback), [Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing), [Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The documented failure behaviour is fallback to the next candidate, not a retry counter. Default retry count: n.a. Backoff: n.a.", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED and per-request: a `fallbacks` array of up to 3 entries (a 4th returns `422`), tried in order, on both `/v3/chat/completions` and Universal AI; entries can pin regions (`model@eu`, `model@us`) and duplicates are not de-duplicated. A second layer is provider-level failover for provider-less model names, switched off with `routing.allow_fallbacks: false`. \"Failed attempts cost nothing\", and `x-edenai-metadata: enabled` returns per-attempt provider, status and region ([Fallback](https://www.edenai.co/docs/v3/general/fallback), [Create Async Job](https://www.edenai.co/docs/api-reference/universal-ai/create-async-job), observed 2026-09-02).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "not_configurable: health is tracked by Eden AI, not by you. The router \"ranks unhealthy providers last\" while still keeping them as a last resort, and there is no interval, threshold, ejection window or circuit-breaker setting on the routing, fallback or rate-limit pages ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing), [Fallback](https://www.edenai.co/docs/v3/general/fallback)). A public status page is published separately ([Eden AI status](https://app-edenai.instatus.com/)).", "rel_multi_region_config": null, "rel_multi_region_note": "Left empty because none of the existing enum values fits: region selection is real but it is per-request and per-endpoint, not a config file, and it is neither undocumented nor non-configurable. You can pin a region on each fallback entry (`model@eu`, `model@us`) ([Fallback](https://www.edenai.co/docs/v3/general/fallback)) or send the whole workload to the EU host `https://api.eu.edenai.run/v3/`, which exposes only EU-compatible providers and returns an error rather than routing outside the EU ([EU AI gateway](https://www.edenai.co/eu)). 275 of 1,010 catalog endpoints carry an `eu` region tag, 378 `us`, 323 `global`, 36 `ap`, 1 `ch` ([GET /v3/models](https://api.edenai.run/v3/models), 2026-09-02).", "rel_load_balancing_config": "per_request", "rel_load_balancing_note": "Weights are NOT user-settable. For a provider-less model name Eden AI spreads traffic across healthy endpoints with internal weighting and optimises for cost by default; your per-request controls are `routing.sort` (`cost`, `speed`, `latency`, `exact`, also usable as a `model:latency` suffix), `routing.allowed_providers`, `routing.allow_fallbacks` and sticky routing (on by default, to preserve provider prompt caches) ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)). Smart routing goes further: `model: \"@edenai\"` lets Eden AI pick the model, optionally constrained by `router_candidates` ([Smart routing](https://www.edenai.co/docs/v3/llms/smart-routing)).", "architecture_class": "vendor_saas", "architecture_note": "vendor_saas: closed hosted service with no published source or runtime statement. The one architectural clue in the docs is the Anthropic endpoint, described as \"native pass-through via litellm\" ([Create Anthropic Message](https://www.edenai.co/docs/api-reference/anthropic-messages/create-anthropic-message), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)), and the catalog's capability/pricing field names (`supports_prompt_caching`, `cache_read_input_token_cost`) match LiteLLM's model-metadata vocabulary ([GET /v3/models](https://api.edenai.run/v3/models)). No language, runtime or edge-network claim is made on the fetched pages.", "self_host_data_plane": "not_documented", "self_host_artifact_note": "No Docker image, Helm chart, binary, npm/PyPI package or Terraform artifact for a customer-run data plane appears on any fetched page or in the complete documentation index, which lists integrations (Claude Code, Cline, Codex CLI, Continue.dev, LangChain, LibreChat, n8n, Open WebUI, OpenAI SDKs, OpenClaw, OpenCode) but no deployment section ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [Eden AI pricing](https://www.edenai.co/pricing)). The public GitHub organisation's 21 repositories are a docs site, a Claude Code skill, an n8n community node, an OpenClaw plugin, a cookbook and forks of third-party projects - no gateway server ([GitHub org API](https://api.github.com/orgs/edenai), [GitHub repos API](https://api.github.com/orgs/edenai/repos), both queried 2026-09-02).", "latency_claims": null, "throughput_claims": null, "streaming_support": "partial", "streaming_note": "partial by endpoint. The LLM endpoint streams over SSE with `stream: true`, `data:` frames and a `[DONE]` marker, and `stream_options.include_usage: true` adds normalized cache usage and `cost` to the final usage event ([Streaming](https://www.edenai.co/docs/v3/llms/streaming), [Prompt caching](https://www.edenai.co/docs/v3/llms/prompt-caching)). The Universal AI expert-model endpoint is explicitly listed as **not** streaming, so OCR, speech and translation results arrive whole or as async jobs ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)). Documented caveat on usage fields: \"Fields that the provider does not report can be absent or zero\" ([Prompt caching](https://www.edenai.co/docs/v3/llms/prompt-caching)).", "guard_pii": "not_documented", "guard_pii_note": "not_documented as a gateway guardrail. No PII detection, masking or redaction parameter appears on the chat-completions, routing, fallback or security pages, or in the documentation index ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions), [Security](https://www.edenai.co/security), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The closest features are self-service expert models you call yourself - `image/anonymization` (api4ai) and named-entity recognition - not an inline filter on LLM traffic ([GET /v3/info](https://api.edenai.run/v3/info)).", "guard_injection": "not_documented", "guard_injection_note": "not_documented: no prompt-injection or jailbreak detection appears on any fetched page, and the marketing phrase \"input protection\" on the security page is not backed by a documented feature ([Security](https://www.edenai.co/security), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "guard_content": "not_documented", "guard_content_note": "not_documented as an enforcing guardrail. Moderation exists as an API you call yourself: OpenAI-compatible `POST /v3/moderations` (default `openai/omni-moderation-latest`, returning `categories`, `category_scores` and `category_applied_input_types`, accepting text and image inputs) and expert-model `text/moderation` from google, microsoft and openai ([Create Moderation](https://www.edenai.co/docs/api-reference/moderations/create-moderation), [GET /v3/info](https://api.edenai.run/v3/info)). Neither is described as intercepting or blocking chat-completions traffic ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions)).", "guard_custom": "not_documented", "guard_custom_note": "not_documented: no regex, wordlist, JSON-schema-validation or webhook-classifier guardrail hook is published on any fetched page or in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "guard_model_policy": "not_documented", "guard_model_policy_note": "not_documented as an enforced policy. `routing.allowed_providers` constrains a single request's provider set and `router_candidates` constrains smart-routing choices, but both are caller-supplied preferences rather than account-level allowlists; the Advanced-plan RBAC table (Admin/Developer) covers project and key administration, not model permissions ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing), [Smart routing](https://www.edenai.co/docs/v3/llms/smart-routing), [Users & Organisation](https://www.edenai.co/docs/v3/general/users-organisation)).", "guard_execution_location": "not_documented", "guard_third_party": null, "guard_failure_mode": "not_documented", "guard_failure_mode_note": "not_documented: with no guardrail feature specified, there is no fail-open or fail-closed statement to record. The only stated failure semantics concern routing - failed attempts \"cost nothing\" and fall through to the next candidate ([Fallback](https://www.edenai.co/docs/v3/general/fallback), [Security](https://www.edenai.co/security)).", "log_content": "configurable", "log_content_note": "configurable: an optional \"Log Retention\" toggle, set per project in the dashboard, switches on storage of full request and response content for debugging and audit; with it off, only metadata is retained ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention), observed 2026-09-02). No retention period is published for the opt-in mode.", "log_opt_out": "yes", "log_opt_out_note": "Yes - and the default is already opted out. Content storage is opt-in per project via the Log Retention toggle, so leaving it off (or switching it off) keeps prompts and outputs unstored; GDPR access, deletion and portability requests go to support@edenai.co ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)).", "tracing_support": "not_documented", "tracing_note": "not_documented: no OpenTelemetry, OTLP, trace-id or span vocabulary appears on any fetched page or in the documentation index ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The nearest facility is per-request routing diagnostics - `x-edenai-metadata: enabled` returns each fallback attempt's provider, status and region - which is debugging metadata, not distributed tracing ([Fallback](https://www.edenai.co/docs/v3/general/fallback)).", "export_destinations": null, "export_destinations_note": "n.a. - no OTLP collector, Datadog/Grafana integration, Kafka or S3 sink, or CSV download is documented; usage and cost data leave only through the cost-monitoring REST endpoints ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "feedback_capture": "no", "feedback_capture_note": "n.a. - no feedback, rating, score or annotation endpoint appears on the monitoring page, in the API reference sections listed in the documentation index (chat, responses, embeddings, moderations, universal-ai, files, info, models, cost-monitoring, user-management) or on the pricing page ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "eval_hooks": "no", "eval_hooks_note": "n.a. as a programmatic eval or scoring hook. The only comparison tooling named is a dashboard feature, \"Compare models & performances\", listed on the self-serve plan ([Eden AI pricing](https://www.edenai.co/pricing)); there is no evals page, dataset, scorer or offline-experiment API in the documentation index ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "observability_requires_saas": "Yes by construction - there is no self-hosted mode, so cost monitoring, the model-comparison view and the log-retention toggle all live in Eden AI's dashboard and API ([Monitoring](https://www.edenai.co/docs/v3/general/monitoring), [Eden AI pricing](https://www.edenai.co/pricing), [Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)).", "log_retention_note": "There is no log-retention tier ladder: retention is a binary per-project toggle (metadata only, or full request/response content) with no stated duration for either mode, and no plan gating is mentioned ([Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)). Checked also on the plans and pricing pages, which do not mention log retention ([Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices), [Eden AI pricing](https://www.edenai.co/pricing)).", "production_meta": { "sources": {}, "qualified": [ "guard_pii", "guard_injection", "guard_content", "guard_custom", "guard_model_policy" ], "latency_prose": "No latency claim of any kind is published by Eden AI - not a gateway-overhead figure, not a percentile, not a time-to-first-token number - on the home, pricing, docs-overview, routing, streaming or status pages ([Eden AI home](https://www.edenai.co/), [AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway), [Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing), [Eden AI status](https://app-edenai.instatus.com/)). Nothing was estimated in its place.", "multi_surface": { "rel_fallback": [ "per_request" ], "rel_load_balancing": [ "per_request" ] }, "throughput_prose": "No throughput claim either. The only rate figures are imposed limits, and they contradict across vendor pages: \"10 reqs/sec (+ on request)\" and \"7 requests/second by default, upgradable to 15 req/sec\" on the plans page, versus \"Every account starts at 10 requests per second\" on the rate-limits page ([Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices), [Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits), both 2026-09-02). \"200k+ developers\" on the home page is a user count, not throughput ([Eden AI home](https://www.edenai.co/)).", "benchmark_independence": "No vendor or third-party benchmark exists to assess. The vendor's quantitative claims are availability and adoption figures (99.99% uptime, 200k+ developers), and independent coverage states neither has been audited ([Founderland analysis, 2026-04-27](https://www.founderland.ai/articles/eden-ai-launches-european-ai-gateway-to-challenge-openrouter-mogx7fzy)); the public status page showed \"All systems operational\" with a 99.98% uptime figure on 2026-09-02 ([Eden AI status](https://app-edenai.instatus.com/))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "The reliability surface is deliberately thin and per-request: fallbacks (max 3), provider failover, region pinning and routing preferences are all body parameters, with no timeout, retry or circuit-breaker knobs at all ([Fallback](https://www.edenai.co/docs/v3/general/fallback), [Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)). Rate limiting is the sharpest operational edge, and the vendor's own numbers contradict: the plans table says \"10 reqs/sec (+ on request)\" while the prose on the same page says \"7 requests/second by default, upgradable to 15 req/sec on request\", and the dedicated page says \"Every account starts at 10 requests per second\", shared across all API keys with the organisation owner's limit acting as the ceiling for members ([Plans & Pricing](https://www.edenai.co/docs/v3/overview/plans-prices), [Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits), both observed 2026-09-02).", "guard_notes": "The gap between marketing and documentation is the story here. The security page answers \"Does Eden AI provide guardrails? Yes\" and lists input protection, output moderation, policy-based routing and fallback rules ([Security](https://www.edenai.co/security)), yet the complete documentation index contains no guardrails page and no request parameter for any of it ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). What you can actually build is DIY: call `/v3/moderations` or a `text/moderation` expert model before or after your LLM call, on your own latency and cost budget ([Create Moderation](https://www.edenai.co/docs/api-reference/moderations/create-moderation), [GET /v3/info](https://api.edenai.run/v3/info)). Every guard field above is therefore not_documented rather than false.", "fit_agent_features_note": "Solid pass-through of agent primitives, discoverable per model. Tool/function calling is listed as a use case for the LLM endpoint and surfaced as catalog capabilities `supports_tools`, `supports_function_calling`, `supports_parallel_function_calling` and `supports_tool_choice`, with `tool_calls` appearing as a streaming finish reason ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models), [GET /v3/models](https://api.edenai.run/v3/models), [Streaming](https://www.edenai.co/docs/v3/llms/streaming)). Structured output supports both `response_format: {\"type\": \"json_object\"}` and `json_schema` with `strict: true` ([Structured output](https://www.edenai.co/docs/v3/llms/structured-output)). Web search is `web_search_options.search_context_size` (`low`/`medium`/`high`, default `medium`), gated by `capabilities.supports_web_search`, with a documented OpenAI caveat: on `/v3/chat/completions` it works only on dedicated search variants such as `openai/gpt-4o-search-preview`, otherwise use the Responses API ([Web search](https://www.edenai.co/docs/v3/llms/web-search)). Extended reasoning is Claude-only via `thinking.budget_tokens` (minimum 1024, and `top_p` is ignored when enabled) ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions)). Statefulness comes from the Responses API with `store` and `previous_response_id` ([Responses](https://www.edenai.co/docs/v3/llms/responses)). No MCP and no built-in tool-execution loop are documented ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "fit_byok_required": "optional", "fit_byok_required_note": "Optional. The first call needs only an Eden AI token plus credits (or a free sandbox token) - no provider account ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call)). Bringing your own provider keys is an explicit alternative that shifts billing to the provider, and the two modes can be mixed ([BYOK](https://www.edenai.co/docs/v3/general/byok)).", "fit_client_sdk_langs": [ "Python", "TypeScript", "JavaScript" ], "fit_client_sdks_note": "No Eden AI SDK exists; the documented clients are the official OpenAI SDKs with a swapped base URL - `openai` for Python and `openai` for TypeScript/JavaScript - plus plain `requests`/`fetch`/cURL examples ([OpenAI Python SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-python), [OpenAI TS/JS SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-typescript), [First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call)). LangChain examples add `langchain-openai` (Python) and `@langchain/openai` (TypeScript) ([LangChain integration](https://www.edenai.co/docs/v3/integrations/langchain)).", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "not_documented: authentication is a bearer token issued by Eden AI, and no AWS SigV4, IAM role, GCP service account, Azure managed identity, OIDC or workload-identity option appears on the BYOK, custom-keys or organisation pages ([BYOK](https://www.edenai.co/docs/v3/general/byok), [Custom API keys](https://www.edenai.co/docs/v3/general/custom-api-keys), [Users & Organisation](https://www.edenai.co/docs/v3/general/users-organisation)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "not_documented as a runtime or deployment target. Cloudflare appears twice in unrelated roles: as one of the 33 LLM serving providers in the catalog (`owned_by: cloudflare`) ([GET /v3/models](https://api.edenai.run/v3/models)), and as a named third-party service provider in the privacy policy alongside Hotjar, Intercom, Stripe, Google and Sentry ([Privacy policy](https://www.edenai.co/privacy)). Neither states that Eden AI's gateway runs on Workers.", "fit_credit_card": "required_for_model_access", "fit_credit_card_note": "Payment is required before real model calls: the quickstart's second prerequisite is \"Credits - ensure your account has sufficient credits\", and an empty balance returns `402 Payment Required` ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call), [Buying credits](https://www.edenai.co/docs/v3/general/buying-credits)). A card is not the only route - credits can be bought by card, PayPal or bank transfer through Stripe, with AWS Marketplace listed as \"coming soon\" - and testing without paying is possible via sandbox tokens or by using your own provider keys ([Buying credits](https://www.edenai.co/docs/v3/general/buying-credits), [Sandbox](https://www.edenai.co/docs/v3/general/sandbox), [BYOK](https://www.edenai.co/docs/v3/general/byok)).", "fit_ecosystem_notes": "Ecosystem coverage is client-side rather than infrastructure-side. Documented integrations: Claude Code, Cline, Codex CLI, Continue.dev, LangChain, LibreChat, n8n (official community node), Open WebUI, OpenAI Python and TS/JS SDKs, OpenClaw and OpenCode ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)). The GitHub organisation backs some of these with first-party code - `n8n-nodes-edenai`, `edenai-skill` (a Claude Code skill), `edenai-openclaw-plugin`, `pi-edenai` and a `cookbook` of notebooks - alongside forks of third-party projects ([GitHub repos API](https://api.github.com/orgs/edenai/repos), 2026-09-02). Absent from both: any Vercel AI SDK provider, Terraform, Kubernetes/Helm or self-hosting path ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "fit_eval_prereqs": "An Eden AI account and API token from `app.edenai.run`, plus either credits or a free sandbox token; no cloud account, provider key, container or cluster is required ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call), [Sandbox](https://www.edenai.co/docs/v3/general/sandbox)).", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, complete and copy-pasteable in three languages on the quickstart: `curl -X POST https://api.edenai.run/v3/chat/completions -H \"Authorization: Bearer YOUR_API_KEY\" -H \"Content-Type: application/json\" -d '{\"model\": \"openai/gpt-4\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello!\"}]}'`, plus Python `requests` and JavaScript `fetch` equivalents ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Eden AI is itself the gateway: \"a unified AI gateway... through a single API\", hosted at `https://api.edenai.run/v3` with routing, fallback and cost tracking built in ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)). It is not documented as a plugin to any API-gateway platform ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)).", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "not_documented: no Kubernetes, Helm, operator or container guidance exists, consistent with there being no self-hosted component at all - the documentation index has no deployment section ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [Eden AI pricing](https://www.edenai.co/pricing)).", "fit_mcp": null, "fit_mcp_note": "Left empty because no enum value describes an absence here: MCP is not mentioned anywhere - no hosted MCP server, no MCP gateway, no MCP tool parameters, and no docs-MCP endpoint - across the documentation index, the API reference sections and the security page ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [Security](https://www.edenai.co/security)). The catalog does expose a per-model `supports_computer_use` capability flag, which is a model attribute passed through, not an Eden AI MCP feature ([GET /v3/models](https://api.edenai.run/v3/models)). Checked 2026-09-02.", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_cloudflare_workers", "fit_kubernetes", "fit_terraform", "fit_vercel_ai_sdk", "fit_mcp" ] }, "fit_model_gate": "none", "fit_model_gate_note": "none: no approval, enablement, waitlist, quota or tier step appears between funding an account and calling any catalog model - the quickstart's only prerequisites are a token and credits, the catalog is publicly readable, and provider-less model names are routed automatically ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call), [GET /v3/models](https://api.edenai.run/v3/models), [Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)). The only gating found is commercial (credits, and the 10 req/s account limit) and organisational (Advanced-plan RBAC over projects and keys) ([Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits), [Users & Organisation](https://www.edenai.co/docs/v3/general/users-organisation)).", "fit_onboarding_notes": "Onboarding is unusually low-friction for a hosted gateway: the model catalog is public and unauthenticated (`GET https://api.edenai.run/v3/models` returned 1,010 endpoints with per-endpoint pricing, capabilities and regions on 2026-09-02, and `GET /v3/embeddings/models` likewise), so you can price and capability-check before signing up ([GET /v3/models](https://api.edenai.run/v3/models), [Embeddings](https://www.edenai.co/docs/v3/llms/embeddings)). Sandbox tokens let you wire up the integration for free before funding an account ([Sandbox](https://www.edenai.co/docs/v3/general/sandbox)). Two friction points to know about: the expert-model endpoint uses a different model-string grammar (`feature/subfeature/provider[/model]`) and a different response envelope from the LLM endpoint ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)), and pre-2026 accounts are still on the old app until end of 2026 ([AI gateway overview](https://www.edenai.co/docs/v3/overview/ai-gateway)).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes, exactly: \"full OpenAI API compatibility... a drop-in replacement\", with the OpenAI Python and TypeScript SDKs pointed at `base_url=\"https://api.edenai.run/v3\"` and the Eden AI key in place of the OpenAI key ([Chat completions](https://www.edenai.co/docs/v3/llms/chat-completions), [OpenAI Python SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-python), [OpenAI TS/JS SDK](https://www.edenai.co/docs/v3/integrations/openai-sdk-typescript)). The one boundary is that the expert-model endpoint is not OpenAI-SDK compatible ([LLMs vs Expert Models](https://www.edenai.co/docs/v3/overview/llms-vs-expert-models)).", "fit_prod_prereqs": "An API token, funded credits (or BYOK provider keys), and awareness of the account rate limit - 10 requests/second by default, shared across all API keys, raised on request, with the organisation owner's limit capping members ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call), [Rate limits](https://www.edenai.co/docs/v3/overview/rate-limits), [BYOK](https://www.edenai.co/docs/v3/general/byok)). Optional production hardening: send traffic to the EU host for residency, create per-environment custom keys with budgets and expiry, and enable the per-project Log Retention toggle only if you actually want prompts stored ([EU AI gateway](https://www.edenai.co/eu), [Custom API keys](https://www.edenai.co/docs/v3/general/custom-api-keys), [Data retention](https://www.edenai.co/docs/v3/data-governance/data-retention)). Advanced-plan features - projects, RBAC, per-environment spend limits, dedicated support with an SLA - require a sales conversation ([Users & Organisation](https://www.edenai.co/docs/v3/general/users-organisation), [Eden AI pricing](https://www.edenai.co/pricing)).", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain" ], "fit_python_frameworks_note": "LangChain is the only documented Python framework: `pip install \"langchain~=1.2\" \"langchain-openai~=1.1\" \"langchain-community~=0.4\" \"langgraph~=1.0\"`, then `ChatOpenAI` with `base_url=\"https://api.edenai.run/v3\"`; the same page covers the TypeScript `@langchain/openai` path ([LangChain integration](https://www.edenai.co/docs/v3/integrations/langchain)). LlamaIndex, Haystack, DSPy and CrewAI: n.a. - not in the documentation index, although the GitHub organisation holds forks of `llama_index` and `haystack-core-integrations` with no Eden AI provider code described ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [GitHub repos API](https://api.github.com/orgs/edenai/repos)).", "fit_quickstart_steps": 2, "fit_quickstart_steps_caveat": "The quickstart numbers only its two prerequisites (API token from the dashboard, then credits or a sandbox token); the call itself is an unnumbered code block, so \"2\" is the page's own count rather than a steps-to-first-call figure ([First LLM Call](https://www.edenai.co/docs/v3/quickstart/first-llm-call)).", "fit_quickstart_url": "https://www.edenai.co/docs/v3/quickstart/first-llm-call", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "not_documented: no Terraform provider, module or registry reference appears in the documentation index or among the 21 public repositories in the GitHub organisation ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt), [GitHub repos API](https://api.github.com/orgs/edenai/repos)). Account configuration is done in the dashboard and per request, not declaratively ([Provider routing](https://www.edenai.co/docs/v3/llms/provider-routing)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "not_documented: no Vercel AI SDK page, provider package or `createOpenAI`-style example appears in the complete documentation index, whose integrations section lists 12 other clients ([docs index (llms.txt)](https://www.edenai.co/docs/llms.txt)), and no `*-vercel-provider` repository exists in the GitHub organisation ([GitHub repos API](https://api.github.com/orgs/edenai/repos)). The endpoint's OpenAI compatibility makes the SDK's OpenAI-compatible provider a plausible route, but that is inference, not documentation.", "fit_vercel_ai_sdk_package": null }, { "slug": "huggingface-inference-providers", "name": "Hugging Face Inference Providers", "company": "Hugging Face, Inc.", "tagline": "Zero-markup routing layer that fans a single Hugging Face token out to 18 third-party inference partners.", "category": "managed_marketplace", "deployment": "managed_only", "license": "Proprietary", "founded_year": null, "hq_country": "United States", "url_home": "https://huggingface.co/docs/inference-providers/index", "url_docs": "https://huggingface.co/docs/inference-providers/index", "url_pricing": "https://huggingface.co/docs/inference-providers/pricing", "url_status": "https://status.huggingface.co/", "url_github": "https://github.com/huggingface", "url_changelog": "https://huggingface.co/changelog", "url_models_api": "https://router.huggingface.co/v1/models", "github_stars": null, "model_count": 136, "model_count_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 18, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Bring-your-own key is called a \"Custom Provider Key\": requests still traverse the Hugging Face router (\"HF routing: Yes\") but are \"Billed by: Provider\", there is no free-tier allowance in that mode, and \"Hugging Face won't charge you for the call\" - so BYOK carries no Hugging Face fee at all ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), 2026-09-03).", "seat_fee_usd_month": 0, "free_tier": "Included monthly credits: $0.10 for signed-in free accounts, $2.00 on PRO, and $2.00 per seat (pooled) on Team and Enterprise, all described as \"subject to change\". Free accounts must purchase credits to continue once the included amount is spent ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), 2026-09-03).", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "No self-hosted deployment exists, so there is no self-host cost line: the service is a hosted proxy (\"your requests go through Hugging Face's proxy infrastructure\") and the only cost is the pass-through provider rate plus optional Hub subscription credits ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": null, "config_as_code": false, "portability_note": "Getting off is easy, taking your data with you is not. Portability of code is excellent: the OpenAI base URL swaps back to any other OpenAI-compatible endpoint in one line. But no usage, log or cost export is documented - the billing dashboard is only described as something you monitor - and there are no stored prompts to export by design ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub billing](https://huggingface.co/docs/hub/billing), [Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Responses (beta)", "Hugging Face InferenceClient task API", "Hub model/provider metadata API" ], "sdk_note": "First-party clients are `huggingface_hub` (Python) and `@huggingface/inference` (JS/TS), but no vendor SDK is required: the docs' own examples use the OpenAI Python and Node SDKs and raw `curl` against `https://router.huggingface.co/v1` ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api)).", "cap_failover": true, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Hugging Face publishes no gateway-overhead figure and no percentile latency for the router. The only latency numbers in its documentation are provider-admission thresholds (under 5 s time-to-first-token for conversational and text models, under 30 s for other tasks) and the per-provider `first_token_latency_ms` and `throughput` values returned by the public models endpoint, which describe upstream providers rather than the proxy hop. Pages checked: [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Hub API](https://huggingface.co/docs/inference-providers/hub-api).", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Not documented for routed inference. The Enterprise Hub \"Storage Regions - Select, manage, and audit the location of your repository data\" feature is about repository data and was deliberately not used here; the Inference Providers security page says nothing about processing location ([Enterprise Hub](https://huggingface.co/enterprise), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Privacy Policy](https://huggingface.co/privacy)).", "zdr": true, "zdr_note": "Zero retention of request content is the default and is stated unconditionally for the routing product: \"We do not store the request body or response when routing requests through Hugging Face.\" No sign-up, contract or enterprise tier is required to get it, and no exception is carved out for the beta Responses API ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), 2026-09-03).", "air_gapped": null, "best_for": "Teams calling open-weight models who want one token, one bill and genuinely zero markup across 18 serverless providers, with automatic failover they do not have to configure.", "strengths": [ "Zero markup, stated four times across the docs: \"Hugging Face charges you the same rates as the provider, with no additional fees\"", "Strong default data posture for a gateway: request bodies and responses are not stored, no training on user data, 30-day debug logs only", "Automatic cross-provider failover plus a real HF-run validation system (6-hourly model tests, sub-5s TTFT admission bar, tool-calling and structured-output checks)", "Routing policy as a one-string change: `:fastest`, `:cheapest`, `:preferred` or an explicit provider pin, with `provider=\"auto\"` as the default", "Public unauthenticated model catalogue at /v1/models carrying per-provider price, context length, tool support and status", "Beta Responses API with remote MCP tools, and an official Vercel AI SDK provider (@ai-sdk/huggingface)", "BYOK mode keeps HF routing while the provider bills you, and Hugging Face charges nothing for those calls" ], "weaknesses": [ "No guardrails of any kind - no PII redaction, moderation, injection detection or model policy for callers", "No caching, prompt management, tracing or OTel export; observability is a billing dashboard and a usage graph", "No user-settable timeout, retry, weight or region: the entire reliability surface is HF-operated", "No SLA and no Inference Providers component on status.huggingface.co - the status page tracks only a single \"huggingface.co\" component", "Compliance material is Hub-scoped, not gateway-scoped: SOC 2 Type 2 is asserted for the Hub \"which Inference Providers is a feature of\", and BAAs/DPAs come via an Enterprise Hub plan", "The company subprocessor list does not name any of the 18 inference partners your prompts are routed to", "Catalogue is open-weight only - no Anthropic, OpenAI or Google frontier models, and no Anthropic Messages surface", "hf-inference is both the router and one of its own 18 partners, and \"as of July 2025 focuses mostly on CPU inference\"", "OpenAI compatibility covers chat and Responses only; embeddings, images, video and speech need InferenceClient", "Free-tier allowance is $0.10 of credits a month, so any real evaluation requires a card" ], "avoid_if": "You need frontier proprietary models, guardrails, prompt management, caching, tracing, an SLA, or any per-request control over timeouts, retries and regions.", "reviews": [ { "url": "https://klymentiev.com/blog/huggingface-inference-api", "why": "The only independent write-up found that separates Inference Providers from Inference Endpoints and the Hub, confirms the pass-through pricing posture, and sets it against OpenRouter's markup; also the source of a credit-figure contradiction noted in the companion file.", "date": "2026-05-10", "type": "written", "title": "Hugging Face Inference API Free Tier Limits & Pricing 2026", "publisher": "Dmytro Klymentiev" }, { "url": "https://fireworks.ai/blog/best-llm-api-providers", "why": "Useful competitive framing of the three-way Hugging Face billing model and the pass-through rate, but read with the conflict in mind: Fireworks is itself one of the 18 upstream partners inside the product it is rating.", "date": "2026-03-04", "type": "written", "title": "Best LLM API Providers in 2026: We Reviewed 8 Options", "publisher": "Fireworks AI" } ], "sources": { "zdr": "https://huggingface.co/docs/inference-providers/security", "company": "https://huggingface.co/terms-of-service", "fit_mcp": "https://huggingface.co/docs/inference-providers/guides/responses-api", "license": "https://huggingface.co/docs/inference-providers/index", "reviews": "https://klymentiev.com/blog/huggingface-inference-api", "url_docs": "https://huggingface.co/docs/inference-providers/index", "url_home": "https://huggingface.co/docs/inference-providers/index", "api_audio": "https://huggingface.co/docs/inference-providers/tasks/automatic-speech-recognition", "free_tier": "https://huggingface.co/docs/inference-providers/pricing", "logs_note": "https://huggingface.co/docs/inference-providers/security", "rel_notes": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "deployment": "https://huggingface.co/docs/inference-providers/index", "extra_fees": "https://huggingface.co/docs/inference-providers/pricing", "hq_country": "https://huggingface.co/terms-of-service", "url_github": "https://huggingface.co/docs/inference-providers/index", "url_status": "https://status.huggingface.co/", "cap_budgets": "https://huggingface.co/enterprise", "guard_notes": "https://huggingface.co/docs/inference-providers/security", "model_count": "https://router.huggingface.co/v1/models", "url_pricing": "https://huggingface.co/docs/inference-providers/pricing", "api_surfaces": "https://huggingface.co/docs/inference-providers/index", "cap_failover": "https://huggingface.co/docs/inference-providers/index", "product_fate": "https://huggingface.co/docs/inference-providers/index", "regions_note": "https://huggingface.co/privacy", "byok_fee_note": "https://huggingface.co/docs/inference-providers/pricing", "cache_support": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "cost_controls": "https://huggingface.co/enterprise", "pricing_model": "https://huggingface.co/docs/inference-providers/pricing", "training_note": "https://huggingface.co/docs/inference-providers/security", "url_changelog": "https://huggingface.co/changelog", "vendor_status": "https://huggingface.co/terms-of-service", "api_batch_note": "https://huggingface.co/docs/inference-providers/tasks/index", "api_embeddings": "https://huggingface.co/docs/inference-providers/tasks/feature-extraction", "byok_mode_note": "https://huggingface.co/docs/inference-providers/pricing", "byok_supported": "https://huggingface.co/docs/inference-providers/pricing", "certifications": "https://huggingface.co/docs/inference-providers/security", "fit_model_gate": "https://huggingface.co/docs/inference-providers/pricing", "latency_claims": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "provider_count": "https://huggingface.co/docs/inference-providers/index", "retention_days": "https://huggingface.co/docs/inference-providers/security", "retention_note": "https://huggingface.co/docs/inference-providers/security", "url_models_api": "https://huggingface.co/docs/inference-providers/hub-api", "guardrails_note": "https://huggingface.co/docs/inference-providers/security", "logs_by_default": "https://huggingface.co/docs/inference-providers/security", "model_count_low": "https://router.huggingface.co/v1/models", "tracing_support": "https://huggingface.co/docs/inference-providers/hub-integration", "api_surface_note": "https://huggingface.co/docs/inference-providers/index", "model_count_high": "https://router.huggingface.co/v1/models", "model_count_note": "https://router.huggingface.co/v1/models", "portability_note": "https://huggingface.co/docs/hub/billing", "rel_retry_config": "https://huggingface.co/docs/inference-providers/index", "token_markup_pct": "https://huggingface.co/docs/inference-providers/pricing", "architecture_note": "https://huggingface.co/docs/inference-providers/index", "cap_observability": "https://huggingface.co/docs/inference-providers/hub-integration", "eu_residency_note": "https://huggingface.co/docs/inference-providers/security", "fit_openai_dropin": "https://huggingface.co/docs/inference-providers/index", "fit_vercel_ai_sdk": "https://ai-sdk.dev/providers/ai-sdk-providers/huggingface", "last_release_note": "https://huggingface.co/changelog", "openai_compatible": "https://huggingface.co/docs/inference-providers/index", "product_kind_note": "https://huggingface.co/docs/inference-providers/index", "rel_fallback_note": "https://huggingface.co/docs/inference-providers/index", "streaming_support": "https://huggingface.co/docs/inference-providers/tasks/chat-completion", "subprocessors_url": "https://huggingface.co/privacy", "architecture_class": "https://huggingface.co/docs/inference-providers/index", "cache_pricing_note": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "fit_quickstart_url": "https://huggingface.co/docs/inference-providers/guides/first-api-call", "merchant_of_record": "https://huggingface.co/docs/inference-providers/pricing", "rel_timeout_config": "https://huggingface.co/docs/inference-providers/index", "seat_fee_usd_month": "https://huggingface.co/pricing", "fit_ecosystem_notes": "https://huggingface.co/docs/inference-providers/hub-integration", "guardrails_location": "https://huggingface.co/docs/inference-providers/security", "key_management_note": "https://huggingface.co/docs/inference-providers/hub-integration", "latency_source_note": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "rel_fallback_config": "https://huggingface.co/docs/inference-providers/index", "api_openai_responses": "https://huggingface.co/docs/inference-providers/guides/responses-api", "self_host_data_plane": "https://huggingface.co/docs/inference-providers/index", "fit_python_frameworks": "https://huggingface.co/docs/inference-providers/integrations/index", "fit_vendor_time_claim": "https://huggingface.co/docs/inference-providers/guides/first-api-call", "rel_health_check_note": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "prompt_transits_vendor": "https://huggingface.co/docs/inference-providers/security", "model_count_source_note": "https://router.huggingface.co/v1/models", "rel_health_check_config": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "self_host_artifact_note": "https://huggingface.co/docs/inference-providers/index", "trains_on_customer_data": "https://huggingface.co/docs/inference-providers/security", "rel_load_balancing_config": "https://huggingface.co/docs/inference-providers/index", "provider_count_source_note": "https://huggingface.co/docs/inference-providers/index", "api_anthropic_messages_note": "https://huggingface.co/docs/inference-providers/index", "api_openai_chat_completions": "https://huggingface.co/docs/inference-providers/index" }, "verified_at": { "zdr": "2026-09-03", "company": "2026-09-03", "fit_mcp": "2026-09-03", "license": "2026-09-03", "reviews": "2026-09-03", "url_docs": "2026-09-03", "url_home": "2026-09-03", "api_audio": "2026-09-03", "free_tier": "2026-09-03", "logs_note": "2026-09-03", "rel_notes": "2026-09-03", "deployment": "2026-09-03", "extra_fees": "2026-09-03", "hq_country": "2026-09-03", "url_github": "2026-09-03", "url_status": "2026-09-03", "cap_budgets": "2026-09-03", "guard_notes": "2026-09-03", "model_count": "2026-09-23", "url_pricing": "2026-09-03", "api_surfaces": "2026-09-03", "cap_failover": "2026-09-03", "product_fate": "2026-09-03", "regions_note": "2026-09-03", "byok_fee_note": "2026-09-03", "cache_support": "2026-09-03", "cost_controls": "2026-09-03", "pricing_model": "2026-09-03", "training_note": "2026-09-03", "url_changelog": "2026-09-03", "vendor_status": "2026-09-03", "api_batch_note": "2026-09-03", "api_embeddings": "2026-09-03", "byok_mode_note": "2026-09-03", "byok_supported": "2026-09-03", "certifications": "2026-09-03", "fit_model_gate": "2026-09-03", "latency_claims": "2026-09-03", "provider_count": "2026-09-03", "retention_days": "2026-09-03", "retention_note": "2026-09-03", "url_models_api": "2026-09-03", "guardrails_note": "2026-09-03", "logs_by_default": "2026-09-03", "model_count_low": "2026-09-23", "tracing_support": "2026-09-03", "api_surface_note": "2026-09-03", "model_count_high": "2026-09-23", "model_count_note": "2026-09-23", "portability_note": "2026-09-03", "rel_retry_config": "2026-09-03", "token_markup_pct": "2026-09-03", "architecture_note": "2026-09-03", "cap_observability": "2026-09-03", "eu_residency_note": "2026-09-03", "fit_openai_dropin": "2026-09-03", "fit_vercel_ai_sdk": "2026-09-03", "last_release_note": "2026-09-03", "openai_compatible": "2026-09-03", "product_kind_note": "2026-09-03", "rel_fallback_note": "2026-09-03", "streaming_support": "2026-09-03", "subprocessors_url": "2026-09-03", "architecture_class": "2026-09-03", "cache_pricing_note": "2026-09-03", "fit_quickstart_url": "2026-09-03", "merchant_of_record": "2026-09-03", "rel_timeout_config": "2026-09-03", "seat_fee_usd_month": "2026-09-03", "fit_ecosystem_notes": "2026-09-03", "guardrails_location": "2026-09-03", "key_management_note": "2026-09-03", "latency_source_note": "2026-09-03", "rel_fallback_config": "2026-09-03", "api_openai_responses": "2026-09-03", "self_host_data_plane": "2026-09-03", "fit_python_frameworks": "2026-09-03", "fit_vendor_time_claim": "2026-09-03", "rel_health_check_note": "2026-09-03", "prompt_transits_vendor": "2026-09-03", "model_count_source_note": "2026-09-23", "rel_health_check_config": "2026-09-03", "self_host_artifact_note": "2026-09-03", "trains_on_customer_data": "2026-09-03", "rel_load_balancing_config": "2026-09-03", "provider_count_source_note": "2026-09-03", "api_anthropic_messages_note": "2026-09-03", "api_openai_chat_completions": "2026-09-03" }, "locked_fields": [], "published": true, "updated_at": "2026-09-23T22:06:27.788109+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed and clearly invested in: the partner roster has grown to 18, a beta Responses API with remote MCP has landed, and Hugging Face publishes an open onboarding spec for new providers. The caveat is operational transparency rather than momentum - there is no Inference Providers component on the status page and no product-specific changelog ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [status.huggingface.co](https://status.huggingface.co/)).", "last_release_date": null, "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "No dated release artefact for this product. The only changelog is Hub-wide, and none of the entries in the fetched window (Granular Feature Access on 2026-08-12, then 2026-08-03, 2026-07-22 MCP Server, 2026-07-21, 2026-07-16) is Inference-Providers-specific; the router is a service with no version number and the docs pages carry no version or date stamp ([Changelog](https://huggingface.co/changelog), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "prompt_transits_vendor": "yes", "prompt_path_note": "Yes, in both billing modes: \"When using Inference Providers, your requests go through Hugging Face's proxy infrastructure\", and even the Custom Provider Key mode is listed as \"HF routing: Yes\". Hugging Face states it does not store the request body or response while doing so ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "logs_by_default": "metadata_only", "logs_note": "Content is not logged: \"We do not store the request body or response when routing requests through Hugging Face. Logs are kept for debugging purposes for up to 30 days, but no user data or tokens are stored.\" Usage metadata is retained for billing and appears in the account dashboard ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), 2026-09-03).", "retention_days": 30, "retention_note": "\"Logs are kept for debugging purposes for up to 30 days, but no user data or tokens are stored\" - so the 30 days covers operational logs, not prompts or completions, which are not stored at all ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), 2026-09-03).", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "\"Hugging Face does not store any user data for training purposes\" on the Inference Providers security page. What each upstream partner does with routed traffic is not covered by that statement and is not summarised anywhere in the Inference Providers docs ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), 2026-09-03).", "subprocessors_url": "https://huggingface.co/privacy", "regions_note": "Routed inference has no published region control. The privacy policy states \"The Company and its servers are located in the United States\", with Hugging Face SAS in Paris as the EU establishment and CNIL as lead authority; the upstream partners include EU-based operators (OVHcloud AI Endpoints, Scaleway) but choosing them is a provider-pinning decision, not a documented residency guarantee ([Privacy Policy](https://huggingface.co/privacy), [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Scoped to the Hub, not to the gateway: \"The Hugging Face Hub, which Inference Providers is a feature of, is SOC2 Type 2 certified\" (https://huggingface.co/docs/inference-providers/security). No report, audit period, auditor or trust portal is named, and no Inference-Providers-specific attestation exists, so the `soc2` field is left unset.", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "Company/Hub level only: \"Hugging Face is GDPR compliant\" and DPAs are offered \"through an Enterprise Plan\" on the Hub security page (https://huggingface.co/docs/hub/security). No Inference-Providers DPA or processing appendix is published, so the `gdpr` field is left unset.", "status": "claimed" }, "iso27001": { "detail": "No ISO 27001 claim on the Inference Providers security page or the Hub security page (https://huggingface.co/docs/inference-providers/security, https://huggingface.co/docs/hub/security).", "status": "not_published" }, "hipaa_baa": { "detail": "Hub/Enterprise level only: Business Associate Addendums are offered \"through an Enterprise Plan\" (https://huggingface.co/docs/hub/security). Nothing ties a BAA to routed inference through the 18 third-party providers, so `hipaa_baa` is left unset.", "status": "claimed" } }, "guardrails_location": "none", "guardrails_note": "None. Hugging Face's own statements point the other way - it does not inspect or store request bodies at all (\"We do not store the request body or response when routing requests through Hugging Face\") - so any moderation must be implemented by the caller or inherited from the upstream provider ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), 2026-09-03).", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "The routing layer itself is $0 and inference is billed at the upstream provider's own rate: \"Hugging Face charges you the same rates as the provider, with no additional fees. We just pass through the provider costs directly.\" Optional Hub subscriptions (PRO $9/month, Team $20/user/month, Enterprise $50/user/month) are not required for access but each carries monthly inference credits, which makes the shape a subscription-plus-passthrough hybrid rather than a pure markup. On the separate question of a credit-purchase fee: none is documented either way - neither the Inference Providers pricing page, the Hub billing FAQ, nor the Hugging Face pricing page states a percentage or minimum on credit purchases, so `credit_fee_pct` and `credit_fee_min_usd` are left unset rather than recorded as zero ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub billing](https://huggingface.co/docs/hub/billing), [Hugging Face pricing](https://huggingface.co/pricing), 2026-09-03).", "min_commitment": "None stated for pay-as-you-go; all users, including free accounts, can purchase credits on demand and enable automatic recharge. Enterprise Hub is quoted per seat per month with \"yearly commit options\" mentioned only on the Enterprise page, which is a Hub-level plan rather than a gateway commitment ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Enterprise Hub](https://huggingface.co/enterprise)).", "overage_note": "Once the included monthly credits are exhausted, usage continues pay-as-you-go at provider rates and is billed to the Hugging Face account; free-tier users must first purchase credits, and automatic recharge can be enabled to avoid interruption ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), 2026-09-03).", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "No gateway cache. Hugging Face does not document an exact-match, prefix or semantic cache of its own on the index, pricing, chat-completion or Responses pages. Provider-side caching passes through economically: the provider billing spec tells partners that \"`0` is a valid cost, for instance when you serve a cached response for free\", so a cached upstream response can reach the caller at zero cost. No cached-token discount or premium is published by Hugging Face ([Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), 2026-09-03).", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not documented; automatic recharge is described but not a spend alert (https://huggingface.co/docs/inference-providers/pricing).", "tier": null, "supported": null }, "virtual_keys": { "note": "Not documented as a gateway feature. Access uses Hub user access tokens, and a fine-grained token can be scoped to the \"Make calls to Inference Providers\" permission, but there is no per-key spend object described (https://huggingface.co/docs/hub/security-tokens).", "tier": null, "supported": null }, "rate_limiting": { "note": "No Inference Providers rate limit is published. The Enterprise page's \"API rate limit - Throughput for programmatic access to the Hub\" (1,000-6,000 req/5 min by tier) is explicitly a Hub API limit and was not used here (https://huggingface.co/enterprise).", "tier": null, "supported": null }, "spend_webhooks": { "note": "Not documented on any fetched billing page.", "tier": null, "supported": null }, "model_allowlists": { "note": "Organisation admins can disable a set of Inference Providers from organisation settings, which is a provider allowlist rather than a model allowlist (https://huggingface.co/docs/inference-providers/pricing).", "tier": "enterprise", "supported": true }, "budget_caps_per_key": { "note": "Not documented. No per-token or per-key budget appears on the pricing or hub-integration pages.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "\"Inference Providers organization billing - Centralized usage, analytics, and spending limits\" is listed as a Team/Enterprise row on the Enterprise Hub feature table, and the Enterprise page repeats \"manage spending limits\" for Inference Providers specifically (https://huggingface.co/enterprise).", "tier": "enterprise", "supported": true } }, "cost_attribution": "Per-provider and per-feature usage appears in the account billing dashboard, organisation settings add \"a graph of your team member's usage over time\", and individual calls can be attributed to an organisation or resource group with the `X-HF-Bill-To` header. Per-key, per-tag and per-customer attribution are not documented ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "cost_export": "No CSV, API, webhook or warehouse cost export is documented. The billing dashboard is described only as something you \"monitor\" ([Hub billing](https://huggingface.co/docs/hub/billing)); the Inference Providers pricing and hub-integration pages describe usage graphs but no export ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "product_kind_note": "A routing/marketplace product rather than a model host: \"a unified proxy layer that sits between your application and multiple AI providers\", with \"Use a single Hugging Face token for all providers\" and centralised billing. It has no standalone marketing site - the docs index is its home page. Scope note: this entry covers only that router, not the Hugging Face Hub repository platform and not Inference Endpoints, which is a dedicated single-model hosting product ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "deployment_modes": [ "saas" ], "deployment_note": "There is nothing to deploy: you point an OpenAI client at `https://router.huggingface.co/v1` with a Hugging Face token, or install `huggingface_hub` / `@huggingface/inference` for the non-chat tasks ([Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "partial", "api_batch": "not_documented", "api_surface_note": "Three surfaces on one base URL plus a client library. `https://router.huggingface.co/v1` serves OpenAI Chat Completions - \"a drop-in compatible endpoint that handles all provider selection automatically on the server side\" - and a beta Responses API; the twenty-odd classic tasks (feature extraction, text-to-image, text-to-video, ASR, classification, translation) are reached through `InferenceClient` in `huggingface_hub`/`@huggingface/inference` rather than the OpenAI path, which the docs say is \"available for chat completion tasks only\". The launch blog also documents provider-native pass-through at `https://router.huggingface.co/{:provider}` ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [API Reference](https://huggingface.co/docs/inference-providers/tasks/index), [launch blog](https://huggingface.co/blog/inference-providers)).", "model_count_low": 136, "model_count_high": 136, "model_count_source_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": 14, "provider_count_high": 18, "provider_count_source_note": "18 counted from the partners table on the docs index (Baseten, Cerebras, Cohere, DeepInfra, Fal AI, Featherless AI, Fireworks, Groq, HF Inference, Novita, Nscale, OVHcloud AI Endpoints, Public AI, Replicate, Scaleway, Together, WaveSpeedAI, Z.ai), of which 15 carry the \"Chat completion (LLM)\" mark ([Inference Providers](https://huggingface.co/docs/inference-providers/index), 2026-09-03). The low end is 14: that is how many distinct providers appeared in the live public router catalogue when it was paged on 2026-09-03 ([router /v1/models](https://router.huggingface.co/v1/models)) - fal-ai, replicate, wavespeed and hyperbolic serve non-chat modalities and so do not appear there.", "first_party_vs_partner": "Almost entirely third-party: Hugging Face describes itself as \"a unified proxy layer that sits between your application and multiple AI providers\". The one exception is `hf-inference`, Hugging Face's own backend, which appears as one of the 18 partners inside its own router and which \"as of July 2025 focuses mostly on CPU inference\" ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [HF Inference](https://huggingface.co/docs/inference-providers/providers/hf-inference), 2026-09-03).", "custom_endpoint_support": "n.a. for end users. There is no way to register your own base URL, self-hosted model or custom provider slug; becoming a provider is a partner-onboarding process Hugging Face runs, requiring a mapping API, a billing API and passing HF's validation suite ([Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "byok_mode": "both", "merchant_of_record": "Differs by mode. Routed Requests: Hugging Face is the merchant - \"billing is managed directly by Hugging Face\" and no provider account is required. Custom Provider Key: the upstream provider bills you (\"Billed by: Provider\"). Organisations can also route the bill through AWS Marketplace by linking the org to an AWS account ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub billing](https://huggingface.co/docs/hub/billing)).", "key_management_note": "Provider keys are stored per user (and per organisation) in Hub account settings; routed access uses a Hub user access token, and a fine-grained token can be scoped to the \"Make calls to Inference Providers\" permission. Organisation billing is targeted with the `X-HF-Bill-To` header, and admins can disable specific providers org-wide. No encryption-at-rest statement for stored provider keys was found ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [User access tokens](https://huggingface.co/docs/hub/security-tokens)).", "api_openai_chat_completions_note": "Yes: `POST https://router.huggingface.co/v1/chat/completions`, described as \"a drop-in compatible endpoint\", with `messages`, tools and streaming ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), 2026-09-03).", "api_anthropic_messages_note": "n.a. - no `/v1/messages` or Anthropic-format surface appears on the docs index, the chat-completion task page, the Responses API guide or the API reference index ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [API Reference](https://huggingface.co/docs/inference-providers/tasks/index)). The catalogue is open-weight-model focused and Anthropic is not one of the 18 partners.", "api_openai_responses_note": "Yes, marked beta: \"Use your existing OpenAI SDKs to access features like multi-provider routing, event streaming, structured outputs, and Remote MCP tools\" at the same `https://router.huggingface.co/v1` base URL, including reasoning-effort and image inputs ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), 2026-09-03).", "api_embeddings_note": "Yes, but not on the OpenAI surface: embeddings are the `feature-extraction` task, documented via `InferenceClient.feature_extraction` and served by HF Inference, Together and Scaleway; there is no `/v1/embeddings` path in the docs ([Feature Extraction](https://huggingface.co/docs/inference-providers/tasks/feature-extraction), [Inference Providers](https://huggingface.co/docs/inference-providers/index), 2026-09-03).", "api_images_note": "Yes: `text-to-image` is a documented task with its own reference page and provider column (fal-ai, Replicate, Together, Nscale, Novita and others), called through `InferenceClient.text_to_image`; image generation is not exposed on the OpenAI-compatible path ([Text to Image](https://huggingface.co/docs/inference-providers/tasks/text-to-image), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "api_audio_note": "Partial: speech-to-text is documented (`automatic-speech-recognition`, plus `audio-classification`) and the partners table has a \"Speech to text\" column, but no text-to-speech task page exists in the API reference index ([Automatic Speech Recognition](https://huggingface.co/docs/inference-providers/tasks/automatic-speech-recognition), [API Reference](https://huggingface.co/docs/inference-providers/tasks/index), [Inference Providers](https://huggingface.co/docs/inference-providers/index), 2026-09-03).", "api_batch_note": "n.a. - no batch or async bulk endpoint on any page fetched: no `/v1/batches`, no job-submission flow ([API Reference](https://huggingface.co/docs/inference-providers/tasks/index), [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Hub API](https://huggingface.co/docs/inference-providers/hub-api)).", "deployment_modes_note": "Hosted SaaS only. Every documented path resolves to `router.huggingface.co`; there is no self-host, VPC or hybrid option in the docs ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index)).", "byok_mode_note": "both: \"Hugging Face Routed Requests\" is the default (one HF token, HF bills you), and \"Custom Provider Key\" lets you store your own provider key so calls are billed by the provider while still being routed by Hugging Face ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), 2026-09-03).", "access_meta": { "qualified": [ "api_embeddings", "api_images", "api_audio", "api_openai_responses", "model_count" ], "model_count_kind": "enumerated", "provider_count_kind": "counted" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "n.a. - no user-settable request timeout. The only published timing thresholds are internal admission rules Hugging Face enforces on providers: models \"must respond in under 5 seconds\" time-to-first-token for conversational and text tasks and under 30 seconds for other tasks, checked by HF's own validation system ([Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion)).", "rel_retry_config": "not_documented", "rel_retry_note": "No retry count, backoff strategy or retry header is published. The documented failure behaviour is provider substitution, not a retry counter: with `provider=\"auto\"` \"requests are automatically routed to alternative providers if the primary provider is flagged as unavailable by our validation system\". Default retry count: n.a. Backoff: n.a. ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "Selected per request through the model id. `provider=\"auto\"` (the default, equivalent to the `:fastest` suffix) falls through to alternative providers when the primary is flagged unavailable; `:preferred` walks the user's configured provider order from account settings; `:cheapest` picks the lowest price per output token; `:groq`-style suffixes pin one provider and disable fallback. Order comes from HF policy or the user's settings list, not from a per-request array ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "Health checking exists but is entirely HF-operated and has no user knobs: every mapped model is tested every 6 hours, a failing provider is \"temporarily removed from the list of active providers\" and then retested hourly until it passes, and the suite enforces latency limits plus tool-calling and structured-output checks for LLMs ([Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), 2026-09-03).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "n.a. - no region selection or cross-region failover for routed inference is documented. The router is a single `router.huggingface.co` endpoint, and the Enterprise Hub \"Storage Regions\" feature governs repository data, not inference ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Enterprise Hub](https://huggingface.co/enterprise)).", "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": "No weighted or proportional load balancing is published. What exists is single-pick policy routing (`:fastest`, `:cheapest`, `:preferred`, explicit provider pin); weights are not user-settable and no traffic-splitting mechanism is described. The provider ordering shown in settings defaults to \"total requests routed by HF over the last 7 days\", which is a display order rather than a balancing policy ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider)).", "architecture_class": "vendor_saas", "architecture_note": "vendor_saas: a closed hosted proxy. Hugging Face states \"The Inference Providers API acts as a unified proxy layer that sits between your application and multiple AI providers\" and \"your requests go through Hugging Face's proxy infrastructure\". No runtime, language or source for the router is published; the open-source artefacts are the client libraries only ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "self_host_data_plane": "not_documented", "self_host_artifact_note": "No Docker image, Helm chart, binary or Terraform artefact for a customer-run router appears anywhere in the documentation: the docs tree covers index, pricing, security, hub integration, hub API, guides, integrations, providers and tasks, with no deployment section ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index)). `huggingface_hub` and `@huggingface/inference` are clients that call `router.huggingface.co`, not a data plane.", "latency_claims": [ { "url": "https://huggingface.co/docs/inference-providers/register-as-a-provider", "unit": "s", "scope": "capacity", "value": "under 5", "metric": "time-to-first-token admission threshold", "publisher": "vendor", "conditions": "Not a measured performance claim: this is the eligibility bar Hugging Face's validation system enforces on partner providers for conversational and text models (under 30 s for other tasks), tested every 6 hours. No percentile, payload, region or hardware is stated, and no gateway-overhead figure is published anywhere." } ], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported on both surfaces: the chat-completion page lists \"Streaming the output\" among what the API supports, and the Responses API guide documents server-sent event streaming with typed events. No documented caveat about stream cancellation or billing on aborted streams was found ([Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api)).", "guard_pii": "not_documented", "guard_pii_note": "n.a. - no guardrail, moderation, PII, prompt-injection or content-filtering feature appears anywhere in the Inference Providers documentation. Pages checked: [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration).", "guard_injection": "not_documented", "guard_injection_note": "n.a. - no guardrail, moderation, PII, prompt-injection or content-filtering feature appears anywhere in the Inference Providers documentation. Pages checked: [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration).", "guard_content": "not_documented", "guard_content_note": "n.a. - no guardrail, moderation, PII, prompt-injection or content-filtering feature appears anywhere in the Inference Providers documentation. Pages checked: [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration).", "guard_custom": "not_documented", "guard_custom_note": "n.a. - no guardrail, moderation, PII, prompt-injection or content-filtering feature appears anywhere in the Inference Providers documentation. Pages checked: [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Chat Completion](https://huggingface.co/docs/inference-providers/tasks/chat-completion), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration).", "guard_model_policy": "not_documented", "guard_model_policy_note": "No model allow/deny policy for callers. The closest documented control is organisation-level: admins \"can disable a set of Inference Providers\" from organisation settings, which restricts providers rather than models ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "guard_execution_location": "not_documented", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "n.a. - there are no guardrails, so no fail-open/fail-closed behaviour is described ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "log_content": "metadata_only", "log_content_note": "metadata_only by vendor statement - request and response bodies are explicitly not stored, and debugging logs are said to contain \"no user data or tokens\" ([Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "log_opt_out": "not_documented", "log_opt_out_note": "n.a. - no opt-out switch is documented, because there is no content logging to opt out of; the 30-day debugging logs are described without a customer control ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "tracing_support": "not_documented", "tracing_note": "n.a. - no OpenTelemetry, OTLP, trace export or request-trace view is documented. The observability surface is billing-shaped: a usage dashboard in account settings and a member-usage graph in organisation settings ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Hub billing](https://huggingface.co/docs/hub/billing), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "export_destinations": [], "export_destinations_note": "None documented. No OTLP endpoint, no log drain, no S3/Kafka/webhook destination appears on any fetched page ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Hub billing](https://huggingface.co/docs/hub/billing), [Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "feedback_capture": "no", "feedback_capture_note": "n.a. - no feedback, rating or annotation API. The Responses API guide documents generation features only, and there is no scoring surface in the docs ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "eval_hooks": "no", "eval_hooks_note": "No gateway-side evaluation or scoring. Hugging Face documents evaluation only as an external harness pointed at the router - a guide for running Inspect AI against Inference Providers models, and Inspect is listed under \"Evaluation Frameworks\" in the integrations table ([Evaluation with Inspect AI](https://huggingface.co/docs/inference-providers/guides/evaluation-inspect-ai), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index)).", "observability_requires_saas": "Yes by construction - the product is hosted-only, so the usage dashboard, per-provider breakdown and organisation usage graph exist solely inside Hugging Face account settings ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Hub billing](https://huggingface.co/docs/hub/billing)).", "log_retention_note": "30 days for debugging logs; no prompt or completion retention at any point ([Inference Providers security](https://huggingface.co/docs/inference-providers/security)).", "production_meta": { "sources": {}, "qualified": [ "guard_pii", "guard_injection", "guard_content", "guard_custom", "guard_model_policy", "latency_claims" ], "latency_prose": "No latency claim about the proxy itself exists. The `:fastest` routing policy is defined as picking \"the provider with the highest throughput (tokens per second)\", and `GET /v1/models` returns per-provider `first_token_latency_ms` and `throughput` values, but both describe upstream providers, not Hugging Face's added hop; the 5 s/30 s figures are admission thresholds ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Hub API](https://huggingface.co/docs/inference-providers/hub-api), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider)).", "multi_surface": { "rel_fallback": [ "per_request" ] }, "throughput_prose": "No RPS or tokens-per-second capacity claim for the router is published. The Hub-wide API rate limits on the Enterprise page (1,000-6,000 requests per 5 minutes by tier) are explicitly scoped to \"programmatic access to the Hub\" and were not used as gateway throughput ([Enterprise Hub](https://huggingface.co/enterprise), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "benchmark_independence": "Nothing to compare: Hugging Face publishes no benchmark of its own routing layer and makes no comparative claim against another gateway on any page fetched ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "The reliability surface is deliberately thin and almost all of it is operated by Hugging Face rather than configured by the caller: routing policy is a suffix on the model id, failover is automatic, health checking runs on a 6-hour cycle, and there is no timeout, retry, region or weight setting anywhere in the docs. Configuration-as-code does not exist for the same reason - the entire control surface is per-request strings plus user/organisation settings pages ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "guard_notes": "The single most important thing to know here: this router has no safety or policy layer whatsoever. There is no PII redaction, no injection detection, no content moderation, no custom-rule engine and no model policy for callers - and because Hugging Face does not retain request bodies, it also cannot offer retrospective content review. Teams with compliance obligations must place their own guardrails in front of, or behind, the router ([Inference Providers security](https://huggingface.co/docs/inference-providers/security), [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api)).", "fit_agent_features_note": "Agent-shaped work is well covered on the chat path: function/tool calling and structured outputs each have their own guide, the Responses API adds tool events, reasoning effort and remote MCP, and the integrations table includes six terminal coding agents (Pi, OpenCode, Codex, Claude Code, Hermes Agent, Roo Code) plus GitHub Copilot Chat in VS Code. HF's validation suite also tests LLMs for tool calling and structured output before a provider stays active ([Function Calling](https://huggingface.co/docs/inference-providers/guides/function-calling), [Structured Outputs](https://huggingface.co/docs/inference-providers/guides/structured-output), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Register as an Inference Provider](https://huggingface.co/docs/inference-providers/register-as-a-provider)).", "fit_byok_required": "no", "fit_byok_required_note": "No: the default \"Hugging Face Routed Requests\" mode needs only a Hugging Face token - \"No separate provider account is required\" - and every quickstart snippet uses `HF_TOKEN` alone ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call)).", "fit_client_sdk_langs": [ "Python", "JavaScript" ], "fit_client_sdks_note": "First-party clients are `huggingface_hub` (Python, `pip install huggingface_hub`) and `@huggingface/inference` (JS/TS, `npm install @huggingface/inference`). The docs additionally use the OpenAI Python and Node SDKs and plain `curl` against the router, and a `hf` CLI ships with `huggingface_hub` for listing warm models ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api)).", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. - authentication is a Hugging Face user access token (fine-grained tokens can be scoped to \"Make calls to Inference Providers\"). No AWS IAM, GCP service account, Azure AD or OIDC federation path for calling the router is documented ([User access tokens](https://huggingface.co/docs/hub/security-tokens), [Inference Providers](https://huggingface.co/docs/inference-providers/index), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. - no Cloudflare Workers, edge-runtime or Workers-AI binding guidance on any fetched page; the integrations catalogue is coding agents, LLM frameworks and IDEs ([Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [Building Your First AI App](https://huggingface.co/docs/inference-providers/guides/building-first-app)).", "fit_credit_card": "not_required", "fit_credit_card_note": "Not required for a first call: signed-in free accounts get $0.10 of included monthly inference credits. It becomes required for sustained use - free-tier users must purchase credits to go beyond the included amount, purchases are credit-card only, and Indian-issued cards are not accepted ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub billing](https://huggingface.co/docs/hub/billing)).", "fit_ecosystem_notes": "The distribution advantage is the Hub itself: inference widgets on model pages, the Inference Playground, and Data Studio AI text-to-SQL all run on Inference Providers and draw down the same credits, and Hub model search can be filtered by provider (`?inference_provider=fireworks-ai`). Outside the Hub, the documented integrations are coding agents and frameworks rather than infrastructure - including an HF Copilot Chat extension that puts these models inside VS Code (1.104.0+), whose page repeats the pricing posture as \"Transparent pricing: what the provider charges is what you pay\" - and Vercel ships an official `@ai-sdk/huggingface` provider that HF's own integrations table does not mention ([Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), [VS Code integration](https://huggingface.co/docs/inference-providers/integrations/vscode), [AI SDK Hugging Face provider](https://ai-sdk.dev/providers/ai-sdk-providers/huggingface)).", "fit_eval_prereqs": "Same as production: an HF account, a token with the Inference Providers permission, and credits; the docs' snippets need only `pip install openai` or `pip install huggingface_hub` ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call)).", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes - copy-pasteable and complete: `OpenAI(base_url=\"https://router.huggingface.co/v1\", api_key=os.getenv(\"HF_TOKEN\"))` then `client.responses.create(...)` or `client.chat.completions.create(...)`, with `curl` equivalents and TypeScript variants; provider pinning is shown as `moonshotai/Kimi-K2-Instruct-0905:groq` ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Inference Providers](https://huggingface.co/docs/inference-providers/index)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It is the gateway. Worth recording for compare pages: three products already rated in this catalogue - Fireworks AI, Groq and Together AI - appear as upstream partners inside it, and Hugging Face's own `hf-inference` backend is simultaneously the router and one of its 18 partners ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [HF Inference](https://huggingface.co/docs/inference-providers/providers/hf-inference)).", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. - nothing to run in a cluster, and no manifests, Helm chart or operator are mentioned; the service is a hosted endpoint ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index)).", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "MCP tools are first-class in the beta Responses API: pass a tool of `type: \"mcp\"` with `server_url`, optional `allowed_tools` and `require_approval`, and the router calls the remote MCP server as part of the response ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), 2026-09-03). Hugging Face is not itself an MCP gateway for arbitrary traffic.", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_cloudflare_workers", "fit_kubernetes", "fit_terraform" ] }, "fit_model_gate": "quota_gated", "fit_model_gate_note": "No approval or enablement step per model - any model with a warm provider mapping is callable immediately - but usage is credit-gated: each tier carries a fixed monthly credit allowance ($0.10 free, $2.00 PRO, $2.00 per Team/Enterprise seat) and free accounts must buy credits to continue past it. Organisation admins can additionally disable specific providers ([Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration)).", "fit_onboarding_notes": "The path of least resistance runs through the Hub rather than a dashboard: find a model with the \"Inference Providers\" filter (or `hf models ls --warm`), test it in the on-page widget or the Playground, then click \"View Code Snippets\". Two friction points are worth knowing: the credit allowance on a free account is $0.10 a month, and the token must be a fine-grained token carrying the \"Make calls to Inference Providers\" permission ([Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call), [Hub Integration](https://huggingface.co/docs/inference-providers/hub-integration), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "fit_openai_dropin": "partial", "fit_openai_dropin_note": "Partial, and the docs say so: the OpenAI-compatible endpoint is \"a drop-in compatible endpoint\" but is \"available for chat completion tasks only\" (plus the beta Responses API). Embeddings, image, video and speech tasks require `InferenceClient` or provider-native paths, so a wholesale OpenAI swap only covers chat workloads ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [API Reference](https://huggingface.co/docs/inference-providers/tasks/index), [Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api)).", "fit_prod_prereqs": "A Hugging Face account, a fine-grained token with the \"Make calls to Inference Providers\" permission stored in `HF_TOKEN`, and remaining inference credits. Nothing else - no cluster, no provider accounts, no card for the first calls ([Responses API (beta)](https://huggingface.co/docs/inference-providers/guides/responses-api), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain", "LlamaIndex", "CrewAI", "Haystack", "PydanticAI", "smolagents", "LiteLLM", "fast-agent", "Inspect" ], "fit_python_frameworks_note": "The integrations table lists LangChain, LlamaIndex, CrewAI, Haystack, PydanticAI, smolagents, LiteLLM, fast-agent and Inspect, each pointing at the framework's own \"Official docs\" rather than a Hugging Face-authored page; NeMo Data Designer and Vision Agents have HF-written getting-started pages ([Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), 2026-09-03).", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": "Three numbered steps in \"Your First Inference Provider Call\" (find a model on the Hub, try the widget, copy the code snippet), but the widget step is optional and the effective code path is shorter - set `HF_TOKEN`, install a client, call the router ([Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call)).", "fit_quickstart_url": "https://huggingface.co/docs/inference-providers/guides/first-api-call", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. - no Terraform provider, module or registry reference appears in the documentation, and there is no resource to provision beyond a token and credits ([Inference Providers](https://huggingface.co/docs/inference-providers/index), [Pricing and Billing](https://huggingface.co/docs/inference-providers/pricing)).", "fit_terraform_ref": null, "fit_third_party_documented": { "vercel_ai_sdk": "Vercel AI SDK (@ai-sdk/huggingface)" }, "fit_vendor_time_claim": "Vendor claim, verbatim: \"This guide will show you how to use a state-of-the-art model in under five minutes, with no infrastructure setup required.\" ([Your First Inference Provider Call](https://huggingface.co/docs/inference-providers/guides/first-api-call))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "An official AI SDK provider package exists, `@ai-sdk/huggingface`, whose default base URL is `https://router.huggingface.co/v1` and which exposes `.responses()` and `.languageModel()` factories plus a model-capability table for tool use and image input. It is documented by Vercel, not by Hugging Face - the AI SDK does not appear in HF's own integrations table ([AI SDK Hugging Face provider](https://ai-sdk.dev/providers/ai-sdk-providers/huggingface), [Integrations Overview](https://huggingface.co/docs/inference-providers/integrations/index), 2026-09-03).", "fit_vercel_ai_sdk_package": "@ai-sdk/huggingface" }, { "slug": "openrouter", "name": "OpenRouter", "company": "OpenRouter, Inc.", "tagline": "Hosted marketplace that routes one OpenAI-compatible API to models from many inference providers.", "category": "managed_marketplace", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2023, "hq_country": "United States", "url_home": "https://openrouter.ai", "url_docs": "https://openrouter.ai/docs/faq", "url_pricing": "https://openrouter.ai/pricing", "url_status": "https://status.openrouter.ai", "url_github": null, "url_changelog": "https://openrouter.ai/docs/changelog", "url_models_api": "https://openrouter.ai/api/v1/models", "github_stars": null, "model_count": 458, "model_count_note": "Count of entries returned by the models API on 2026-09-25. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 80, "token_markup_pct": 0, "credit_fee_pct": 5.5, "credit_fee_min_usd": 0.8, "byok_supported": true, "byok_fee_note": "Free up to $25,000/mo of list-price inference (pay-as-you-go) or $200,000/mo (Enterprise), then 5% of the equivalent OpenRouter cost; measured by list-price cost, not request count.", "seat_fee_usd_month": null, "free_tier": "Free tier with 25+ free models and 50 free-model requests/day; 1,000/day after purchasing at least $10 in credits.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Crypto (Coinbase) credit purchase fee", "amount": "5%" }, { "label": "Card credit purchase fee minimum", "amount": "$0.80 per purchase" } ], "self_host_cost_note": null, "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": false, "portability_note": "Aggregated usage export (CSV/PDF), not per-request logs.", "api_surfaces": [ "OpenAI Chat Completions", "OpenRouter Management API" ], "sdk_note": "OpenAI-compatible endpoint plus a TypeScript SDK and a Management API under /api/v1/keys for provisioning per-customer keys.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": false, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": false, "cap_pii_redaction": false, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": true, "cap_audio": null, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No vendor-published overhead figure. Third-party estimates diverge: Respan cites roughly 25-40 ms over direct provider calls, TrueFoundry cites about 40 ms typical production overhead, mcp.directory estimates a 100-150 ms hop, and an interleaved TTFT benchmark by LLM Gateway (2026-07-22) measured OpenRouter median cold TTFT of 1392 ms end-to-end rather than isolated gateway overhead.", "throughput_rps": null, "soc2": true, "hipaa_baa": null, "gdpr": true, "eu_residency": true, "eu_residency_note": "In-region routing in the EU and US for enterprise accounts on request via https://eu.openrouter.ai and https://us.openrouter.ai; prompts and completions stay in the selected region.", "zdr": true, "zdr_note": "Zero data retention is OpenRouter's own policy on all accounts with no plan gate, and can be enforced globally, per model group, per guardrail or per request.", "air_gapped": null, "best_for": "Teams that want the broadest possible model and provider catalog behind one OpenAI-compatible key with unified billing.", "strengths": [ "Largest published catalog: 500+ models, 80+ providers", "No token markup; provider list prices pass through", "Public models endpoint with per-model pricing", "ZDR routing, EU/US in-region routing, SOC 2 and GDPR posture", "Per-key credit limits with daily/weekly/monthly resets" ], "weaknesses": [ "5.5% credit-purchase fee (min $0.80) makes small top-ups expensive - $5 buys $5 of credit for $5.80", "BYOK documentation describes two different meters ($25k list-price cost vs 1M requests)", "No native content guardrails or PII redaction", "Managed only - no self-hosting or VPC option" ], "avoid_if": "You need self-hosting, air-gapped deployment, or a gateway that charges nothing on credit purchases.", "reviews": [ { "url": "https://techcrunch.com/2026/08/19/stripe-didnt-really-buy-openrouter-because-of-the-singularity/", "why": "TechCrunch's reporters analyse why a payments company paid $7B+ for an inference router, arguing the real prize is OpenRouter's position in metering and billing AI spend rather than model access.", "date": "2026-08-19", "type": "written", "title": "Stripe didn't really buy OpenRouter because of the 'singularity'", "publisher": "TechCrunch" }, { "url": "https://dev.to/goose_oss/how-openrouter-unlocked-our-workshop-strategy-5082", "why": "Rizel Scarlett explains how the Goose team used OpenRouter to hand out one key covering many models for live workshops, including the practical limits they hit.", "date": "2025-07-30", "type": "written", "title": "How OpenRouter Unlocked Our Workshop Strategy", "publisher": "DEV Community (Goose / Block open-source team)" }, { "url": "https://www.youtube.com/watch?v=fjd2hm6-qtM", "why": "Hands-on walkthrough on an established data/AI teaching channel: API-key creation, OpenAI-SDK compatibility, auto-router, model rankings, logs and credit tracking.", "date": "2026-03-25", "type": "video", "title": "What is OpenRouter | All about OpenRouter in 10 minutes", "publisher": "YouTube - codebasics" }, { "url": "https://www.youtube.com/watch?v=DBgINesX__k", "why": "Datadog's Ryan MacLean demos OpenRouter's request builder, curl/jq scripting, pricing and privacy settings, and a live chat app against multiple providers.", "date": "2025-09-11", "type": "video", "title": "OpenRouter: Unified LLM Interface and Model Orchestration", "publisher": "YouTube - Datadog Community" }, { "url": "https://news.ycombinator.com/item?id=49330139", "why": "Practitioner thread on the acquisition where commenters discuss shifting production load onto and away from OpenRouter, credits, and self-host/migration options.", "date": "2026-08-17", "type": "community", "title": "Stripe to Buy OpenRouter for $7B", "publisher": "Hacker News" }, { "url": "https://www.reddit.com/r/LocalLLaMA/comments/1jydnif/anyone_use_openrouter_in_production/", "why": "21-comment thread with real numbers: ~2M tokens/day in production and 2-3 overload errors per 100 requests on direct Anthropic versus OpenRouter fallbacks to Bedrock and Vertex.", "date": "2025-04-13", "type": "community", "title": "Anyone use openrouter in production?", "publisher": "Reddit - r/LocalLLaMA" } ], "sources": { "zdr": "https://openrouter.ai/enterprise", "gdpr": "https://openrouter.ai/enterprise", "soc2": "https://openrouter.ai/enterprise", "company": "https://en.wikipedia.org/wiki/OpenRouter", "license": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "sdk_note": "https://openrouter.ai/docs/features/provisioning-api-keys", "url_docs": "https://openrouter.ai/docs/faq", "zdr_note": "https://openrouter.ai/docs/faq", "cap_video": "https://openrouter.ai/models", "free_tier": "https://openrouter.ai/pricing", "deployment": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "extra_fees": "https://openrouter.ai/docs/faq", "hq_country": "https://en.wikipedia.org/wiki/OpenRouter", "url_status": "https://status.openrouter.ai", "zdr_status": "https://openrouter.ai/docs/guides/features/zdr", "acquired_by": "https://stripe.com/newsroom/news/stripe-agrees-to-acquire-openrouter", "cap_budgets": "https://openrouter.ai/pricing", "model_count": "https://openrouter.ai/api/v1/models", "url_pricing": "https://openrouter.ai/pricing", "api_surfaces": "https://openrouter.ai/docs/features/provisioning-api-keys", "cap_failover": "https://openrouter.ai/enterprise", "eu_residency": "https://openrouter.ai/docs/features/privacy-and-logging", "founded_year": "https://en.wikipedia.org/wiki/OpenRouter", "product_fate": "https://openrouter.ai/blog/announcements/openrouter-is-joining-stripe/", "regions_note": "https://openrouter.ai/enterprise", "acquired_date": "https://www.reuters.com/technology/payments-firm-stripe-buy-ai-developer-platform-openrouter-2026-08-19/", "byok_fee_note": "https://openrouter.ai/pricing", "cap_image_gen": "https://openrouter.ai/models", "url_changelog": "https://openrouter.ai/docs/changelog", "vendor_status": "https://openrouter.ai/blog/announcements/openrouter-is-joining-stripe/", "byok_supported": "https://openrouter.ai/docs/use-cases/byok", "cap_guardrails": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "certifications": "https://openrouter.ai/enterprise", "config_as_code": "https://openrouter.ai/docs/guides/features/presets", "credit_fee_pct": "https://openrouter.ai/docs/faq", "openrouter_faq": "https://openrouter.ai/docs/faq", "provider_count": "https://openrouter.ai/pricing", "retention_days": "https://openrouter.ai/docs/guides/privacy/data-collection", "url_models_api": "https://openrouter.ai/docs/faq", "cap_rate_limits": "https://openrouter.ai/pricing", "logs_by_default": "https://openrouter.ai/docs/guides/privacy/data-collection", "model_count_low": "https://openrouter.ai/api/v1/models", "openrouter_byok": "https://openrouter.ai/docs/use-cases/byok", "cap_cache_simple": "https://openrouter.ai/pricing", "cap_virtual_keys": "https://openrouter.ai/docs/features/provisioning-api-keys", "model_count_high": "https://openrouter.ai/api/v1/models", "model_count_note": "https://openrouter.ai/api/v1/models", "token_markup_pct": "https://openrouter.ai/docs/faq", "acquisition_terms": "https://www.reuters.com/technology/payments-firm-stripe-buy-ai-developer-platform-openrouter-2026-08-19/", "cap_observability": "https://openrouter.ai/enterprise", "cap_pii_redaction": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "eu_residency_note": "https://openrouter.ai/docs/features/privacy-and-logging", "last_release_date": "https://openrouter.ai/docs/changelog", "openai_compatible": "https://openrouter.ai/docs/faq", "openrouter_limits": "https://openrouter.ai/docs/api-reference/limits", "subprocessors_url": "https://openrouter.ai/authorized-sub-processors", "cap_cache_semantic": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "cap_load_balancing": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway", "credit_fee_min_usd": "https://openrouter.ai/docs/faq", "guardrails_location": "https://openrouter.ai/docs/guides/features/guardrails", "latency_source_note": "https://www.respan.ai/market-map/compare/openrouter-vs-vercel-ai-gateway", "requires_vendor_sdk": "https://openrouter.ai/docs/quickstart", "data_export_available": "https://openrouter.ai/docs/cookbook/administration/activity-export", "openrouter_enterprise": "https://openrouter.ai/enterprise", "prompt_transits_vendor": "https://openrouter.ai/privacy", "cap_conditional_routing": "https://openrouter.ai/pricing", "model_count_source_note": "https://openrouter.ai/api/v1/models", "trains_on_customer_data": "https://openrouter.ai/privacy", "openrouter_prompt_caching": "https://openrouter.ai/docs/features/prompt-caching" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_video": "2026-08-29", "free_tier": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-09-25", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "eu_residency": "2026-08-29", "founded_year": "2026-08-29", "byok_fee_note": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "url_models_api": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_low": "2026-09-25", "cap_cache_simple": "2026-08-29", "cap_virtual_keys": "2026-08-29", "model_count_high": "2026-09-25", "model_count_note": "2026-09-25", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "credit_fee_min_usd": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29", "model_count_source_note": "2026-09-25" }, "locked_fields": [], "published": true, "updated_at": "2026-09-25T16:55:57.770309+00:00", "vendor_status": "acquisition_pending", "acquired_by": "Stripe", "acquired_date": "2026-08-19", "acquisition_terms": "Terms undisclosed; Reuters reported slightly more than $8B", "product_fate": "Stripe signed a definitive agreement to acquire OpenRouter on 19 August 2026, subject to customary closing conditions and expected to close in the coming weeks. OpenRouter says it will continue to operate as it is: same mission, same name, same product, same roadmap, and that nothing about an existing integration changes.", "last_release_date": "2026-08-19", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Hosted router only; there is no self-hosted option. Your input is transmitted to whichever model provider you select.", "logs_by_default": "metadata_only", "logs_note": "Both logging toggles are off by default. Token counts, latency and similar metadata are stored; prompt and completion content is not.", "retention_days": 0, "retention_note": "Prompt retention is always opt-in. Metadata is retained with no published window.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "OpenRouter does not train on your inputs or outputs, and offers a 1% discount if you opt in to letting it. Downstream model providers are a separate question: some of them may train on what you send.", "subprocessors_url": "https://openrouter.ai/authorized-sub-processors", "regions_note": "US servers by default. Enterprise adds EU region locking and in-region routing. No self-hosted or customer-VPC option.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II, asserted on the enterprise page", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "described as GDPR compatible", "status": "claimed" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "vendor_saas", "guardrails_note": "PII detection and redaction run inside OpenRouter's gateway, so the data has already reached them before it is redacted.", "acquired_date_basis": "announced", "security_incidents": [], "pricing_model": "markup_only", "pricing_model_note": "Credit top-up fee (5.5% Stripe / 5% crypto) with a $0.80 minimum; 0% token markup; BYOK charged 5% above a monthly list-price allowance ($25k Pay-as-you-go / $200k Enterprise).", "min_commitment": "None stated. Unused credits may expire one year after purchase.", "overage_note": "No log/trace-retention or request-volume overage. Only after-the-fact charge is the BYOK 5% once the monthly list-price allowance is exceeded.", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Provider caching flows through unchanged. Read discounts vary by provider: 75%/50% for OpenAI, 90% for Anthropic/Alibaba/DeepSeek, 50% for Groq, ~80% for Z.AI, 75% for Gemini implicit. Cache-write premium: 0% for pre-GPT-5.6 OpenAI/Grok/Moonshot/Groq/Gemini; +25% for GPT-5.6+, Alibaba explicit and Anthropic 5-min; +100% for Anthropic 1-hour. OpenRouter itself adds 0% on cached traffic.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated as spend alerts; auto top-up threshold is the closest control.", "tier": null, "supported": null }, "virtual_keys": { "note": "API keys plus Management API for programmatic key management; 'virtual key' terminology not used.", "tier": null, "supported": true }, "rate_limiting": { "note": "Platform-governed, not user-configurable. 20 req/min and 50 or 1,000 req/day on free models; global capacity governance across accounts.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Per-key credit limits enforced pre-request via limit/limit_remaining.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Per model, provider and API key on the Activity page; session_id grouping across turns. Per-user/team/customer not stated.", "cost_export": "API only (/api/v1/key, credits API, /api/v1/generation). CSV, webhook, S3 and warehouse export not stated.", "product_kind_note": "Routing proxy in front of many providers. The vendor's FAQ says \"OpenRouter is a proxy that sends your requests to the model provider for it to be completed\" and \"OpenRouter is a drop-in replacement for OpenAI\" ([OpenRouter FAQ](https://openrouter.ai/docs/faq)); the quickstart calls it \"access to hundreds of AI models through a single API endpoint\" that \"handles fallbacks automatically\" ([OpenRouter Quickstart](https://openrouter.ai/docs/quickstart)).", "deployment_modes": [ "saas" ], "deployment_note": "Nothing to install: customers call `https://openrouter.ai/api/v1` ([Quickstart](https://openrouter.ai/docs/quickstart)). The \"self-host\"-like lever is BYOK/own capacity: \"Use your AWS/GCP/Azure credits and get the best throughput combining your limits plus ours\" ([Enterprise](https://openrouter.ai/enterprise)). Tier gating exists for BYOK volume, not deployment: pay-as-you-go includes $25,000/month of BYOK inference free, Enterprise $200,000 ([FAQ](https://openrouter.ai/docs/faq)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap: \"any SDKs that support OpenAI by default also support OpenRouter\" ([FAQ](https://openrouter.ai/docs/faq)); first-party SDKs (`@openrouter/sdk`, Python `openrouter`, Agent SDK) are optional ([Quickstart](https://openrouter.ai/docs/quickstart)). Schemas are \"very similar to the OpenAI Chat API\" but not identical, unsupported params are silently ignored, and streaming is SSE with an occasional comment payload to ignore; usage arrives once in the final chunk ([API overview](https://openrouter.ai/docs/api_reference/overview)). Batch is text-only — image/audio/video/file parts are rejected ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).", "model_count_low": 458, "model_count_high": 458, "model_count_source_note": "Count of entries returned by the models API on 2026-09-25. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": 83, "provider_count_high": 83, "provider_count_source_note": "\"83 of 83 providers\" listed on the providers directory, page undated ([OpenRouter Providers](https://openrouter.ai/providers)).", "first_party_vs_partner": "Routed to third-party providers: \"We pass through the pricing of the underlying providers; there is no markup on inference pricing\" and OpenRouter \"sends your requests to the model provider\" ([FAQ](https://openrouter.ai/docs/faq)); routing falls back \"to other providers or GPUs\" ([API overview](https://openrouter.ai/docs/api_reference/overview)). No statement of OpenRouter-owned inference hardware on any page fetched.", "custom_endpoint_support": "Partially: not documented for arbitrary vLLM/Ollama URLs, but BYOK supports customer-specific Azure deployment URLs — per-deployment config takes an explicit `endpoint_url` such as `https://example-project.openai.azure.com/openai/deployments/...` plus `api_key`, `model_id`, `model_slug` ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)). Generic private-URL registration: not documented.", "byok_mode": "both", "merchant_of_record": "Differs by mode. Credits mode: OpenRouter invoices — \"Credits are simply deposits on OpenRouter that you use for LLM inference… we deduct the request cost from your credits\" ([FAQ](https://openrouter.ai/docs/faq)). BYOK mode: the upstream provider bills — \"the provider bills you directly for inference and OpenRouter charges only the BYOK fee\" (5% of list, deducted from credits, above the plan allowance) ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart), [BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).", "key_management_note": "\"Your provider keys are securely encrypted and used for all requests routed through the specified provider\"; keys live in workspace BYOK settings, can be Prioritized or Fallback, and can be filtered by `allowed_models`, `allowed_api_key_hashes`, `allowed_user_ids` (100 entries max each) ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)). Content visibility: \"OpenRouter does not store your prompts or responses, unless you opt in\"; metadata (tokens, latency) is always stored and \"does not include the content of your prompts or responses\" ([Data collection](https://openrouter.ai/docs/guides/privacy/data-collection), [FAQ](https://openrouter.ai/docs/faq)). Batch inputs/results are stored as JSONL in Google Cloud Storage and deleted after 30 days ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).", "api_openai_chat_completions_note": "yes: `POST /api/v1/chat/completions`, \"implements the OpenAI API specification for /completions and /chat/completions endpoints\" ([API overview](https://openrouter.ai/docs/api_reference/overview), [FAQ](https://openrouter.ai/docs/faq)).", "api_anthropic_messages_note": "yes: native Anthropic Messages surface documented as `POST /messages` on the `https://openrouter.ai/api/v1` server ([Create a message](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-a-message)); also a valid Batch API shape as `/v1/messages` ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).", "api_openai_responses_note": "yes: \"OpenAI-compatible Responses API endpoints\" are listed in the API reference navigation ([API reference](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-a-message)) and `/v1/responses` is an accepted Batch endpoint shape ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).", "api_embeddings_note": "yes: `/v1/embeddings` is a documented batch shape (\"Embeddings are rolling out on providers that support them\") and models can be filtered with `/api/v1/models?output_modalities=embeddings` ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart), [Models](https://openrouter.ai/docs/guides/overview/models)).", "api_images_note": "yes: `POST /api/v1/images` ([Image generation](https://openrouter.ai/docs/guides/overview/multimodal/image-generation)).", "api_audio_note": "yes: STT and TTS endpoint categories are listed in the API reference, and audio input/output runs through `/api/v1/chat/completions` ([List all models](https://openrouter.ai/docs/api/api-reference/models/list-all-models-and-their-properties), [Audio](https://openrouter.ai/docs/guides/overview/multimodal/audio)).", "api_batch_note": "yes: `POST https://openrouter.ai/api/beta/batches`, 24h completion window, text-only ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).", "deployment_modes_note": "Hosted SaaS only (documented). The enterprise page frames the model as \"Bring your own capacity and use OpenRouter as SaaS\" ([OpenRouter Enterprise](https://openrouter.ai/enterprise)); no self-host, VPC, on-prem or air-gapped mode appears on the quickstart, FAQ, API overview, BYOK or enterprise pages fetched ([FAQ](https://openrouter.ai/docs/faq), [BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).", "byok_mode_note": "both: \"OpenRouter supports both OpenRouter credits and the option to bring your own provider keys (BYOK)\" ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).", "access_meta": { "qualified": [], "model_count_kind": "enumerated", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "No request-timeout header or key appears on the fetched routing/performance docs. The closest published control is a routing preference, not a timeout: `preferred_max_latency` (with `preferred_min_throughput`), evaluated against provider p50/p90/p99 over a rolling 5-minute window ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "rel_retry_config": "not_documented", "rel_retry_note": "No retry-count key or backoff strategy is published; the documented failure behavior is provider fallback, not a retry counter. Default retry count: `n.a.` Backoff: `n.a.` ([OpenRouter model fallbacks](https://openrouter.ai/docs/guides/routing/model-fallbacks)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED. `provider.order` sets an explicit ordered provider list and `allow_fallbacks` (default `true`) controls whether other providers may be used ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)). Model-level fallback uses the `models` array, tried in order; via the Anthropic-compatible surface the key is `fallbacks`, limited to 3 entries ([OpenRouter model fallbacks](https://openrouter.ai/docs/guides/routing/model-fallbacks)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "It is automatic unhealthy-provider ejection, not a configurable health check or circuit breaker: OpenRouter tracks provider errors and removes a failing provider from rotation using a 30-second outage window ([OpenRouter reliability and failover](https://openrouter.ai/blog/insights/reliability-failover/)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "The fetched pages describe edge execution on Cloudflare Workers but no configurable cross-region failover ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "rel_load_balancing_config": "per_request", "rel_load_balancing_note": "Weights are NOT user-settable: OpenRouter load-balances with price-inverse-square weighting internally; the user-facing controls are `order`, `only`, `sort` and the latency/throughput preferences ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)).", "architecture_class": "edge_worker", "architecture_note": "Stated runtime: \"Edge computing using Cloudflare Workers\" ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "self_host_data_plane": "not_documented", "self_host_artifact_note": "No Docker image, Helm chart, binary or npm package for a self-hosted data plane appears on the fetched pages ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported. Documented caveat: on stream cancellation some upstreams still bill — Bedrock, Groq, Google and Mistral are named as continuing to charge for cancelled streams ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "guard_pii": "sync_block", "guard_pii_note": "Guardrails include a DLP scanner with 7 built-in sensitive-data types, using regex plus Presidio NLP for names/addresses; per-type action is Flag, Redact, or Block, and Block returns a 403 before the model is called, so it both redacts and blocks depending on configuration ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails), [OpenRouter guardrails announcement](https://openrouter.ai/blog/announcements/guardrails/))", "guard_injection": "sync_block", "guard_injection_note": "Prompt-injection detection is regex/pattern-based (30+ patterns mapped to OWASP LLM risks), not a model classifier, and shares the Flag/Redact/Block actions ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))", "guard_content": "not_documented", "guard_content_note": "The Guardrails reference documents DLP and prompt-injection scanners; no toxicity/moderation category is described ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))", "guard_custom": "sync_block", "guard_custom_note": "Policies take the form of configurable scanner rules with per-type actions; regex patterns underpin the DLP and injection scanners ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))", "guard_model_policy": "yes", "guard_model_policy_note": "Model allowlists and provider allowlists are enforced and intersected on the request path ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))", "guard_execution_location": "vendor_saas", "guard_third_party": [ "Microsoft Presidio" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "No fail-open/fail-closed statement on the guardrails pages ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))", "log_content": "metadata_only", "log_content_note": "Prompt and completion text is not stored unless input/output logging is explicitly enabled ([Input/output logging](https://openrouter.ai/docs/guides/features/input-output-logging), [Data collection](https://openrouter.ai/docs/guides/privacy/data-collection))", "log_opt_out": "yes", "log_opt_out_note": "Body logging is opt-in, not opt-out: leaving I/O logging off keeps usage metrics while storing no prompt/completion text ([Input/output logging](https://openrouter.ai/docs/guides/features/input-output-logging))", "tracing_support": "not_documented", "tracing_note": "`n.a.` — No OpenTelemetry or agent-trace support documented on the fetched pages ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "export_destinations": [ "CSV export" ], "export_destinations_note": "Activity export as CSV and PDF from the activity/administration API; no log/metric/trace streaming destinations documented ([Activity export cookbook](https://openrouter.ai/docs/cookbook/administration/activity-export))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — No feedback or score endpoint documented ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` — No eval or online-scoring product documented ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "observability_requires_saas": null, "log_retention_note": "Opt-in prompt/completion logs are retained for at least 3 months ([Data collection](https://openrouter.ai/docs/guides/privacy/data-collection))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` No numeric latency figure is published on the fetched pages. Only the measurement machinery is documented: provider p50/p90/p99 latency over a rolling 5-minute window used for `preferred_max_latency` ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "multi_surface": {}, "throughput_prose": "`n.a.` Only `preferred_min_throughput` as a routing threshold; no measured RPS or tokens/sec figure ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).", "benchmark_independence": "No figures published, so nothing to attribute ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "not_documented", "guard_custom_default": "choose_at_setup", "rel_notes": "Reliability here is largely vendor-operated rather than user-configured: you choose ordering and preferences, the platform does ejection and weighting. `allow_fallbacks: false` is the switch that makes routing deterministic at the cost of resilience ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)).", "guard_notes": "Guardrails are enforced alongside Zero-Data-Retention routing: ZDR enforcement is applied on the same request path as the model/provider allowlists ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails)).", "fit_agent_features_note": "The `@openrouter/agent` Agent SDK provides multi-turn conversation loops, tool execution, stop conditions, and state management through `callModel`. It sends prompts, handles model tool calls, executes tools, feeds results back, and returns the final response; sessions/threads and A2A are not documented. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))", "fit_byok_required": "no", "fit_byok_required_note": "No. Only an OpenRouter key is used in the examples ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart)); the FAQ states OpenRouter \"provides a unified API to access all the major LLM models\" with no BYO-key requirement ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "fit_client_sdk_langs": [ "TypeScript", "Python", "Go" ], "fit_client_sdks_note": "Official client SDKs include the TypeScript package `@openrouter/sdk`, the Python package `openrouter` with the `OpenRouter` class, and the Go package `go-sdk` at `github.com/OpenRouterTeam/go-sdk`. The Python SDK supports synchronous and asynchronous clients. ([OpenRouter client SDK overview](https://openrouter.ai/docs/client-sdks/overview))", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Partially: OpenRouter's BYOK docs accept AWS credentials (`accessKeyId`, `secretAccessKey`, `region`) for an AWS IAM user/role and a Google Cloud service-account key JSON for Vertex; Azure is configured with an `api_key`. Calls to OpenRouter still authenticate with an OpenRouter API key, so this is upstream credential reuse rather than IAM-based access to the gateway. ([OpenRouter BYOK authentication docs](https://openrouter.ai/docs/guides/overview/auth/byok))", "fit_cloudflare_workers": "vendor_runtime_only", "fit_cloudflare_workers_note": "OpenRouter states that it uses edge computing with Cloudflare Workers to stay close to applications. This documents OpenRouter’s own infrastructure use, not a user-facing Cloudflare Workers integration. ([OpenRouter latency and performance guide](https://openrouter.ai/docs/guides/best-practices/latency-and-performance))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated as required; a free allowance exists: \"All new users receive a small free allowance to test out OpenRouter,\" and free models have \"low rate limits (50 requests per day total)\" ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "fit_ecosystem_notes": "OpenRouter provides access to hundreds of models through a unified API, with automatic fallbacks and provider/model routing; its SDK documentation describes access to 400+ models. OpenRouter also documents remote MCP hosting and uses Cloudflare Workers, edge caching, and provider routing in its own service infrastructure; self-hosting and single-cloud requirements are not documented. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))", "fit_eval_prereqs": "An OpenRouter API key (examples use the `` placeholder). No account, credit card, cloud account, cluster, or separate provider key is stated as a prerequisite on the quickstart ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes. `POST https://openrouter.ai/api/v1/chat/completions` with a bearer OpenRouter key and body `{\"model\": \"~openai/gpt-latest\", \"messages\": [{\"role\":\"user\",\"content\":\"What is the meaning of life?\"}]}`, shown in Python, TypeScript `fetch`, and shell `curl` ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))", "fit_gateway_platform": "third_party_documented", "fit_gateway_platform_note": "OpenRouter itself documents no gateway plugin, but Apache APISIX's `ai-proxy` / `ai-proxy-multi` plugins list `openrouter` as a supported provider, proxying to `openrouter.ai` via the OpenAI-compatible driver (third-party gateway documentation, not OpenRouter's). ([Apache APISIX ai-proxy plugin docs](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. (not documented)", "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "OpenRouter documents remote MCP support through `@openrouter/mcp`, connecting to MCP servers over Streamable HTTP or SSE and exposing their tools as first-class `callModel` tools. Its hosted MCP server is `https://mcp.openrouter.ai/mcp`; the quickstart states that it requires no installation and uses OAuth login. ([OpenRouter Agent SDK MCP tools](https://openrouter.ai/docs/agent-sdk/call-model/mcp-tools))", "fit_meta": { "not_documented_fields": [ "fit_kubernetes" ] }, "fit_model_gate": "none", "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota, or waitlist gate is mentioned; \"OpenRouter gives you access to hundreds of AI models through a single API endpoint\" ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart)); the FAQ also states no gate ([OpenRouter FAQ](https://openrouter.ai/docs/faq))", "fit_onboarding_notes": "Optional client SDKs exist (`npm install @openrouter/sdk`, `pip install openrouter`) and an Agent SDK (`npm install @openrouter/agent`), but the plain REST/OpenAI-SDK path needs no new dependency. OpenRouter headers `HTTP-Referer` and `X-OpenRouter-Title` are explicitly \"Optional.\" A remote MCP server at `https://mcp.openrouter.ai/mcp` is offered with \"nothing to install\" ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. OpenRouter documents OpenAI-compatible usage with the exact base URL `https://openrouter.ai/api/v1`; its Responses API endpoint is `https://openrouter.ai/api/v1/responses` and requires an OpenRouter API key. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))", "fit_prod_prereqs": "n.a. (not documented) — no database, Redis, cluster, load balancer, or control-plane requirement appears on the quickstart ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "LangChain integration is documented for Python and JavaScript applications, and LlamaIndex integration is documented for Python and TypeScript RAG applications. LlamaIndex also documents the `llama-index-llms-openrouter` package and `llama_index.llms.openrouter.OpenRouter` class. ([OpenRouter frameworks and integrations overview](https://openrouter.ai/docs/guides/community/frameworks-and-integrations-overview))", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "The quickstart is prose plus a code sample with no numbered procedure, so there is no step count to report.", "fit_quickstart_url": "https://openrouter.ai/docs/quickstart", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "community_provider", "fit_terraform_note": "Terraform Registry lists an OpenRouter provider at `cloudopsworks/openrouter` and another at `mcmarkj/openrouter`. The documented versions are `0.3.0` and `0.0.3`, respectively, and both support Terraform `0.13+` per the registry pages. ([Terraform Registry cloudopsworks/openrouter provider](https://registry.terraform.io/providers/cloudopsworks/openrouter/latest))", "fit_terraform_ref": "cloudopsworks/openrouter 0.3.0 and mcmarkj/openrouter 0.0.3 on the Registry — neither is published by OpenRouter", "fit_third_party_documented": { "gateway_platform": "Apache APISIX" }, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "OpenRouter documents the official provider package `@openrouter/ai-sdk-provider` for the Vercel AI SDK, including use in a Next.js application. ([OpenRouter Vercel AI SDK provider](https://github.com/OpenRouterTeam/ai-sdk-provider))", "fit_vercel_ai_sdk_package": "@openrouter/ai-sdk-provider" }, { "slug": "requesty", "name": "Requesty", "company": "Requesty", "tagline": "Hosted router with a flat 5% fee on inference, EU data residency and enterprise governance controls.", "category": "managed_marketplace", "deployment": "managed_only", "license": null, "founded_year": null, "hq_country": "United Kingdom", "url_home": "https://requesty.ai/", "url_docs": "https://docs.requesty.ai/", "url_pricing": "https://www.requesty.ai/pricing", "url_status": "https://status.requesty.ai", "url_github": null, "url_changelog": "https://docs.requesty.ai/changelog", "url_models_api": "https://router.requesty.ai/v1/models", "github_stars": null, "model_count": 706, "model_count_note": "Count of entries returned by the models API on 2026-09-14. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 20, "token_markup_pct": 5, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "0% markup on your own provider contracts when using BYOK; keeps negotiated provider and committed-use pricing.", "seat_fee_usd_month": 0, "free_tier": "Free plan: all free models, 200 requests/day, routing, caching, fallbacks, EU residency, no credit card.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": null, "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": false, "portability_note": "Analytics export is CSV/PDF summaries.", "api_surfaces": [ "OpenAI Chat Completions", "Anthropic Messages", "Embeddings", "Audio", "Images" ], "sdk_note": "Base URL https://router.requesty.ai/v1 works with the OpenAI SDK; documented integrations for LangChain, Vercel AI SDK, LlamaIndex, Haystack, Pydantic AI and Claude Code.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 16, "latency_source_note": "Vendor benchmark (Requesty blog, 2026-04-28) claims roughly 16 ms of added overhead including fallback, load balancing, caching and observability, versus about 55 ms for managed OpenRouter and 124 ms for self-hosted LiteLLM; no methodology is published. A third-party comparison repeats an approximate 8 ms P50 figure, also without methodology.", "throughput_rps": null, "soc2": false, "hipaa_baa": null, "gdpr": true, "eu_residency": true, "eu_residency_note": "EU traffic pinned to AWS eu-central-1 (Frankfurt) via https://router.eu.requesty.ai/v1, with request processing, logging, caching and analytics kept in region; US data in Virginia and APAC in Singapore.", "zdr": true, "zdr_note": "Organisation-wide zero data retention is enforced server-side but must be requested in writing, and it also disables Requesty's own caching.", "air_gapped": false, "best_for": "European teams that want one router with EU data residency, PII scrubbing and a predictable flat 5% fee.", "strengths": [ "Flat 5% fee with no seat fees, no minimum spend and no top-up fee", "0% markup when bringing your own provider keys", "EU-only routing endpoint with in-region logging, caching and analytics", "Semantic caching, PII redaction, guardrails and budget controls", "Public models endpoint returning live pricing" ], "weaknesses": [ "5% markup is charged on every token, unlike zero-markup gateways", "Its own pages state 600+, 400+ and 300+ models inconsistently", "SOC 2 Type II is only in progress, not attested", "Governance features (RBAC, PII detection, guardrails, SSO) are gated to Enterprise", "Managed SaaS only: no self-hosting, VPC or air-gapped option" ], "avoid_if": "You need a zero-markup gateway, a completed SOC 2 attestation today, or a self-hosted deployment.", "reviews": [ { "url": "https://www.datacamp.com/tutorial/requesty-tutorial", "why": "Independent step-by-step tutorial that builds against Requesty's OpenAI-compatible endpoint and compares its routing, caching and cost controls with OpenRouter and LiteLLM.", "date": "2025-12-01", "type": "written", "title": "Requesty Tutorial: The Unified LLM Gateway", "publisher": "DataCamp" }, { "url": "https://www.youtube.com/watch?v=aXw0ew7GEvw", "why": "Vasilij Nevlev runs the same prompt through both gateways across four rounds, testing auto-routing, caching, budget caps, PII scrubbing and EU-only model filtering.", "date": "2026-07-17", "type": "video", "title": "OpenRouter vs Requesty: Which AI Gateway You Should Run?", "publisher": "YouTube - AiGentic Lab" } ], "sources": { "zdr": "https://requesty.ai/", "gdpr": "https://requesty.ai/", "soc2": "https://requesty.ai/", "cap_mcp": "https://www.requesty.ai/pricing", "sdk_note": "https://docs.requesty.ai/", "url_docs": "https://docs.requesty.ai/", "zdr_note": "https://requesty.ai/", "cap_audio": "https://docs.requesty.ai/", "free_tier": "https://www.requesty.ai/pricing", "air_gapped": "https://www.truefoundry.com/blog/requesty-vs-openrouter", "deployment": "https://www.truefoundry.com/blog/requesty-vs-openrouter", "hq_country": "https://www.businessinsider.com/pitch-deck-ai-startup-requesty-seed-2025-9", "url_status": "https://status.requesty.ai", "zdr_status": "https://www.requesty.ai/security", "cap_budgets": "https://requesty.ai/", "model_count": "https://router.requesty.ai/v1/models", "url_pricing": "https://www.requesty.ai/pricing", "api_surfaces": "https://docs.requesty.ai/", "cap_failover": "https://requesty.ai/", "eu_residency": "https://requesty.ai/", "product_fate": "https://docs.requesty.ai/changelog", "regions_note": "https://www.requesty.ai/security", "byok_fee_note": "https://requesty.ai/", "cap_image_gen": "https://docs.requesty.ai/", "url_changelog": "https://docs.requesty.ai/changelog", "vendor_status": "https://www.requesty.ai/blog/requesty-raises-3m", "byok_supported": "https://requesty.ai/", "cap_embeddings": "https://docs.requesty.ai/", "cap_guardrails": "https://requesty.ai/", "certifications": "https://www.requesty.ai/security", "config_as_code": "https://docs.requesty.ai/features/latency-routing", "provider_count": "https://www.requesty.ai/pricing", "retention_days": "https://www.requesty.ai/security", "sla_uptime_pct": "https://www.requesty.ai/enterprise", "url_models_api": "https://docs.requesty.ai/features/supported-models", "cap_rate_limits": "https://requesty.ai/", "logs_by_default": "https://www.requesty.ai/security", "model_count_low": "https://router.requesty.ai/v1/models", "cap_cache_simple": "https://docs.requesty.ai/", "model_count_high": "https://router.requesty.ai/v1/models", "model_count_note": "https://router.requesty.ai/v1/models", "requesty_pricing": "https://www.requesty.ai/pricing", "token_markup_pct": "https://www.requesty.ai/pricing", "cap_observability": "https://requesty.ai/", "cap_pii_redaction": "https://requesty.ai/", "eu_residency_note": "https://www.requesty.ai/blog/agentic-routing-benchmarked", "last_release_date": "https://docs.requesty.ai/changelog", "openai_compatible": "https://docs.requesty.ai/", "subprocessors_url": "https://www.requesty.ai/privacy/subprocessors", "cap_cache_semantic": "https://requesty.ai/", "cap_load_balancing": "https://docs.requesty.ai/", "seat_fee_usd_month": "https://requesty.ai/", "guardrails_location": "https://www.requesty.ai/security", "latency_overhead_ms": "https://www.requesty.ai/blog/agentic-routing-benchmarked", "latency_source_note": "https://www.requesty.ai/blog/agentic-routing-benchmarked", "requesty_api_limits": "https://docs.requesty.ai/features/api-limits", "requires_vendor_sdk": "https://docs.requesty.ai/frameworks/openai", "cap_prompt_management": "https://requesty.ai/", "data_export_available": "https://docs.requesty.ai/features/analytics-exports", "requesty_auto_caching": "https://docs.requesty.ai/features/auto-caching", "prompt_transits_vendor": "https://www.requesty.ai/security", "cap_conditional_routing": "https://www.requesty.ai/pricing", "model_count_source_note": "https://router.requesty.ai/v1/models", "trains_on_customer_data": "https://www.requesty.ai/privacy" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "soc2": "2026-08-29", "cap_mcp": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "free_tier": "2026-08-29", "air_gapped": "2026-08-29", "deployment": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-09-14", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "eu_residency": "2026-08-29", "byok_fee_note": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "provider_count": "2026-08-29", "url_models_api": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_low": "2026-09-14", "cap_cache_simple": "2026-08-29", "model_count_high": "2026-09-14", "model_count_note": "2026-09-14", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29", "model_count_source_note": "2026-09-14" }, "locked_fields": [], "published": true, "updated_at": "2026-09-14T20:21:32.956859+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed and sold to new customers, with three features shipped on 26 August 2026. Raised a $3M seed led by 20VC.", "last_release_date": "2026-08-26", "sla_uptime_pct": 99.99, "sla_url": "https://www.requesty.ai/enterprise", "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Hosted gateway; all traffic flows through Requesty's infrastructure in Frankfurt.", "logs_by_default": "full_content", "logs_note": "On self-serve plans prompt and output logging is on by default, encrypted and held in the EU. You can disable it per API key.", "retention_days": 30, "retention_note": "Thirty days on self-serve, reducible to zero by disabling logging per key.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Absolute on any paid plan. The free plan is different: models labelled Training Permitted retain prompts and outputs and train on them, and those models are free-plan only.", "subprocessors_url": "https://www.requesty.ai/privacy/subprocessors", "regions_note": "EU only — all infrastructure in Frankfurt, with an EU endpoint. No US region and no customer-VPC option.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II expected Q3 2026", "status": "in_progress" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA on request; EU residency in Frankfurt", "status": "available" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Requesty runs its own PII detection model before the prompt reaches the model provider — inside its gateway, not yours.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "markup_only", "pricing_model_note": "Flat 5% per-token markup, no seats, no minimum spend. Free tier caps free-model use at 200 req/day; Enterprise is custom.", "min_commitment": "None — 'no minimum spend'.", "overage_note": "No retention, request-volume or export overage stated on the pricing page.", "cache_support": "passthrough", "cache_read_discount_pct": 90, "cache_write_premium_pct": null, "cache_pricing_note": "Passthrough of provider caching with automatic breakpoint insertion; Requesty explicitly does not store responses server-side. Vendor says cache hits are billed 'at a fraction of the normal input token cost (up to 90% savings)'; cache writes 'have extra costs' for some providers, no number published. A per-request `false` toggle lets you avoid write premiums.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Slack + webhook alerts fire as limits are approached (reported after spend).", "tier": null, "supported": true }, "virtual_keys": { "note": "Org admins issue keys; policies scope to a key.", "tier": null, "supported": true }, "rate_limiting": { "note": "Requesty imposes no rate limits of its own; upstream 429s are handled by routing retries.", "tier": null, "supported": false }, "spend_webhooks": { "note": "Webhook alerts on approaching spend limits.", "tier": null, "supported": true }, "model_allowlists": { "note": "'Approved models & policies' is an Enterprise bullet; 'Approved models whitelist' in the comparison table.", "tier": "enterprise", "supported": true }, "budget_caps_per_key": { "note": "Each API key has its own monthly spend cap, also per service account; enforced pre-request.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "'Groups & team budgets' listed; 'Teams, groups & spend controls' is an Enterprise bullet. Tier assignment ambiguous.", "tier": "enterprise", "supported": true } }, "cost_attribution": "Per API key and per service account (each carries its own cap and usage). User/team/tag/customer splits listed as feature names without confirmation.", "cost_export": "'Data exports' is listed in the comparison table; a Management API can be queried programmatically. Specific formats (CSV/S3/warehouse) not stated.", "product_kind_note": "Routing proxy: \"One API. 300+ models. OpenAI-compatible\" and the docs call it a \"pass-through gateway\" that \"normalizes the schema across models and providers through a single API\" ([Requesty quickstart](https://docs.requesty.ai/), [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)). Marketed as an \"Enterprise AI Gateway\" ([Requesty Enterprise](https://www.requesty.ai/enterprise)).", "deployment_modes": [ "saas", "self_host", "air_gapped" ], "deployment_note": "SaaS endpoints `https://router.requesty.ai/v1` and `https://api.requesty.ai/v1` ([quickstart](https://docs.requesty.ai/), [Enterprise](https://www.requesty.ai/enterprise)); the Helm guide covers a Kubernetes chart with `/health` and `/ready` probes and air-gapped installs ([Helm guide](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369)). Enterprise tier gates SSO, RBAC, custom SLAs and EU residency ([Enterprise](https://www.requesty.ai/enterprise)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap: \"If you're already using the OpenAI SDK, point it at Requesty and you're done. No SDK changes, no new client to learn\"; LangChain, Vercel AI SDK, LlamaIndex, Haystack and Pydantic AI work out of the box ([quickstart](https://docs.requesty.ai/)). Streaming caveat: you must pass `stream_options: {\"include_usage\": true}` to get a final usage chunk ([quickstart](https://docs.requesty.ai/)). Model IDs are provider-prefixed (e.g. `openai/gpt-4o`) ([quickstart](https://docs.requesty.ai/)).", "model_count_low": 706, "model_count_high": 706, "model_count_source_note": "Count of entries returned by the models API on 2026-09-14. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "n.a. as a number. The BYOK page lists 6 connectable providers (OpenAI, Anthropic, Google AI Studio, Vertex AI, Mistral AI, xAI) and says \"More providers will be added over time\"; the models page names OpenAI, Anthropic, Google, DeepSeek, xAI, Mistral \"and more\". Both pages undated ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys), [Supported models](https://docs.requesty.ai/features/supported-models)).", "first_party_vs_partner": "Routed to third-party providers; open-source models are \"Hosted or bring-your-own endpoints\" ([quickstart](https://docs.requesty.ai/)). No Requesty-owned inference hardware is claimed on any page fetched.", "custom_endpoint_support": "Partially documented: the quickstart lists \"bring-your-own endpoints\" for open-source models ([quickstart](https://docs.requesty.ai/)), but no page fetched documents how to register a vLLM/Ollama/SageMaker URL — mechanism n.a.", "byok_mode": "both", "merchant_of_record": "Differs by mode. Credits: Requesty (account credits consumed per request, `cost` returned in `usage`) ([quickstart](https://docs.requesty.ai/)). BYOK: the upstream provider — \"Cost Control: Use your own billing relationships with providers\" and \"keep provider discounts and committed-use pricing\" ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys), [quickstart](https://docs.requesty.ai/)). Explicit invoicing language: not stated.", "key_management_note": "Upstream keys are added per provider in the dashboard, \"at most one key per provider\", usable in fallback policies, with usage tracked across the organization ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys)). Requesty's own API keys are \"hashed at rest\", shown once, and scoped service-account keys are supported; prompts may be logged up to 30 days unless disabled, and org-wide Zero Data Retention requires a written request ([Security](https://www.requesty.ai/security)). Enterprise page claims zero data retention and no training on customer data ([Enterprise](https://www.requesty.ai/enterprise)). Encryption/KMS specifics for upstream keys: n.a.", "api_openai_chat_completions_note": "yes: `POST https://router.requesty.ai/v1/chat/completions` ([quickstart](https://docs.requesty.ai/), [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).", "api_anthropic_messages_note": "yes: `POST /v1/messages`, documented as `https://router.requesty.ai/anthropic/v1/messages` for the Anthropic SDK ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis), [quickstart](https://docs.requesty.ai/)).", "api_openai_responses_note": "yes: `POST /v1/responses` ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).", "api_embeddings_note": "yes (\"Create vector embeddings from text…\"), documented under the inference API family ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).", "api_images_note": "yes: image generation and editing \"using DALL-E, Stable Diffusion, and other image models\" ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).", "api_audio_note": "yes, both: Text to Speech and Speech to Text endpoints ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).", "api_batch_note": "n.a. (no batch/async bulk endpoint on any Requesty page fetched: [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis), [quickstart](https://docs.requesty.ai/)).", "deployment_modes_note": "Hosted SaaS (EU/Frankfurt region available) ([Enterprise](https://www.requesty.ai/enterprise), [Security](https://www.requesty.ai/security)). Self-host on Kubernetes via Helm, incl. air-gapped, is documented in a vendor blog post ([Self-hosting Requesty on Kubernetes](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369)) while the enterprise page says self-hosting is \"not offered at this time\" ([Enterprise](https://www.requesty.ai/enterprise)) — see contradictions. — self_host and air_gapped are contested: a vendor engineering post documents a Helm/Kubernetes install including air-gapped, while the enterprise page states self-hosting is not offered at this time.", "byok_mode_note": "both: platform credits (\"New accounts include free credits to start routing immediately\") plus BYOK (\"Use your own API keys with Requesty… Choose between Requesty's keys or your own keys for each model\") ([quickstart](https://docs.requesty.ai/), [BYOK](https://docs.requesty.ai/features/bring-your-own-keys)).", "access_meta": { "qualified": [ "api_audio" ], "model_count_kind": "enumerated", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "No timeout header or key on the fetched fallback/load-balancing/latency-routing/limits pages ([Requesty API limits](https://docs.requesty.ai/features/api-limits.md)).", "rel_retry_config": "dashboard_only", "rel_retry_note": "Retries are part of a policy configured in the Requesty console: 0–10 retries per model, exponential backoff 500ms → 1s → 2s → 4s with ±10% jitter ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).", "rel_retry_default_note": null, "rel_fallback_config": "dashboard_only", "rel_fallback_note": "ORDERED. Fallback policies are created in the dashboard and referenced from the request as `policy/` in the `model` field ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": "No upstream health check, circuit breaker or provider-ejection mechanism is documented on the fetched pages; latency routing selects by observed latency instead ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Regional Bedrock endpoints can be targeted as separate models, but no cross-region failover feature is documented ([Requesty load balancing policies](https://docs.requesty.ai/features/load-balancing-policies)).", "rel_load_balancing_config": "dashboard_only", "rel_load_balancing_note": "Weights ARE supported and must total 100% across the models in a load-balancing policy ([Requesty load balancing policies](https://docs.requesty.ai/features/load-balancing-policies)).", "architecture_class": "vendor_saas", "architecture_note": "Runtime is never disclosed on the fetched pages; only the hosted endpoint `https://router.requesty.ai/v1` is documented ([Requesty streaming](https://docs.requesty.ai/features/streaming)).", "self_host_data_plane": "not_documented", "self_host_artifact_note": "No self-host artifact appears on the fetched pages ([Requesty streaming](https://docs.requesty.ai/features/streaming)).", "latency_claims": [ { "url": "https://docs.requesty.ai/features/streaming", "unit": "", "scope": "marketing", "value": "up to 80% lower", "metric": "perceived wait time", "publisher": "vendor", "conditions": "Streaming marketing claim. Not a gateway-overhead measurement; conditions not stated." } ], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported over Server-Sent Events with `\"stream\": true`. Caveats: streaming responses do not include a `usage` object by default — you must pass `stream_options: {\"include_usage\": true}`, which adds an extra chunk before `data: [DONE]` with an empty `choices` array; the docs also tell you to implement your own retry/non-streaming fallback for stream interruptions ([Requesty streaming](https://docs.requesty.ai/features/streaming)).", "guard_pii": "sync_observe", "guard_pii_note": "PII scanners (`pii/v2/fast`, `pii/v2/ml`) support actions Disabled, Report, and Mask; masking rewrites the request but the docs document no hard block/deny action ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))", "guard_injection": "not_documented", "guard_injection_note": "The guardrails page documents PII and secrets scanners only ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))", "guard_content": "not_documented", "guard_content_note": "No toxicity/moderation scanner documented ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))", "guard_custom": "not_documented", "guard_custom_note": "Policy form is a choice of vendor-supplied scanners plus an action; no regex, word list, or JSON-schema enforcement documented ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails), [Managed policies](https://docs.requesty.ai/features/managed-policies))", "guard_model_policy": "yes", "guard_model_policy_note": "Approved Models restricts the callable catalog, and Access Lists apply org-, group-, and key-level allowlists ([Approved models](https://docs.requesty.ai/features/approved-models), [Access lists](https://docs.requesty.ai/features/access-lists))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "No timeout or failure behaviour stated ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))", "log_content": "full_request_and_response", "log_content_note": "The Logs view shows full message history for a request, i.e. prompt and completion text is stored ([Requesty Logs](https://docs.requesty.ai/features/logs))", "log_opt_out": "not_documented", "log_opt_out_note": "No documented header or setting to suppress bodies while keeping metrics ([Requesty Logs](https://docs.requesty.ai/features/logs))", "tracing_support": "otel", "tracing_note": "Requesty groups related requests into Traces keyed by `trace_id`, giving multi-step visibility; OpenTelemetry support is not documented ([Requesty Logs](https://docs.requesty.ai/features/logs))", "export_destinations": [], "export_destinations_note": "`n.a.` — No external log/metric/trace destinations documented ([Requesty Logs](https://docs.requesty.ai/features/logs))", "feedback_capture": "partial", "feedback_capture_note": "Logs carry a Feedback column, but no feedback API endpoint or header is documented ([Requesty Logs](https://docs.requesty.ai/features/logs))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` — No eval or scoring product documented ([Requesty Logs](https://docs.requesty.ai/features/logs))", "observability_requires_saas": null, "log_retention_note": "`n.a.` — No retention window stated ([Requesty Logs](https://docs.requesty.ai/features/logs))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "One marketing figure only: streaming reduces \"perceived wait time by up to 80%\" — metric is perceived wait time, not gateway overhead; conditions not stated ([Requesty streaming](https://docs.requesty.ai/features/streaming)). No p50/p99/overhead figures ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).", "multi_surface": {}, "throughput_prose": "`n.a.` ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).", "benchmark_independence": "The single \"up to 80%\" figure is a vendor claim about its own product, with no methodology ([Requesty streaming](https://docs.requesty.ai/features/streaming))." }, "guard_pii_default": "sync_observe", "guard_injection_default": "choose_at_setup", "guard_content_default": "not_documented", "guard_custom_default": "choose_at_setup", "rel_notes": "Everything reliability-related is a named dashboard \"policy\" invoked as `policy/` in the `model` field, so reliability config is not in your repo and not per-request ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).", "guard_notes": "Guardrail verdicts are written into the logs, so Requesty's scanners function primarily as inline detectors with masking, and the docs never promise request denial ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails)).", "fit_agent_features_note": "Requesty documents function calling, agent routing policies with preferred models, fallback chains, and cost caps, and agent analytics for latency, cost, and success rates. Session reconstruction can replay full conversation sessions for debugging; sessions/threads as an agent primitive, multi-step orchestration, and A2A are not stated. ([Requesty homepage](https://www.requesty.ai/))", "fit_byok_required": "no", "fit_byok_required_note": "No. Only `REQUESTY_API_KEY` is needed; \"Bring your own keys\" is listed as an optional feature ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Requesty documents the OpenAI SDK for Python and TypeScript/Node.js using the `openai` package, installed with `pip install openai` or `npm install openai`, and the `OpenAI` client class. The Anthropic SDK is also documented through `https://router.requesty.ai/anthropic/v1/messages`; cURL requires no installation. ([Requesty quickstart](https://docs.requesty.ai/quickstart))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "user_facing", "fit_cloudflare_workers_note": "Requesty can run behind Cloudflare Workers, which can handle authentication, rate limiting, request validation, forwarding, response caching, and global edge deployment. ([Requesty behind Cloudflare Workers](https://www.requesty.ai/blog/edge-deployments-running-requesty-behind-cloudflare-workers-1751655483))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated as required; \"New accounts include free credits to start routing immediately.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart)). Enterprise page: \"You can start on the pay-as-you-go plan with no commitment\" ([Requesty Enterprise](https://www.requesty.ai/enterprise))", "fit_ecosystem_notes": "Requesty is documented as a hosted, OpenAI-compatible gateway with unified routing, fallback, caching, analytics, BYOK, and access to hundreds of models through a single endpoint; documented model counts include 300+ and 600+. Official evidence also documents EU routing at `https://router.eu.requesty.ai/v1` and self-hosting on Kubernetes, including air-gapped deployments. ([Requesty homepage](https://www.requesty.ai/))", "fit_eval_prereqs": "A Requesty account (\"Sign up at app.requesty.ai\") and an API key exported as `REQUESTY_API_KEY`. No credit card, cloud account, cluster, or separate provider key is stated ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes. OpenAI-SDK Python/TypeScript plus curl: `client.chat.completions.create(model=\"openai/gpt-4o\", messages=[{\"role\":\"user\",\"content\":\"Hello, who are you?\"}])` against `https://router.requesty.ai/v1`, returning an OpenAI-compatible completion ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "Helm repo https://helm.requesty.ai (chart name not stated)", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Requesty documents self-hosting on Kubernetes, including air-gapped environments. The official Helm repository is named `requesty` and is located at `https://helm.requesty.ai`; the chart name is not stated. ([Requesty Kubernetes Helm deployment guide](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "Requesty provides an MCP Gateway that authenticates and routes requests to multiple MCP servers, supports tool discovery and whitelisting, and integrates with MCP-compatible tools such as Claude Code, Cursor, and Roo Code. It supports `streamable-http` and `sse`; `stdio` is coming soon, and the Roo Code endpoint is `https://router.requesty.ai/mcp`. ([Requesty MCP Gateway docs](https://docs.requesty.ai/features/mcp-gateway))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_gateway_platform", "fit_terraform" ] }, "fit_model_gate": "none", "fit_model_gate_note": "None for self-serve: \"Every model lives behind one endpoint. Swap `model` in the request to switch providers.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart)). Restriction is an admin option, not a gate: \"Enterprise plans let admins restrict access to an approved list of models and providers\" ([Requesty Enterprise](https://www.requesty.ai/enterprise))", "fit_onboarding_notes": "Only two config changes are required (Requesty key instead of `OPENAI_API_KEY`, Requesty base URL instead of `https://api.openai.com/v1`); an Anthropic-SDK path exists at `https://router.requesty.ai/anthropic/v1/messages`; OpenAI-compatible frameworks (LangChain, Vercel AI SDK, LlamaIndex, Haystack, Pydantic AI) \"work with Requesty out of the box\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Requesty documents OpenAI SDK drop-in compatibility with `base_url=\"https://router.requesty.ai/v1\"`; the EU endpoint is `https://router.eu.requesty.ai/v1`. ([Requesty OpenAI framework docs](https://docs.requesty.ai/frameworks/openai))", "fit_prod_prereqs": "Nothing infrastructural; enterprise adds a sales-led path of 3 steps (\"Talk to our team\" → \"Custom onboarding\" → \"Go live\") and \"Enterprise plans are priced on request\" for SSO/RBAC/SLA ([Requesty Enterprise](https://www.requesty.ai/enterprise))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "Requesty documents LangChain integration using `langchain_openai.ChatOpenAI`, `langchain_core.prompts.PromptTemplate`, and `langchain_core.runnables.RunnableLambda`. LlamaIndex is also documented as an integration, including a TypeScript integration, but its package and class names are not stated. ([Requesty LangChain framework docs](https://docs.requesty.ai/frameworks/langchain))", "fit_quickstart_steps": 4, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.requesty.ai/quickstart", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented)", "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": "Vendor claim, verbatim: \"One API. 300+ models. OpenAI-compatible. Route your first request in under 2 minutes.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Yes. The official Vercel AI SDK provider package is `@requesty/ai-sdk`, with the provider instance `requesty` imported from `@requesty/ai-sdk`. ([Requesty Vercel AI SDK docs](https://docs.requesty.ai/frameworks/vercel-ai-sdk))", "fit_vercel_ai_sdk_package": "@requesty/ai-sdk" }, { "slug": "velokey", "name": "Velokey", "company": null, "tagline": "Hosted access to text, image and video models through one account.", "category": "managed_marketplace", "deployment": "managed_only", "license": "Proprietary hosted service", "founded_year": null, "hq_country": null, "url_home": "https://velokey.ai/", "url_docs": "https://docs.velokey.ai/api/introduction", "url_pricing": "https://velokey.ai/pricing", "url_status": "https://velokey.ai/status", "url_github": null, "url_changelog": null, "url_models_api": "https://api.velokey.ai/v1/models", "github_stars": null, "model_count": 94, "model_count_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 20, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": "Provider-key passthrough and any associated fees are not published.", "seat_fee_usd_month": null, "free_tier": "$0.50 signup credit advertised; confirm current eligibility and promotional terms.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": null, "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Retest media task handling and each model when migrating.", "api_surfaces": [ "OpenAI Chat Completions", "Asynchronous image tasks", "Asynchronous video tasks", "OpenAI Responses" ], "sdk_note": "Existing OpenAI-compatible clients; media uses task polling.", "cap_failover": true, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": true, "cap_audio": null, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Vendor describes millisecond-range routing; no precise comparable measurement.", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "Privacy policy excludes output storage, but image docs promise permanent hosted outputs. Upstream policies also apply.", "air_gapped": null, "best_for": "Developers evaluating consolidated access to language and media models.", "strengths": [ "Shared account and billing", "Familiar chat interface", "Asynchronous media workflows" ], "weaknesses": [ "Conflicting output-retention statements", "Separate gateway fee percentages not published", "Enterprise assurance not established" ], "avoid_if": "You require verified end-to-end ZDR, contractual uptime or documented private deployment.", "reviews": [], "sources": { "zdr": "https://velokey.ai/privacy", "name": "https://velokey.ai/", "slug": "https://velokey.ai/", "license": "https://velokey.ai/terms", "sla_url": "https://velokey.ai/terms", "tagline": "https://velokey.ai/", "avoid_if": "https://velokey.ai/", "best_for": "https://velokey.ai/", "category": "https://velokey.ai/", "sdk_note": "https://docs.velokey.ai/api/introduction", "url_docs": "https://velokey.ai/", "url_home": "https://velokey.ai/", "zdr_note": "https://velokey.ai/privacy", "cap_video": "https://docs.velokey.ai/api/video-models/introduction", "free_tier": "https://velokey.ai/", "logs_note": "https://velokey.ai/privacy", "rel_notes": "https://velokey.ai/", "strengths": "https://velokey.ai/", "api_images": "https://docs.velokey.ai/api/image-models/introduction", "deployment": "https://velokey.ai/", "url_status": "https://velokey.ai/", "weaknesses": "https://velokey.ai/", "zdr_status": "https://velokey.ai/privacy", "cost_export": "https://velokey.ai/privacy", "guard_notes": "https://docs.velokey.ai/api/image-models/introduction", "log_content": "https://velokey.ai/privacy", "model_count": "https://api.velokey.ai/v1/models", "url_pricing": "https://velokey.ai/", "api_surfaces": "https://docs.velokey.ai/api/introduction", "cap_failover": "https://velokey.ai/", "overage_note": "https://velokey.ai/terms", "byok_fee_note": "https://velokey.ai/terms", "cap_image_gen": "https://docs.velokey.ai/api/image-models/introduction", "pricing_model": "https://velokey.ai/terms", "training_note": "https://velokey.ai/terms", "byok_mode_note": "https://velokey.ai/terms", "fit_eval_shape": "https://docs.velokey.ai/api/introduction", "fit_prod_shape": "https://docs.velokey.ai/api/introduction", "min_commitment": "https://velokey.ai/pricing", "provider_count": "https://velokey.ai/about", "retention_note": "https://velokey.ai/terms", "streaming_note": "https://docs.velokey.ai/api/text-models/gpt-5-5-chat-completions", "url_models_api": "https://docs.velokey.ai/api/introduction", "api_images_note": "https://docs.velokey.ai/api/image-models/introduction", "deployment_note": "https://velokey.ai/about", "logs_by_default": "https://velokey.ai/privacy", "model_count_low": "https://api.velokey.ai/v1/models", "api_surface_note": "https://docs.velokey.ai/api/introduction", "cost_attribution": "https://velokey.ai/privacy", "deployment_modes": "https://velokey.ai/about", "fit_eval_prereqs": "https://docs.velokey.ai/api/introduction", "fit_eval_snippet": "https://docs.velokey.ai/api/introduction", "fit_prod_prereqs": "https://docs.velokey.ai/api/introduction", "log_content_note": "https://velokey.ai/privacy", "model_count_high": "https://api.velokey.ai/v1/models", "model_count_note": "https://api.velokey.ai/v1/models", "portability_note": "https://docs.velokey.ai/api/introduction", "prompt_path_note": "https://velokey.ai/privacy", "architecture_note": "https://velokey.ai/about", "cap_observability": "https://velokey.ai/", "fit_byok_required": "https://docs.velokey.ai/api/introduction", "fit_openai_dropin": "https://docs.velokey.ai/api/introduction", "openai_compatible": "https://docs.velokey.ai/api/introduction", "product_kind_note": "https://velokey.ai/about", "rel_fallback_note": "https://velokey.ai/", "streaming_support": "https://docs.velokey.ai/api/text-models/gpt-5-5-chat-completions", "architecture_class": "https://velokey.ai/about", "fit_quickstart_url": "https://docs.velokey.ai/api/introduction", "log_retention_note": "https://velokey.ai/privacy", "merchant_of_record": "https://velokey.ai/terms", "pricing_model_note": "https://velokey.ai/terms", "provider_count_low": "https://velokey.ai/about", "export_destinations": "https://velokey.ai/privacy", "key_management_note": "https://velokey.ai/about", "latency_source_note": "https://velokey.ai/about", "provider_count_high": "https://velokey.ai/about", "requires_vendor_sdk": "https://docs.velokey.ai/api/introduction", "api_openai_responses": "https://docs.velokey.ai/api/text-models/gpt-5-5-responses", "fit_client_sdk_langs": "https://docs.velokey.ai/api/introduction", "fit_client_sdks_note": "https://docs.velokey.ai/api/introduction", "fit_gateway_platform": "https://velokey.ai/about", "fit_onboarding_notes": "https://docs.velokey.ai/api/introduction", "fit_quickstart_steps": "https://docs.velokey.ai/api/introduction", "data_export_available": "https://velokey.ai/privacy", "deployment_modes_note": "https://velokey.ai/about", "fit_eval_snippet_note": "https://docs.velokey.ai/api/introduction", "first_party_vs_partner": "https://velokey.ai/about", "fit_byok_required_note": "https://docs.velokey.ai/api/introduction", "fit_openai_dropin_note": "https://docs.velokey.ai/api/introduction", "prompt_transits_vendor": "https://velokey.ai/privacy", "fit_agent_features_note": "https://docs.velokey.ai/api/text-models/gpt-5-5-chat-completions", "model_count_source_note": "https://api.velokey.ai/v1/models", "trains_on_customer_data": "https://velokey.ai/terms", "export_destinations_note": "https://velokey.ai/privacy", "api_openai_responses_note": "https://docs.velokey.ai/api/text-models/gpt-5-5-responses", "fit_gateway_platform_note": "https://velokey.ai/about", "provider_count_source_note": "https://velokey.ai/about", "api_openai_chat_completions": "https://docs.velokey.ai/api/introduction", "fit_quickstart_steps_caveat": "https://docs.velokey.ai/api/introduction", "api_openai_chat_completions_note": "https://docs.velokey.ai/api/introduction" }, "verified_at": { "zdr": "2026-09-19", "name": "2026-09-19", "slug": "2026-09-19", "license": "2026-09-19", "sla_url": "2026-09-19", "tagline": "2026-09-19", "avoid_if": "2026-09-19", "best_for": "2026-09-19", "category": "2026-09-19", "sdk_note": "2026-09-19", "url_docs": "2026-09-19", "url_home": "2026-09-19", "zdr_note": "2026-09-19", "cap_video": "2026-09-19", "free_tier": "2026-09-19", "logs_note": "2026-09-19", "rel_notes": "2026-09-19", "strengths": "2026-09-19", "api_images": "2026-09-19", "deployment": "2026-09-19", "url_status": "2026-09-19", "weaknesses": "2026-09-19", "zdr_status": "2026-09-19", "cost_export": "2026-09-19", "guard_notes": "2026-09-19", "log_content": "2026-09-19", "model_count": "2026-09-26", "url_pricing": "2026-09-19", "api_surfaces": "2026-09-19", "cap_failover": "2026-09-19", "overage_note": "2026-09-19", "byok_fee_note": "2026-09-19", "cap_image_gen": "2026-09-19", "pricing_model": "2026-09-19", "training_note": "2026-09-19", "byok_mode_note": "2026-09-19", "fit_eval_shape": "2026-09-19", "fit_prod_shape": "2026-09-19", "min_commitment": "2026-09-19", "provider_count": "2026-09-19", "retention_note": "2026-09-19", "streaming_note": "2026-09-19", "url_models_api": "2026-09-19", "api_images_note": "2026-09-19", "deployment_note": "2026-09-19", "logs_by_default": "2026-09-19", "model_count_low": "2026-09-26", "api_surface_note": "2026-09-19", "cost_attribution": "2026-09-19", "deployment_modes": "2026-09-19", "fit_eval_prereqs": "2026-09-19", "fit_eval_snippet": "2026-09-19", "fit_prod_prereqs": "2026-09-19", "log_content_note": "2026-09-19", "model_count_high": "2026-09-26", "model_count_note": "2026-09-26", "portability_note": "2026-09-19", "prompt_path_note": "2026-09-19", "architecture_note": "2026-09-19", "cap_observability": "2026-09-19", "fit_byok_required": "2026-09-19", "fit_openai_dropin": "2026-09-19", "openai_compatible": "2026-09-19", "product_kind_note": "2026-09-19", "rel_fallback_note": "2026-09-19", "streaming_support": "2026-09-19", "architecture_class": "2026-09-19", "fit_quickstart_url": "2026-09-19", "log_retention_note": "2026-09-19", "merchant_of_record": "2026-09-19", "pricing_model_note": "2026-09-19", "provider_count_low": "2026-09-19", "export_destinations": "2026-09-19", "key_management_note": "2026-09-19", "latency_source_note": "2026-09-19", "provider_count_high": "2026-09-19", "requires_vendor_sdk": "2026-09-19", "api_openai_responses": "2026-09-19", "fit_client_sdk_langs": "2026-09-19", "fit_client_sdks_note": "2026-09-19", "fit_gateway_platform": "2026-09-19", "fit_onboarding_notes": "2026-09-19", "fit_quickstart_steps": "2026-09-19", "data_export_available": "2026-09-19", "deployment_modes_note": "2026-09-19", "fit_eval_snippet_note": "2026-09-19", "first_party_vs_partner": "2026-09-19", "fit_byok_required_note": "2026-09-19", "fit_openai_dropin_note": "2026-09-19", "prompt_transits_vendor": "2026-09-19", "fit_agent_features_note": "2026-09-19", "model_count_source_note": "2026-09-26", "trains_on_customer_data": "2026-09-19", "export_destinations_note": "2026-09-19", "api_openai_responses_note": "2026-09-19", "fit_gateway_platform_note": "2026-09-19", "provider_count_source_note": "2026-09-19", "api_openai_chat_completions": "2026-09-19", "fit_quickstart_steps_caveat": "2026-09-19", "api_openai_chat_completions_note": "2026-09-19" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:17:00.525715+00:00", "vendor_status": null, "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": null, "last_release_date": null, "sla_uptime_pct": null, "sla_url": "https://velokey.ai/terms", "claim_basis": { "avoid_if": "editorial", "best_for": "editorial", "category": "editorial", "strengths": "editorial", "weaknesses": "editorial" }, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Requests pass through Velokey to upstream services.", "logs_by_default": "metadata_only", "logs_note": "Policy says metadata only; image-output storage conflicts with its blanket exclusion.", "retention_days": null, "retention_note": "No uniform retention duration established.", "zdr_status": "depends", "trains_on_customer_data": "no", "training_note": "Vendor excludes its own training; upstream policies apply separately.", "subprocessors_url": null, "regions_note": null, "certifications": {}, "guardrails_location": null, "guardrails_note": null, "acquired_date_basis": null, "security_incidents": [], "pricing_model": null, "pricing_model_note": "Prepaid usage credits; model-specific units. Separate markup not published.", "min_commitment": "No plan minimums advertised; minimum credit purchase not established.", "overage_note": "Check retry/failure charges before production.", "cache_support": null, "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": null, "cost_controls": {}, "cost_attribution": "Model, token and billing metadata.", "cost_export": "Usage and transaction export.", "product_kind_note": "Marketplace/access layer, not a model developer.", "deployment_modes": [ "saas" ], "deployment_note": "Hosted service documented; no customer deployment package established.", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "yes", "api_embeddings": "not_documented", "api_images": "partial", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Compatibility is endpoint- and model-specific.", "model_count_low": 94, "model_count_high": 94, "model_count_source_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": 20, "provider_count_high": null, "provider_count_source_note": "Vendor claims 20+ upstream providers; route pool not independently enumerated.", "first_party_vs_partner": "Third-party models.", "custom_endpoint_support": null, "byok_mode": null, "merchant_of_record": "Velokey sells credits; legal contracting entity not identified in reviewed terms.", "key_management_note": "Per-key usage and revocation advertised.", "api_openai_chat_completions_note": "Bearer key and /v1/chat/completions.", "api_anthropic_messages_note": null, "api_openai_responses_note": "POST /v1/responses documented for GPT5.5; model-specific parity must be tested.", "api_embeddings_note": null, "api_images_note": "Asynchronous task ID and polling; not a synchronous OpenAI image-response substitute.", "api_audio_note": null, "api_batch_note": null, "deployment_modes_note": "Only hosted access verified.", "byok_mode_note": "Documented funding path uses platform credits; BYOK availability not established.", "access_meta": { "qualified": [ "api_images" ], "model_count_kind": "enumerated", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "not_documented", "rel_timeout_note": null, "rel_retry_config": "not_documented", "rel_retry_note": null, "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "Vendor-managed failover when healthy alternative routes are available; caller controls not documented.", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": null, "rel_multi_region_config": "not_documented", "rel_multi_region_note": null, "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": null, "architecture_class": "vendor_saas", "architecture_note": "Implementation runtime not published.", "self_host_data_plane": "not_documented", "self_host_artifact_note": null, "latency_claims": [], "throughput_claims": [], "streaming_support": "partial", "streaming_note": "Chat stream parameter documented; media is asynchronous.", "guard_pii": "not_documented", "guard_pii_note": null, "guard_injection": "not_documented", "guard_injection_note": null, "guard_content": "not_documented", "guard_content_note": null, "guard_custom": "not_documented", "guard_custom_note": null, "guard_model_policy": "not_documented", "guard_model_policy_note": null, "guard_execution_location": "not_documented", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": null, "log_content": "metadata_only", "log_content_note": "Operational logs; generated media storage is a separate concern.", "log_opt_out": "not_documented", "log_opt_out_note": null, "tracing_support": "not_documented", "tracing_note": null, "export_destinations": [ "Console export" ], "export_destinations_note": "Usage and transaction records.", "feedback_capture": "not_documented", "feedback_capture_note": null, "eval_hooks": "not_documented", "eval_hooks_note": null, "observability_requires_saas": null, "log_retention_note": "Metadata retention period not published.", "production_meta": {}, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Test failure behavior and retry billing for your selected route.", "guard_notes": "Do not infer configurable gateway guardrails from upstream content restrictions.", "fit_agent_features_note": "Chat schema includes tools and tool_choice; agent framework compatibility not tested.", "fit_byok_required": "no", "fit_byok_required_note": "The documented credit-funded path uses a Velokey key.", "fit_client_sdk_langs": [ "Python" ], "fit_client_sdks_note": "OpenAI client example on homepage.", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": null, "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": null, "fit_credit_card": "not_documented", "fit_credit_card_note": null, "fit_ecosystem_notes": null, "fit_eval_prereqs": "Account, Velokey key and eligible model.", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "curl example available.", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Application requests traverse the hosted service.", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": null, "fit_mcp": "not_documented", "fit_mcp_note": null, "fit_meta": {}, "fit_model_gate": "not_documented", "fit_model_gate_note": null, "fit_onboarding_notes": "Use the account model endpoint to confirm routable IDs.", "fit_openai_dropin": "partial", "fit_openai_dropin_note": "Chat-compatible; media return shapes differ.", "fit_prod_prereqs": "Validate model capabilities, charges and data handling.", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [], "fit_python_frameworks_note": null, "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": "Editorial grouping: create key, configure client, select model. Not a timed test.", "fit_quickstart_url": "https://docs.velokey.ai/api/introduction", "fit_self_host_docs": "not_documented", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": null, "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": null, "fit_vercel_ai_sdk_package": null }, { "slug": "ai-gateway-hq", "name": "AI Gateway HQ", "company": "AI Gateway HQ LLC", "tagline": "Hosted AI routing, budget enforcement and policy controls with metadata-based reporting.", "category": "managed_gateway", "deployment": "managed_only", "license": "Proprietary hosted service", "founded_year": null, "hq_country": null, "url_home": "https://aigatewayhq.com/", "url_docs": "https://aigatewayhq.com/docs/", "url_pricing": "https://aigatewayhq.com/pricing/", "url_status": "https://aigatewayhq.com/status/", "url_github": null, "url_changelog": null, "url_models_api": "https://api.aigatewayhq.com/v1/models", "github_stars": null, "model_count": null, "model_count_note": "No exhaustive public total; configured routes and the reviewed managed catalog have different scopes.", "provider_count": 11, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "0% applies to BYOK inference. Flex gateway usage is $0.10/1,000 successful requests; managed Bedrock rates are separate.", "seat_fee_usd_month": null, "free_tier": "No-card simulated Test Lab; live inference needs provider funding and gateway entitlement.", "enterprise_from_usd_year": 36000, "extra_fees": [ { "label": "Flex gateway usage", "amount": "$0.10 per 1,000 successful requests", "request_rate_usd": 0.0001 }, { "label": "Alternative Company subscription", "amount": "$499/month; 2,000,000 successful requests ceiling", "optional": true }, { "label": "Alternative Portfolio subscription", "amount": "$1,500/month sponsor base; participating company costs separate", "optional": true } ], "self_host_cost_note": "Design engagement from $60,000/year; not a ready-to-run deployment license.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Compatible clients reduce integration work; policies, aliases and metadata require a separate migration plan.", "api_surfaces": [ "OpenAI Responses", "OpenAI Chat Completions", "Anthropic Messages", "Embeddings" ], "sdk_note": "Use an upstream SDK with the gateway URL, workload key and configured model alias.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No comparable numeric gateway-overhead benchmark established.", "throughput_rps": null, "soc2": false, "hipaa_baa": null, "gdpr": null, "eu_residency": false, "eu_residency_note": "Current hosted production is US-E1; no generally available EU deployment established.", "zdr": null, "zdr_note": "No default prompt persistence is narrower than universal ZDR; cache, support and upstream terms matter.", "air_gapped": null, "best_for": "Teams evaluating hosted budget and policy controls with metadata-based reporting.", "strengths": [ "Pre-request reservations with scoped workload keys", "Compatible OpenAI and Anthropic interfaces", "Metadata-only default request evidence" ], "weaknesses": [ "No generally available private deployment or SLA", "No current SOC 2 certification claim", "Most documented BYOK integrations remain Beta" ], "avoid_if": "You need a current self-host package, audited assurance, or a public exhaustive model inventory.", "reviews": [], "sources": { "zdr": "https://aigatewayhq.com/legal/privacy/", "name": "https://aigatewayhq.com/legal/terms/", "slug": "https://aigatewayhq.com/legal/terms/", "soc2": "https://aigatewayhq.com/trust/", "cap_mcp": "https://aigatewayhq.com/platform/governance/", "company": "https://aigatewayhq.com/legal/terms/", "fit_mcp": "https://aigatewayhq.com/platform/governance/", "license": "https://aigatewayhq.com/legal/terms/", "sla_url": "https://aigatewayhq.com/trust/", "tagline": "https://aigatewayhq.com/", "avoid_if": "https://aigatewayhq.com/", "best_for": "https://aigatewayhq.com/", "category": "https://aigatewayhq.com/", "sdk_note": "https://aigatewayhq.com/docs/", "url_docs": "https://aigatewayhq.com/", "url_home": "https://aigatewayhq.com/", "zdr_note": "https://aigatewayhq.com/legal/privacy/", "byok_mode": "https://aigatewayhq.com/pricing/", "free_tier": "https://aigatewayhq.com/pricing/", "guard_pii": "https://aigatewayhq.com/docs/", "hipaa_baa": "https://aigatewayhq.com/trust/", "logs_note": "https://aigatewayhq.com/legal/privacy/", "rel_notes": "https://aigatewayhq.com/platform/routing/", "strengths": "https://aigatewayhq.com/", "deployment": "https://aigatewayhq.com/pricing/", "extra_fees": "https://aigatewayhq.com/pricing/", "url_status": "https://aigatewayhq.com/", "weaknesses": "https://aigatewayhq.com/", "zdr_status": "https://aigatewayhq.com/legal/privacy/", "access_meta": "https://aigatewayhq.com/integrations/", "cap_budgets": "https://aigatewayhq.com/platform/cost-control/", "cost_export": "https://aigatewayhq.com/platform/cost-control/", "guard_notes": "https://aigatewayhq.com/platform/governance/", "log_content": "https://aigatewayhq.com/legal/privacy/", "url_pricing": "https://aigatewayhq.com/", "api_surfaces": "https://aigatewayhq.com/docs/", "cap_failover": "https://aigatewayhq.com/platform/routing/", "eu_residency": "https://aigatewayhq.com/trust/", "fit_mcp_note": "https://aigatewayhq.com/platform/governance/", "guard_custom": "https://aigatewayhq.com/docs/", "overage_note": "https://aigatewayhq.com/pricing/", "regions_note": "https://aigatewayhq.com/trust/", "byok_fee_note": "https://aigatewayhq.com/pricing/", "cache_support": "https://aigatewayhq.com/platform/routing/", "cost_controls": "https://aigatewayhq.com/platform/cost-control/", "fit_cloud_iam": "https://aigatewayhq.com/platform/security/", "fit_terraform": "https://aigatewayhq.com/integrations/", "guard_content": "https://aigatewayhq.com/docs/", "pricing_model": "https://aigatewayhq.com/pricing/", "training_note": "https://aigatewayhq.com/legal/privacy/", "api_embeddings": "https://aigatewayhq.com/docs/", "byok_mode_note": "https://aigatewayhq.com/pricing/", "byok_supported": "https://aigatewayhq.com/pricing/", "cap_embeddings": "https://aigatewayhq.com/docs/", "cap_guardrails": "https://aigatewayhq.com/platform/governance/", "certifications": "https://aigatewayhq.com/trust/", "fit_eval_shape": "https://aigatewayhq.com/docs/", "fit_model_gate": "https://aigatewayhq.com/integrations/", "fit_prod_shape": "https://aigatewayhq.com/docs/", "guard_pii_note": "https://aigatewayhq.com/docs/", "min_commitment": "https://aigatewayhq.com/pricing/", "provider_count": "https://aigatewayhq.com/integrations/", "rel_retry_note": "https://aigatewayhq.com/platform/routing/", "retention_note": "https://aigatewayhq.com/legal/privacy/", "streaming_note": "https://aigatewayhq.com/platform/routing/", "url_models_api": "https://aigatewayhq.com/docs/", "cap_rate_limits": "https://aigatewayhq.com/platform/cost-control/", "deployment_note": "https://aigatewayhq.com/pricing/", "fit_credit_card": "https://aigatewayhq.com/pricing/", "guard_injection": "https://aigatewayhq.com/docs/", "guardrails_note": "https://aigatewayhq.com/platform/governance/", "logs_by_default": "https://aigatewayhq.com/legal/privacy/", "production_meta": "https://aigatewayhq.com/docs/", "api_surface_note": "https://aigatewayhq.com/docs/", "cap_cache_simple": "https://aigatewayhq.com/platform/routing/", "cap_virtual_keys": "https://aigatewayhq.com/platform/cost-control/", "cost_attribution": "https://aigatewayhq.com/platform/cost-control/", "deployment_modes": "https://aigatewayhq.com/pricing/", "fit_eval_prereqs": "https://aigatewayhq.com/docs/", "fit_eval_snippet": "https://aigatewayhq.com/docs/", "fit_prod_prereqs": "https://aigatewayhq.com/docs/", "log_content_note": "https://aigatewayhq.com/legal/privacy/", "model_count_note": "https://aigatewayhq.com/integrations/", "portability_note": "https://aigatewayhq.com/docs/", "prompt_path_note": "https://aigatewayhq.com/legal/privacy/", "rel_retry_config": "https://aigatewayhq.com/platform/routing/", "rel_timeout_note": "https://aigatewayhq.com/platform/routing/", "token_markup_pct": "https://aigatewayhq.com/pricing/", "architecture_note": "https://aigatewayhq.com/trust/", "cap_observability": "https://aigatewayhq.com/docs/", "cap_pii_redaction": "https://aigatewayhq.com/docs/", "eu_residency_note": "https://aigatewayhq.com/trust/", "fit_byok_required": "https://aigatewayhq.com/pricing/", "fit_openai_dropin": "https://aigatewayhq.com/docs/", "guard_custom_note": "https://aigatewayhq.com/docs/", "guard_pii_default": "https://aigatewayhq.com/docs/", "openai_compatible": "https://aigatewayhq.com/docs/", "product_kind_note": "https://aigatewayhq.com/", "rel_fallback_note": "https://aigatewayhq.com/platform/routing/", "streaming_support": "https://aigatewayhq.com/platform/routing/", "subprocessors_url": "https://aigatewayhq.com/legal/privacy/", "architecture_class": "https://aigatewayhq.com/trust/", "cache_pricing_note": "https://aigatewayhq.com/platform/routing/", "cap_load_balancing": "https://aigatewayhq.com/platform/routing/", "fit_cloud_iam_note": "https://aigatewayhq.com/platform/security/", "fit_quickstart_url": "https://aigatewayhq.com/docs/", "fit_self_host_docs": "https://aigatewayhq.com/pricing/", "fit_terraform_note": "https://aigatewayhq.com/integrations/", "guard_failure_mode": "https://aigatewayhq.com/docs/", "guard_model_policy": "https://aigatewayhq.com/platform/governance/", "log_retention_note": "https://aigatewayhq.com/legal/privacy/", "merchant_of_record": "https://aigatewayhq.com/pricing/", "pricing_model_note": "https://aigatewayhq.com/pricing/", "provider_count_low": "https://aigatewayhq.com/integrations/", "rel_fallback_shape": "https://aigatewayhq.com/platform/routing/", "rel_timeout_config": "https://aigatewayhq.com/platform/routing/", "api_embeddings_note": "https://aigatewayhq.com/docs/", "export_destinations": "https://aigatewayhq.com/docs/", "fit_ecosystem_notes": "https://aigatewayhq.com/integrations/", "fit_model_gate_note": "https://aigatewayhq.com/integrations/", "guardrails_location": "https://aigatewayhq.com/platform/governance/", "key_management_note": "https://aigatewayhq.com/platform/security/", "latency_source_note": "https://aigatewayhq.com/trust/", "provider_count_high": "https://aigatewayhq.com/integrations/", "rel_fallback_config": "https://aigatewayhq.com/platform/routing/", "requires_vendor_sdk": "https://aigatewayhq.com/docs/", "self_host_cost_note": "https://aigatewayhq.com/pricing/", "api_openai_responses": "https://aigatewayhq.com/docs/", "fit_client_sdk_langs": "https://aigatewayhq.com/docs/", "fit_client_sdks_note": "https://aigatewayhq.com/docs/", "fit_credit_card_note": "https://aigatewayhq.com/pricing/", "fit_gateway_platform": "https://aigatewayhq.com/docs/", "fit_onboarding_notes": "https://aigatewayhq.com/docs/", "fit_quickstart_steps": "https://aigatewayhq.com/docs/", "guard_custom_default": "https://aigatewayhq.com/docs/", "self_host_data_plane": "https://aigatewayhq.com/pricing/", "data_export_available": "https://aigatewayhq.com/docs/", "deployment_modes_note": "https://aigatewayhq.com/pricing/", "fit_eval_snippet_note": "https://aigatewayhq.com/docs/", "guard_content_default": "https://aigatewayhq.com/docs/", "rel_health_check_note": "https://aigatewayhq.com/platform/routing/", "api_anthropic_messages": "https://aigatewayhq.com/docs/", "first_party_vs_partner": "https://aigatewayhq.com/", "fit_byok_required_note": "https://aigatewayhq.com/pricing/", "fit_openai_dropin_note": "https://aigatewayhq.com/docs/", "prompt_transits_vendor": "https://aigatewayhq.com/legal/privacy/", "rel_retry_default_note": "https://aigatewayhq.com/platform/routing/", "cap_conditional_routing": "https://aigatewayhq.com/platform/routing/", "custom_endpoint_support": "https://aigatewayhq.com/integrations/", "fit_agent_features_note": "https://aigatewayhq.com/integrations/", "guard_failure_mode_note": "https://aigatewayhq.com/docs/", "guard_injection_default": "https://aigatewayhq.com/docs/", "guard_model_policy_note": "https://aigatewayhq.com/platform/governance/", "model_count_source_note": "https://aigatewayhq.com/integrations/", "rel_health_check_config": "https://aigatewayhq.com/platform/routing/", "rel_load_balancing_note": "https://aigatewayhq.com/platform/routing/", "self_host_artifact_note": "https://aigatewayhq.com/pricing/", "trains_on_customer_data": "https://aigatewayhq.com/legal/privacy/", "enterprise_from_usd_year": "https://aigatewayhq.com/pricing/", "export_destinations_note": "https://aigatewayhq.com/docs/", "guard_execution_location": "https://aigatewayhq.com/platform/governance/", "api_openai_responses_note": "https://aigatewayhq.com/docs/", "fit_gateway_platform_note": "https://aigatewayhq.com/docs/", "rel_load_balancing_config": "https://aigatewayhq.com/platform/routing/", "provider_count_source_note": "https://aigatewayhq.com/integrations/", "api_anthropic_messages_note": "https://aigatewayhq.com/docs/", "api_openai_chat_completions": "https://aigatewayhq.com/docs/", "fit_quickstart_steps_caveat": "https://aigatewayhq.com/docs/", "observability_requires_saas": "https://aigatewayhq.com/trust/", "api_openai_chat_completions_note": "https://aigatewayhq.com/docs/" }, "verified_at": { "zdr": "2026-09-17", "name": "2026-09-17", "slug": "2026-09-17", "soc2": "2026-09-17", "cap_mcp": "2026-09-17", "company": "2026-09-17", "fit_mcp": "2026-09-17", "license": "2026-09-17", "sla_url": "2026-09-17", "tagline": "2026-09-17", "avoid_if": "2026-09-17", "best_for": "2026-09-17", "category": "2026-09-17", "sdk_note": "2026-09-17", "url_docs": "2026-09-17", "url_home": "2026-09-17", "zdr_note": "2026-09-17", "byok_mode": "2026-09-17", "free_tier": "2026-09-17", "guard_pii": "2026-09-17", "hipaa_baa": "2026-09-17", "logs_note": "2026-09-17", "rel_notes": "2026-09-17", "strengths": "2026-09-17", "deployment": "2026-09-17", "extra_fees": "2026-09-17", "url_status": "2026-09-17", "weaknesses": "2026-09-17", "zdr_status": "2026-09-17", "access_meta": "2026-09-17", "cap_budgets": "2026-09-17", "cost_export": "2026-09-17", "guard_notes": "2026-09-17", "log_content": "2026-09-17", "url_pricing": "2026-09-17", "api_surfaces": "2026-09-17", "cap_failover": "2026-09-17", "eu_residency": "2026-09-17", "fit_mcp_note": "2026-09-17", "guard_custom": "2026-09-17", "overage_note": "2026-09-17", "regions_note": "2026-09-17", "byok_fee_note": "2026-09-17", "cache_support": "2026-09-17", "cost_controls": "2026-09-17", "fit_cloud_iam": "2026-09-17", "fit_terraform": "2026-09-17", "guard_content": "2026-09-17", "pricing_model": "2026-09-17", "training_note": "2026-09-17", "api_embeddings": "2026-09-17", "byok_mode_note": "2026-09-17", "byok_supported": "2026-09-17", "cap_embeddings": "2026-09-17", "cap_guardrails": "2026-09-17", "certifications": "2026-09-17", "fit_eval_shape": "2026-09-17", "fit_model_gate": "2026-09-17", "fit_prod_shape": "2026-09-17", "guard_pii_note": "2026-09-17", "min_commitment": "2026-09-17", "provider_count": "2026-09-17", "rel_retry_note": "2026-09-17", "retention_note": "2026-09-17", "streaming_note": "2026-09-17", "url_models_api": "2026-09-17", "cap_rate_limits": "2026-09-17", "deployment_note": "2026-09-17", "fit_credit_card": "2026-09-17", "guard_injection": "2026-09-17", "guardrails_note": "2026-09-17", "logs_by_default": "2026-09-17", "production_meta": "2026-09-17", "api_surface_note": "2026-09-17", "cap_cache_simple": "2026-09-17", "cap_virtual_keys": "2026-09-17", "cost_attribution": "2026-09-17", "deployment_modes": "2026-09-17", "fit_eval_prereqs": "2026-09-17", "fit_eval_snippet": "2026-09-17", "fit_prod_prereqs": "2026-09-17", "log_content_note": "2026-09-17", "model_count_note": "2026-09-17", "portability_note": "2026-09-17", "prompt_path_note": "2026-09-17", "rel_retry_config": "2026-09-17", "rel_timeout_note": "2026-09-17", "token_markup_pct": "2026-09-17", "architecture_note": "2026-09-17", "cap_observability": "2026-09-17", "cap_pii_redaction": "2026-09-17", "eu_residency_note": "2026-09-17", "fit_byok_required": "2026-09-17", "fit_openai_dropin": "2026-09-17", "guard_custom_note": "2026-09-17", "guard_pii_default": "2026-09-17", "openai_compatible": "2026-09-17", "product_kind_note": "2026-09-17", "rel_fallback_note": "2026-09-17", "streaming_support": "2026-09-17", "subprocessors_url": "2026-09-17", "architecture_class": "2026-09-17", "cache_pricing_note": "2026-09-17", "cap_load_balancing": "2026-09-17", "fit_cloud_iam_note": "2026-09-17", "fit_quickstart_url": "2026-09-17", "fit_self_host_docs": "2026-09-17", "fit_terraform_note": "2026-09-17", "guard_failure_mode": "2026-09-17", "guard_model_policy": "2026-09-17", "log_retention_note": "2026-09-17", "merchant_of_record": "2026-09-17", "pricing_model_note": "2026-09-17", "provider_count_low": "2026-09-17", "rel_fallback_shape": "2026-09-17", "rel_timeout_config": "2026-09-17", "api_embeddings_note": "2026-09-17", "export_destinations": "2026-09-17", "fit_ecosystem_notes": "2026-09-17", "fit_model_gate_note": "2026-09-17", "guardrails_location": "2026-09-17", "key_management_note": "2026-09-17", "latency_source_note": "2026-09-17", "provider_count_high": "2026-09-17", "rel_fallback_config": "2026-09-17", "requires_vendor_sdk": "2026-09-17", "self_host_cost_note": "2026-09-17", "api_openai_responses": "2026-09-17", "fit_client_sdk_langs": "2026-09-17", "fit_client_sdks_note": "2026-09-17", "fit_credit_card_note": "2026-09-17", "fit_gateway_platform": "2026-09-17", "fit_onboarding_notes": "2026-09-17", "fit_quickstart_steps": "2026-09-17", "guard_custom_default": "2026-09-17", "self_host_data_plane": "2026-09-17", "data_export_available": "2026-09-17", "deployment_modes_note": "2026-09-17", "fit_eval_snippet_note": "2026-09-17", "guard_content_default": "2026-09-17", "rel_health_check_note": "2026-09-17", "api_anthropic_messages": "2026-09-17", "first_party_vs_partner": "2026-09-17", "fit_byok_required_note": "2026-09-17", "fit_openai_dropin_note": "2026-09-17", "prompt_transits_vendor": "2026-09-17", "rel_retry_default_note": "2026-09-17", "cap_conditional_routing": "2026-09-17", "custom_endpoint_support": "2026-09-17", "fit_agent_features_note": "2026-09-17", "guard_failure_mode_note": "2026-09-17", "guard_injection_default": "2026-09-17", "guard_model_policy_note": "2026-09-17", "model_count_source_note": "2026-09-17", "rel_health_check_config": "2026-09-17", "rel_load_balancing_note": "2026-09-17", "self_host_artifact_note": "2026-09-17", "trains_on_customer_data": "2026-09-17", "enterprise_from_usd_year": "2026-09-17", "export_destinations_note": "2026-09-17", "guard_execution_location": "2026-09-17", "api_openai_responses_note": "2026-09-17", "fit_gateway_platform_note": "2026-09-17", "rel_load_balancing_config": "2026-09-17", "provider_count_source_note": "2026-09-17", "api_anthropic_messages_note": "2026-09-17", "api_openai_chat_completions": "2026-09-17", "fit_quickstart_steps_caveat": "2026-09-17", "observability_requires_saas": "2026-09-17", "api_openai_chat_completions_note": "2026-09-17" }, "locked_fields": [], "published": true, "updated_at": "2026-09-17T18:42:22.802043+00:00", "vendor_status": null, "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": null, "last_release_date": null, "sla_uptime_pct": null, "sla_url": "https://aigatewayhq.com/trust/", "claim_basis": { "avoid_if": "editorial", "best_for": "editorial", "category": "editorial", "strengths": "editorial", "weaknesses": "editorial" }, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Hosted gateway processes content before forwarding to the selected provider.", "logs_by_default": "metadata_only", "logs_note": "Ordinary evidence excludes prompt/output bodies; optional features can change the path.", "retention_days": null, "retention_note": "Record- and configuration-dependent; no universal day count.", "zdr_status": "depends", "trains_on_customer_data": "no", "training_note": "Gateway processor role excludes a separate model-training purpose; upstream terms are independent.", "subprocessors_url": "https://aigatewayhq.com/legal/subprocessors/", "regions_note": "Hosted production US-E1; managed Bedrock uses reviewed U.S. regions.", "certifications": { "soc2": { "detail": "Vendor explicitly does not claim SOC 2 certification.", "status": "not_published" }, "fedramp": { "detail": "No authorization claimed.", "status": "not_published" }, "hipaa_baa": { "detail": "Regulated use requires a separately executed agreement.", "status": "not_published" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Hosted policy boundary; execution of tools remains the client responsibility.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Flex usage meter or Company subscription; provider inference is separate. Portfolio has a separate sponsor boundary.", "min_commitment": "Flex has no subscription. Hosted enterprise starts at $36,000/year.", "overage_note": "Company and Portfolio allowances are hard monthly ceilings, not automatic overages.", "cache_support": "exact", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Opt-in volatile exact cache; gateway billing still runs. Streaming, tools and idempotent requests bypass it.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "supported": true }, "virtual_keys": { "supported": true }, "rate_limiting": { "supported": true }, "budget_caps_per_key": { "supported": true }, "budget_caps_per_team": { "note": "Organization and workload-key scopes; dedicated team hierarchy not established.", "supported": null } }, "cost_attribution": "Workload, environment, client, data class, provider account, route and model in retained evidence.", "cost_export": "Durable period reporting and invoice-import automation are not currently included.", "product_kind_note": "Hosted governance gateway; optional managed Bedrock access is a separate funding path.", "deployment_modes": [ "saas" ], "deployment_note": "Customer-VPC deployment is not generally available.", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Documented interfaces do not imply support for every upstream API feature.", "model_count_low": null, "model_count_high": null, "model_count_source_note": "Anonymous models endpoint requires authentication; do not infer model availability from provider integrations.", "provider_count_low": 11, "provider_count_high": 11, "provider_count_source_note": "Enumerated ten named model services plus OpenRouter; not a model count. Most BYOK integrations are labelled Beta.", "first_party_vs_partner": "Routes to third-party model services; it does not supply its own foundation models.", "custom_endpoint_support": "Named compatible integrations; arbitrary custom-host support not established.", "byok_mode": "both", "merchant_of_record": "Customer pays BYOK provider directly; managed Bedrock is prepaid through AI Gateway HQ.", "key_management_note": "Write-only provider credentials encrypted using tenant-bound KMS context.", "api_openai_chat_completions_note": "POST /v1/chat/completions; compatible target required.", "api_anthropic_messages_note": "POST /v1/messages with compatible provider targets.", "api_openai_responses_note": "POST /v1/responses; route capabilities still constrain requests.", "api_embeddings_note": "POST /v1/embeddings; configure an eligible target.", "api_images_note": null, "api_audio_note": null, "api_batch_note": null, "deployment_modes_note": "Private-deployment design is separately scoped, not an available self-host license.", "byok_mode_note": "BYOK or funded managed Bedrock; managed access requires purchased credit and a reusable payment method.", "access_meta": { "qualified": [ "api_openai_chat_completions", "api_openai_responses", "api_anthropic_messages", "api_embeddings" ], "provider_count_kind": "enumerated" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "Route attempt/time boundaries configured by administrators. Console configuration is documented; whether this is dashboard-only or also configurable through code is not established.", "rel_retry_config": "not_documented", "rel_retry_note": "Bounded attempts; exact numeric defaults not published. Console configuration is documented; whether this is dashboard-only or also configurable through code is not established.", "rel_retry_default_note": "Hedging is off by default; this does not establish retry count.", "rel_fallback_config": "not_documented", "rel_fallback_note": "Fallback constrained by capabilities, region, time and cost. Console configuration is documented; whether this is dashboard-only or also configurable through code is not established.", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "Health-aware selection and quota cooldowns. Console configuration is documented; whether this is dashboard-only or also configurable through code is not established.", "rel_multi_region_config": "not_documented", "rel_multi_region_note": null, "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": "Priority and weighted credential pools. Console configuration is documented; whether this is dashboard-only or also configurable through code is not established.", "architecture_class": "vendor_saas", "architecture_note": "Hosted AWS production; no independently measured overhead recorded.", "self_host_data_plane": "no", "self_host_artifact_note": "No generally available customer deployment; paid design work is distinct.", "latency_claims": [], "throughput_claims": [], "streaming_support": "partial", "streaming_note": "Compatible BYOK streams supported; exact cache and enforcing output checks exclude streaming.", "guard_pii": "not_documented", "guard_pii_note": "Non-streaming output PII/secret checks are documented; a prompt-input PII blocking guarantee is not established.", "guard_injection": "not_documented", "guard_injection_note": null, "guard_content": "not_documented", "guard_content_note": null, "guard_custom": "sync_block", "guard_custom_note": "Bounded JSON Schema and configured tool profiles before release; enforcing inspection requires stream=false.", "guard_model_policy": "sync_block", "guard_model_policy_note": "Enforce can deny or restrict routes; Observe and Shadow allow traffic.", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "Identity, tenant and explicit policy denial fail closed; a general guardrail timeout guarantee is not established.", "log_content": "metadata_only", "log_content_note": "Identity, route, timing and usage metadata; caching/support may differ.", "log_opt_out": "not_documented", "log_opt_out_note": null, "tracing_support": "not_documented", "tracing_note": null, "export_destinations": [ "Signed HTTPS audit webhooks" ], "export_destinations_note": "Administrative metadata only; not a complete prompt or trace export.", "feedback_capture": "not_documented", "feedback_capture_note": null, "eval_hooks": "not_documented", "eval_hooks_note": null, "observability_requires_saas": "Current hosted console stores operating metadata.", "log_retention_note": "Usage, security, audit and billing records have different retention purposes.", "production_meta": { "qualified": [ "guard_custom", "guard_model_policy" ] }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Validate configured target capabilities; fallback is not a guarantee of upstream parity.", "guard_notes": "Organization/key controls do not establish delegated hierarchy or non-overridable parent policies.", "fit_agent_features_note": "Client configuration guidance for Codex CLI, Claude Code and OpenCode.", "fit_byok_required": "optional", "fit_byok_required_note": "BYOK for customer accounts; managed Bedrock uses purchased credit.", "fit_client_sdk_langs": [ "Python" ], "fit_client_sdks_note": "Examples use upstream OpenAI and Anthropic SDKs.", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Workload gateway key for requests; OIDC/SAML is administrative identity.", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": null, "fit_credit_card": "not_required", "fit_credit_card_note": "Test Lab requires no card; managed Bedrock requires a reusable payment method.", "fit_ecosystem_notes": "Most BYOK provider connections are Beta; managed Bedrock is labelled Available.", "fit_eval_prereqs": "Workspace, provider credential or managed entitlement, route alias and workload key.", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Python and curl examples are published.", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Applications call this gateway before configured upstream services.", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": null, "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Checks MCP declarations and emitted calls; not evidence of an MCP-server proxy.", "fit_meta": {}, "fit_model_gate": "enablement_step", "fit_model_gate_note": "Configure eligible targets; managed models are reviewed for account availability.", "fit_onboarding_notes": "Start with test credentials and inspect decisions before enforcing rules.", "fit_openai_dropin": "partial", "fit_openai_dropin_note": "Endpoint replacement also requires gateway credentials and configured aliases.", "fit_prod_prereqs": "Validate route capabilities, rates, output caps and enforcing policies before production.", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [], "fit_python_frameworks_note": null, "fit_quickstart_steps": 5, "fit_quickstart_steps_caveat": "Vendor lists five setup steps; no elapsed-time test performed.", "fit_quickstart_url": "https://aigatewayhq.com/docs/", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "Internal AWS module exists; customer-facing resource provider is planned, not available.", "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": null, "fit_vercel_ai_sdk_package": null }, { "slug": "braintrust-gateway", "name": "Braintrust Gateway", "company": "Braintrust", "tagline": "Multi-provider gateway inside Braintrust's eval and observability platform, with caching and span-level tracing.", "category": "managed_gateway", "deployment": "both", "license": "MIT", "founded_year": null, "hq_country": "United States", "url_home": "https://www.braintrust.dev", "url_docs": "https://www.braintrust.dev/docs/deploy/gateway", "url_pricing": "https://www.braintrust.dev/pricing", "url_status": "https://status.braintrust.dev", "url_github": "https://github.com/braintrustdata/braintrust-proxy", "url_changelog": "https://www.braintrust.dev/docs/reference/release-notes", "url_models_api": null, "github_stars": 409, "model_count": 100, "model_count_note": "Proxy docs state over 100 models; the supported-models page enumerates 18 providers (OpenAI, Azure, Anthropic, Google, Vertex AI, Bedrock, Together, Fireworks, Groq, Mistral, Lepton, Cerebras, Perplexity, xAI, Replicate, Ollama, Databricks, Baseten). Model-list endpoints require authentication.", "provider_count": 18, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Provider keys are configured at organization or project level (project keys override org keys); the proxy can even be used with only a provider key and no Braintrust account.", "seat_fee_usd_month": 0, "free_tier": "Starter $0/month: unlimited users, $10 model credits, 1 GB processed data, 10k scores, 14-day retention. The Braintrust-hosted Gateway is free during public preview, with pricing to be announced before GA.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Pro platform fee", "amount": "$249/month (includes $249 model credits, 5 GB data, 50k scores)" }, { "label": "Processed data overage", "amount": "$4/GB (Starter), $3/GB (Pro)" }, { "label": "Scores overage", "amount": "$2.50 per 1,000 (Starter), $1.50 per 1,000 (Pro)" }, { "label": "Extended retention", "amount": "$0.50 per GB per month beyond the included 30 days (Pro, up to 180 days)" } ], "self_host_cost_note": "Self-hosted Braintrust deployments run the Gateway inside the customer's own data plane so LLM traffic stays in their account; the standalone proxy is MIT-licensed on GitHub. On-prem/hybrid deployment is an Enterprise-plan feature, and independent reviews note there is no free self-hosted tier below Enterprise.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Logs and spans export to your own S3/GCS bucket as JSONL or Parquet.", "api_surfaces": [ "OpenAI Chat Completions", "Anthropic Messages", "Google Gemini", "AWS Bedrock", "OpenAI Realtime" ], "sdk_note": "Point the OpenAI, Anthropic or Gemini SDK at https://gateway.braintrust.dev with a Braintrust user key (sk-) or service token (bt-st-); cross-SDK calls are supported.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": null, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": false, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": true, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": true, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No published gateway overhead; the global endpoint uses latency-based DNS routing with health checks across five regions (us-east-1, us-west-2, eu-west-1, ap-southeast-1). A third-party comparison confirms no measured Braintrust proxy overhead is published.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": null, "eu_residency": true, "eu_residency_note": "EU West Gateway endpoint https://gateway.euw.braintrust.dev runs in eu-west-1 (Ireland); Gateway routing is separate from the organization's data-plane region.", "zdr": null, "zdr_note": "Zero data retention applies only to Braintrust's own built-in Topics inference endpoints, where it is on by default. There is no ZDR mode for your trace data; the equivalents are retention policies or running the data plane yourself.", "air_gapped": null, "best_for": "Teams that already run evals and tracing in Braintrust and want their model traffic to flow through the same platform.", "strengths": [ "Hosted Gateway is free during public preview, with unlimited users on all plans", "Deep eval/tracing integration: spans, scorers, datasets, CI quality gates", "Cross-SDK access (OpenAI, Anthropic, Gemini, Bedrock) plus Realtime audio and embeddings", "End-to-end encrypted cache with 1-week default TTL and per-request cache control", "Time-limited temporary credentials for frontend/mobile clients; MIT-licensed proxy on GitHub" ], "weaknesses": [ "Gateway pricing after general availability is unannounced, so cost is unpredictable", "Narrower provider list (18) and no public model-catalog endpoint", "No real-time guardrails - evaluation happens after the fact", "Steep jump from free Starter to $249/month Pro, with per-GB and per-score overages that agent workloads can inflate", "Self-hosting is effectively Enterprise-only" ], "avoid_if": "You want a standalone routing marketplace or inline guardrails rather than an eval-centric platform.", "reviews": [ { "url": "https://www.lennysnewsletter.com/p/how-braintrust-uses-ai-agents-evals", "why": "Long-form independent interview with Braintrust CEO Ankur Goyal on how the platform (proxy, evals, CI) is used in practice, including agent-driven benchmarking work.", "date": "2026-06-15", "type": "written", "title": "How Braintrust uses AI agents, evals, and CI to ship better AI products", "publisher": "Lenny's Newsletter" }, { "url": "https://www.youtube.com/watch?v=97iykOemOn4", "why": "Hamel Husain, an independent AI-evals practitioner, records a hands-on walkthrough of Braintrust (datasets, tracing, BTQL, SME review) as part of a tool-by-tool comparison.", "date": "2025-10-01", "type": "video", "title": "LLM Eval Tools Compared: Braintrust", "publisher": "YouTube - Hamel Husain" } ], "sources": { "soc2": "https://www.braintrust.dev/pricing", "license": "https://github.com/braintrustdata/braintrust-proxy", "sdk_note": "https://www.braintrust.dev/docs/deploy/gateway", "url_docs": "https://www.braintrust.dev/docs/deploy/gateway", "zdr_note": "https://www.braintrust.dev/docs/guides/proxy", "cap_audio": "https://www.braintrust.dev/docs/guides/proxy", "cap_evals": "https://www.braintrust.dev/pricing", "free_tier": "https://www.braintrust.dev/docs/deploy/gateway", "hipaa_baa": "https://www.braintrust.dev/pricing", "deployment": "https://www.braintrust.dev/docs/deploy/gateway", "extra_fees": "https://www.braintrust.dev/pricing", "hq_country": "https://vantaige.io/ai-tool/braintrust", "url_github": "https://github.com/braintrustdata/braintrust-proxy", "url_status": "https://status.braintrust.dev", "zdr_status": "https://www.braintrust.dev/docs/security", "model_count": "https://www.braintrust.dev/docs/guides/proxy", "url_pricing": "https://www.braintrust.dev/pricing", "api_surfaces": "https://www.braintrust.dev/docs/deploy/gateway", "cap_failover": "https://www.braintrust.dev/docs/deploy/gateway", "eu_residency": "https://www.braintrust.dev/docs/deploy/gateway", "github_stars": "https://api.github.com/repos/braintrustdata/braintrust-proxy", "product_fate": "https://www.braintrust.dev/docs/changelog", "regions_note": "https://www.braintrust.dev/legal/dpa", "byok_fee_note": "https://www.braintrust.dev/docs/guides/proxy", "url_changelog": "https://www.braintrust.dev/docs/reference/release-notes", "vendor_status": "https://www.braintrust.dev/blog/announcing-series-b", "byok_supported": "https://www.braintrust.dev/docs/deploy/gateway", "cap_embeddings": "https://www.braintrust.dev/docs/deploy/gateway", "cap_guardrails": "https://vantaige.io/ai-tool/braintrust", "certifications": "https://www.braintrust.dev/docs/security", "provider_count": "https://www.braintrust.dev/docs/deploy/supported-models", "retention_days": "https://www.braintrust.dev/docs/admin/data-management/retention", "logs_by_default": "https://www.braintrust.dev/docs/observe", "cap_cache_simple": "https://www.braintrust.dev/docs/guides/proxy", "cap_virtual_keys": "https://www.braintrust.dev/docs/guides/proxy", "model_count_note": "https://www.braintrust.dev/docs/deploy/supported-models", "cap_observability": "https://www.braintrust.dev/docs/deploy/gateway", "eu_residency_note": "https://www.braintrust.dev/docs/deploy/gateway", "openai_compatible": "https://www.braintrust.dev/docs/deploy/gateway", "subprocessors_url": "https://www.braintrust.dev/legal/dpa", "braintrust_pricing": "https://www.braintrust.dev/pricing", "cap_load_balancing": "https://www.braintrust.dev/docs/guides/proxy", "seat_fee_usd_month": "https://www.braintrust.dev/pricing", "guardrails_location": "https://www.braintrust.dev/blog/security-data-control", "latency_source_note": "https://www.braintrust.dev/docs/deploy/gateway", "requires_vendor_sdk": "https://www.braintrust.dev/docs/deploy/gateway", "self_host_cost_note": "https://www.braintrust.dev/docs/deploy/gateway", "cap_prompt_management": "https://www.braintrust.dev/docs/guides/proxy", "data_export_available": "https://www.braintrust.dev/docs/admin/data-management/export", "braintrust_billing_faq": "https://www.braintrust.dev/docs/admin/billing/faq", "prompt_transits_vendor": "https://www.braintrust.dev/docs/security", "trains_on_customer_data": "https://www.braintrust.dev/blog/security-data-control", "braintrust_ai_proxy_blog": "https://www.braintrust.dev/blog/ai-proxy", "braintrust_plans_and_limits": "https://www.braintrust.dev/docs/plans-and-limits" }, "verified_at": { "soc2": "2026-08-29", "license": "2026-09-14", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "cap_evals": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_github": "2026-08-29", "url_status": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "eu_residency": "2026-08-29", "github_stars": "2026-08-29", "byok_fee_note": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "provider_count": "2026-08-29", "cap_cache_simple": "2026-08-29", "cap_virtual_keys": "2026-08-29", "model_count_note": "2026-08-29", "cap_observability": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-14T20:21:32.73295+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed, with an $80M Series B led by ICONIQ announced in February 2026 and new dashboard features in August 2026.", "last_release_date": null, "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Braintrust dates its changelog by month only, so no exact release date is published. The latest entry is August 2026.", "prompt_transits_vendor": "depends", "prompt_path_note": "Three shapes: multi-tenant SaaS, bring-your-own-cloud where Braintrust operates the data plane inside your account, or full self-hosting. In BYOC the SDK and browser talk straight to your data plane, so no AI data crosses Braintrust's control plane.", "logs_by_default": "full_content", "logs_note": "Once instrumented, Braintrust captures every request as a trace. That is the product, not a side effect — the data plane stores prompts, outputs, datasets and traces.", "retention_days": 14, "retention_note": "Fourteen days on Starter, thirty on Pro (extendable to 180 in 30-day steps), up to 365 on Enterprise.", "zdr_status": "depends", "trains_on_customer_data": "no", "training_note": "Prompts, traces and customer conversations are never used to improve models.", "subprocessors_url": "https://www.braintrust.dev/legal/dpa", "regions_note": "Hosted data plane in the US or the EU on Pro. BYOC and self-hosted keep data in your own cloud account and region.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPAs can be executed", "status": "available" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "detail": "BAAs can be executed", "status": "available" } }, "guardrails_location": "both", "guardrails_note": "Global masking functions redact PII before logging. In BYOC or self-hosted that masking runs inside your own cloud.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Flat monthly platform fee + included model credits then per-token rates + usage overages (processed data, scores, retention). Starter $0 with $10/mo credits; Pro $249/mo with $249/mo credits (drops to $100/mo credits after 2026-09-01); Enterprise annual invoice. AI proxy blog says the gateway itself is free 'as a beta', but Braintrust's docs index the proxy page as 'deprecated' — treat the free-proxy framing as unverified.", "min_commitment": "None for Starter/Pro. Enterprise is annual-invoiced.", "overage_note": "Processed data: 1 GB then +$4/GB (Starter) / 5 GB then +$3/GB (Pro). Scores: 10k then $2.50/1k (Starter) / 50k then $1.50/1k (Pro). Retention: 14 days (Starter) / 30 days then +$0.50/GB/month up to 180 days (Pro). Model credits do not roll over. Retention bills what you ingest into each month you keep beyond the base window, not what's currently stored. Deleting data does not reduce processed-data charges.", "cache_support": "exact", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact-request response caching in the proxy: requests with temperature=0 or `seed` are cached, encrypted per API key, controllable via x-bt-use-cache. No gateway-layer cached-token pricing; Braintrust's own model rates do publish cache-hit prices (GLM-5.2 81% off, Kimi K3 90% off). Cache hits avoid the provider call entirely, but cached traffic still counts as processed data at ingestion.", "cost_controls": { "enforcement": "after_the_fact", "spend_alerts": { "note": "Automated emails at 60% and 100% of Topics credit; Starter alerts at 80/90/100% of included usage. Reported after spend.", "tier": null, "supported": true }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Platform-wide only (10,000 ops/10s), not configurable — returns HTTP 429.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Not stated as cost splits. Environments tag object versions but are not described as cost attribution.", "cost_export": "S3 / Google Cloud Storage scheduled export automations on Enterprise only; price not stated. CSV, webhook and warehouse export not stated.", "product_kind_note": "An evaluation/observability platform whose gateway also proxies: the vendor calls it the \"Braintrust AI proxy\", giving \"unified access to models from OpenAI, Anthropic, Google, AWS, Mistral, and third-party providers through a single API\" with \"automatic caching, observability, and multi-provider support\" ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). The proxy is now deprecated in favour of the Braintrust Gateway ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway docs](https://www.braintrust.dev/docs/deploy/gateway)).", "deployment_modes": [ "saas", "self_host", "hybrid_vpc" ], "deployment_note": "Hybrid model: Braintrust-managed control plane plus a customer-hosted data plane — AWS via Terraform on ECS/EC2, GCP and Azure via Kubernetes/Helm; the self-hosted data-plane URL is configured in organization settings ([Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting)). \"Self-hosted Braintrust deployments include a built-in proxy that runs in your environment\" and the self-hosted Gateway serves `/v1/proxy` ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)). The OSS proxy ships deployment wrappers for Vercel, Cloudflare, AWS Lambda and Express ([GitHub](https://github.com/braintrustdata/braintrust-proxy)). Tier gating for hybrid: not stated on pages fetched; note self-hosted deployments have built-in models disabled by default while SaaS enables them ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "partial", "api_batch": "not_documented", "api_surface_note": "Base-URL swap with the standard OpenAI drivers (\"You can communicate with the proxy via the standard OpenAI drivers/API\"), no proprietary SDK required ([GitHub](https://github.com/braintrustdata/braintrust-proxy)). Caveats: paths sit under `/v1/proxy`; temporary credentials and WebSocket realtime work only on the standalone `https://braintrustproxy.com/v1` host, not `api.braintrust.dev/v1/proxy`; requests to `api.braintrust.dev/v1/proxy` now route to the Gateway with \"no code or URL changes\"; caching triggers when `temperature=0` or a seed is set ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [API ref](https://www.braintrust.dev/docs/api-reference/proxy/proxy-chatcompletions)). Self-hosted reasoning support needs data plane v0.0.74+, project-level provider precedence v2.0.0+ ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "model_count_low": 100, "model_count_high": 100, "model_count_source_note": "\"over 100 models\" (page undated) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "provider_count_low": 9, "provider_count_high": 9, "provider_count_source_note": "n.a. as a number; standard providers named are OpenAI, Anthropic, Google, AWS Bedrock, Azure OpenAI, Together AI, Fireworks, Groq, Replicate (page undated) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "first_party_vs_partner": "Third-party routing with a Braintrust-billed \"built-in models\" option: built-in open-source models (e.g. `glm-5.2`, `kimi-k3`, `deepseek-v4-flash-0731`) consume Braintrust model credits at on-demand rates and are disabled by default on self-hosted deployments ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)). Whether models run on Braintrust hardware: not stated ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "custom_endpoint_support": "yes: \"Add custom models or endpoints to use with the proxy\", with custom providers supporting \"self-hosted models\", fine-tuned models and proprietary AI services, configured at org or project level with endpoint, headers, streaming and cost metadata ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "byok_mode": "both", "merchant_of_record": "BYOK: the upstream provider (customer's own provider keys; project-level keys let you \"manage separate billing\") ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). Built-in models: Braintrust, via model credits and on-demand rates ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)). Explicit \"who invoices\" language: not stated.", "key_management_note": "Provider keys are configured at organization or project scope, project overriding org; project-level keys isolate usage and billing ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). Keys are stored as one-way cryptographic hashes, never plaintext ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)); in self-hosting, project-level encrypted provider secrets stay in the customer data plane while org-level encrypted secrets sit in the control plane, and Braintrust servers/employees do not require data-plane access ([Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting)). The Gateway page states Braintrust \"cannot see your data and does not store or log API keys\" ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)); the hosted OSS proxy is described as having \"end-to-end encryption\" ([GitHub](https://github.com/braintrustdata/braintrust-proxy)).", "api_openai_chat_completions_note": "yes: `POST /v1/proxy/chat/completions`, \"fully compatible with the OpenAI SDK\" by setting base URL `https://api.braintrust.dev/v1/proxy` ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Proxy chat/completions API ref](https://www.braintrust.dev/docs/api-reference/proxy/proxy-chatcompletions)). Note the path is prefixed with `/v1/proxy`, not bare `/v1`.", "api_anthropic_messages_note": "yes: native Anthropic schema at `/v1/proxy/anthropic/messages`; `anthropic-version` and `x-api-key` headers are not required ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)); the Gateway page also lists Anthropic messages support ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "api_openai_responses_note": "yes: Responses is listed among the Gateway's supported OpenAI surfaces ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "api_embeddings_note": "yes: `/embeddings` (and Gemini `embedContent`/`batchEmbedContents`) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "api_images_note": "n.a. (no image-generation endpoint on the proxy or gateway pages fetched: [AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "api_audio_note": "Partly: the proxy supports the OpenAI Realtime API over WebSockets at `/realtime` with audio output modalities and audio-transcript stream events; standalone STT/TTS endpoints are not documented ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "api_batch_note": "n.a. for generic batch inference; only Gemini `batchEmbedContents` is listed ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "deployment_modes_note": "Hosted SaaS (`https://api.braintrust.dev/v1/proxy`, `https://gateway.braintrust.dev` in public preview), self-host / BYOC data plane (hybrid in the customer's own cloud account), and open-source self-deploy of the proxy ([Gateway](https://www.braintrust.dev/docs/deploy/gateway), [Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers), [GitHub braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)). On-prem/air-gapped: not documented.", "byok_mode_note": "both: you can use the proxy with \"your API key from any supported provider\" (even without a Braintrust account) or use \"a single Braintrust API key to access all AI providers through one interface\", and built-in models bill against Braintrust model credits ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers)).", "access_meta": { "qualified": [ "api_batch" ], "model_count_kind": "vendor_total", "provider_count_kind": "enumerated" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "No request-timeout header or key on the fetched proxy/gateway docs ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "rel_retry_config": "not_documented", "rel_retry_note": "Default count and backoff: `n.a.` The documented failure path is failover to the next endpoint, not a retry counter ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED, comma-separated: header `x-bt-fallback-providers`. Response headers `x-bt-used-endpoint`, `x-bt-failover-from` and `x-bt-failover-to` report what happened; `x-bt-endpoint-name` pins a specific endpoint ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "`not_configurable`, and what exists is health checking at the gateway's own edge rather than upstream provider ejection: the global endpoint `gateway.braintrust.dev` uses DNS latency-based routing plus health checks across `us-east-1`, `eu-west-1`, `us-west-2` and `ap-southeast-1` ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "rel_multi_region_config": "not_configurable", "rel_multi_region_note": "The vendor runs multiple regions behind one hostname with DNS health-checked routing; there is no user-facing cross-region failover config ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "rel_load_balancing_config": "not_configurable", "rel_load_balancing_note": "`not_configurable` at the weight level: the proxy automatically load-balances across multiple API keys configured for the same provider, and no weight key is documented ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy)).", "architecture_class": "edge_worker", "architecture_note": "Repo is TypeScript 99.0% and is documented as deployable to Vercel, Cloudflare Workers, AWS Lambda or Express ([braintrustdata/braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)).", "self_host_data_plane": "yes", "self_host_artifact_note": "MIT-licensed repo with deploy targets for Vercel, Cloudflare, AWS Lambda and a Node/Express server ([braintrustdata/braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported, and failover is documented as working for streaming requests. Caveat: caching is disabled for requests that went through failover ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).", "guard_pii": "not_documented", "guard_pii_note": "The AI proxy documentation describes routing, caching, and logging; no PII scanner ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))", "guard_injection": "not_documented", "guard_injection_note": "Same ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))", "guard_content": "not_documented", "guard_content_note": "Same ([Braintrust proxy repo](https://github.com/braintrustdata/braintrust-proxy))", "guard_custom": "not_documented", "guard_custom_note": "No request-path policy engine documented; policies exist as scorers applied to logs, not as blocking checks ([View logs](https://www.braintrust.dev/docs/observe/view-logs))", "guard_model_policy": "not_documented", "guard_model_policy_note": "No admin model allow/deny list documented for the proxy ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))", "guard_execution_location": "not_documented", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))", "log_content": "configurable", "log_content_note": "Logging through the proxy requires passing an `x-bt-parent` header to name the destination project; without it, requests are not logged, and the repo lists automatic logging as \"coming soon\" ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy), [Braintrust proxy repo](https://github.com/braintrustdata/braintrust-proxy)). When logging is enabled, full request/response spans including prompt and completion text are stored ([View logs](https://www.braintrust.dev/docs/observe/view-logs))", "log_opt_out": "yes", "log_opt_out_note": "Omitting `x-bt-parent` sends the request through without logging ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))", "tracing_support": "otel", "tracing_note": "OpenTelemetry is supported natively: Braintrust exposes an OTel-compatible backend endpoint and documents OpenLLMetry and Vercel AI SDK instrumentation, with multi-step spans forming agent traces ([Braintrust OpenTelemetry](https://www.braintrust.dev/docs/integrations/sdk-integrations/opentelemetry), [View logs](https://www.braintrust.dev/docs/observe/view-logs))", "export_destinations": [ "Braintrust" ], "export_destinations_note": "CSV and JSON download from the logs UI, plus `bt sync pull` for NDJSON extraction to local/customer storage; the OTel endpoint is an ingest, not an export ([View logs](https://www.braintrust.dev/docs/observe/view-logs), [Braintrust OpenTelemetry](https://www.braintrust.dev/docs/integrations/sdk-integrations/opentelemetry))", "feedback_capture": "yes", "feedback_capture_note": "`POST /v1/project_logs/{project_id}/feedback` attaches scores, comments, and metadata to a logged span ([Feedback API](https://www.braintrust.dev/docs/api-reference/logs/feedback-for-project-logs-events))", "eval_hooks": "yes", "eval_hooks_note": "Yes — online scoring runs automated scorers against logged production traffic inside Braintrust; it is a Braintrust platform feature ([View logs](https://www.braintrust.dev/docs/observe/view-logs))", "observability_requires_saas": "The proxy can be self-hosted in the customer environment, but logs, spans, and online scoring are Braintrust platform features, so full observability is tied to the Braintrust control plane ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy), [View logs](https://www.braintrust.dev/docs/observe/view-logs))", "log_retention_note": "`n.a.` — No default retention window found on the fetched pages ([View logs](https://www.braintrust.dev/docs/observe/view-logs))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` No numeric latency figure on the fetched pages ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "multi_surface": {}, "throughput_prose": "`n.a.` ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).", "benchmark_independence": "No figures published ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "The response headers (`x-bt-failover-from` / `x-bt-failover-to` / `x-bt-used-endpoint`) make failover observable per request, which is unusual and valuable for debugging silent degradation ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).", "guard_notes": "This is an eval/observability platform with a proxy attached rather than a policy gateway: the strongest features are span-level logs, OTel ingest, feedback, and online scorers, with no request-path enforcement ([View logs](https://www.braintrust.dev/docs/observe/view-logs)).", "fit_agent_features_note": "Braintrust documents agent observability and tooling features rather than gateway passthrough capabilities. These include tool/function-call spans, Vercel AI SDK agent classes such as `Agent`, `ToolLoopAgent`, `WorkflowAgent`, and `HarnessAgent`, multi-step tool interactions, OpenAI Responses API function calls and web search spans, Anthropic server-side tool use, and tracing for Claude Code, Codex, OpenCode, and pi sessions. Sessions/threads and A2A are not documented. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))", "fit_byok_required": "optional", "fit_byok_required_note": "Optional, not required: with an account \"you can use a single Braintrust API key to access all AI providers\"; without an account \"you can use the proxy with individual provider API keys.\" For tracing, \"you must use a `BRAINTRUST_API_KEY` rather than a provider's key.\" ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))", "fit_client_sdk_langs": [ "TypeScript", "Python", "Go", "Java", "Ruby", "C#" ], "fit_client_sdks_note": "Documented provider clients include the OpenAI SDK (`openai` for TypeScript/Python), Anthropic SDK (`@anthropic-ai/sdk` and `anthropic`), Google GenAI SDK (`@google/genai` and `google.genai`), and cURL. Braintrust’s official/native SDKs are documented for TypeScript/JavaScript (`braintrust`), Python (`braintrust`), Go, Java, Ruby, and C#/.NET; `@braintrust/otel` provides OpenTelemetry functionality. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Braintrust documents keyless cloud authentication and IAM reuse. AWS Bedrock supports STS `AssumeRole`; Azure AI Foundry and Azure OpenAI support Microsoft Entra Workload Identity using Braintrust-signed OIDC tokens; Google Vertex AI supports Workload Identity Federation using Braintrust-signed OIDC tokens. Self-hosted GCP deployments also use Workload Identity for GCS, and Azure uses workload identity for Key Vault. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))", "fit_cloudflare_workers": "user_facing", "fit_cloudflare_workers_note": "Cloudflare Workers are supported. Documented integrations include `OpenAIRealtimeWebSocket` from `openai/realtime/websocket` for realtime connections and `wrapCloudflareAIChat()` from `braintrust` for Cloudflare AI Chat tracing; Cloudflare Workers require `nodejs_compat`, a `BRAINTRUST_API_KEY` Wrangler secret, and flushing with `ctx.waitUntil(logger.flush())`. ([Braintrust Cloudflare AI Chat integration](https://www.braintrust.dev/docs/integrations/agent-frameworks/cloudflare-ai-chat))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated on the proxy page ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy)); the hosted Gateway \"is described as in public preview and free to use; pricing is announced before general availability\" ([Use the Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway))", "fit_ecosystem_notes": "Braintrust offers Braintrust-hosted, BYOC, and self-hosted deployments across AWS, GCP, and Azure. The hosted Gateway is at `https://gateway.braintrust.dev`, while self-hosted data planes can run the Gateway and serve `/v1/proxy`; the AI proxy documentation says the AI proxy is deprecated and recommends the Gateway for production use. Hosted Gateway logging writes to the organization’s configured data plane, and provider API keys are configured in Braintrust. ([Braintrust Gateway documentation](https://www.braintrust.dev/docs/deploy/gateway))", "fit_eval_prereqs": "Either a Braintrust API key or a provider key: \"You can use the proxy without a Braintrust account by providing your API key from any supported provider.\" No credit card, cloud account, or cluster is stated ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes. `client.chat.completions.create({model: \"gpt-4o-mini\", messages: [{role:\"user\", content:\"What is a proxy?\"}], seed: 1})` with `baseURL: \"https://api.braintrust.dev/v1/proxy\"`; cURL equivalent posts to `.../v1/proxy/chat/completions` ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))", "fit_gateway_platform": "service_mesh", "fit_gateway_platform_note": "Chart-managed Istio integration is documented through an Istio `VirtualService`; enable `virtualService.workloadIsolation.enabled: true`, along with `virtualService.enabled: true` and `api.workloadIsolation.enabled: true`. No Kong, APISIX, Envoy, nginx, or other gateway plugin is documented. ([Braintrust self-hosting scaling configuration](https://ai-proxy-5k9q3mg0z.preview.braintrust.dev/docs/admin/self-hosting/configure/scaling))", "fit_helm_chart": "braintrust from oci://public.ecr.aws/braintrust/helm/braintrust", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Official self-hosting deployment uses the Helm chart named `braintrust`, installed from `oci://public.ecr.aws/braintrust/helm/braintrust`, with the chart repository at `https://github.com/braintrustdata/helm`. Other evidence documents chart versions `6.2.1`, `6.13.0+`, and `v5.0.1+` for specific self-hosting and scaling features. ([Braintrust self-hosting deployment](https://www.braintrust.dev/docs/admin/self-hosting/deploy))", "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "Braintrust provides a hosted MCP server at `https://api.braintrust.dev/mcp` using streamable HTTP, with support for Claude Code, Cursor, Codex, VS Code, and other MCP-compatible clients. It supports reading and writing Braintrust data, querying logs, authoring prompts and scorers, configuring monitoring, and running evals; self-hosted deployments use the MCP URL shown in organization data-plane settings. ([Braintrust MCP integration](https://www.braintrust.dev/docs/integrations/developer-tools/mcp))", "fit_meta": { "not_documented_fields": [] }, "fit_model_gate": "none", "fit_model_gate_note": "No model gate: \"The proxy supports over 100 models... New models are added regularly.\" One org-level enablement gate exists for infra monitoring: \"The Infra dashboard option is available once Braintrust has enabled infrastructure monitoring for your organization.\" ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy), [Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))", "fit_onboarding_notes": "The proxy is deprecated: \"Deprecated. The AI proxy is deprecated and will no longer be regularly maintained,\" directing users to the gateway \"for production-grade reliability.\" WebSocket/realtime use requires a different base URL (`https://braintrustproxy.com/v1`). Temporary, time-limited credentials can be minted via `/credentials` for browser/mobile clients. Source is open at github.com/braintrustdata/braintrust-proxy ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. The AI proxy is “fully compatible with the OpenAI SDK”; set `baseURL`/`base_url` to `https://api.braintrust.dev/v1/proxy`. The Gateway documentation also supports keeping the OpenAI SDK and changing the base URL to `https://gateway.braintrust.dev`, with a Braintrust API key. ([Braintrust Gateway documentation](https://www.braintrust.dev/docs/deploy/gateway))", "fit_prod_prereqs": "Hosted Gateway: only `BRAINTRUST_API_KEY` ([Use the Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway)). Self-hosted data plane: Braintrust API, PostgreSQL 17+ (8+ vCPU, 64GB+ RAM, 1000GB+, 15,000+ IOPS), Redis 7+, object storage and Brainstore (NVMe, 150,000+ IOPS, dedicated Kubernetes nodes; 2+ readers, 2+ fast readers, 1+ writer), production API 4+ instances ([Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "LangChain and LlamaIndex integrations are documented. The LlamaIndex integration uses packages including `braintrust`, `llama-index`, `llama-index-llms-openai`, and `openinference-instrumentation-llama-index`, with `braintrust.auto_instrument()`, `setup_llamaindex()`, `LlamaIndexInstrumentor`, and `BraintrustSpanProcessor`. ([Braintrust LlamaIndex integration](https://www.braintrust.dev/docs/integrations/sdk-integrations/llamaindex))", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "The page carrying the quickstart and first-call snippet is marked deprecated, and it publishes no numbered procedure. The current Gateway page has no quickstart at all.", "fit_quickstart_url": "https://www.braintrust.dev/docs/guides/proxy", "fit_self_host_docs": "documented_no_command", "fit_self_host_install": "No literal command is published; self-hosting uses \"official Terraform modules\" on AWS (ECS/EC2) and GCP/Azure (Kubernetes and Helm), with Braintrust hosting the control plane ([Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))", "fit_terraform": "official_modules", "fit_terraform_note": "Official Terraform modules are documented for AWS, GCP, and Azure: `github.com/braintrustdata/terraform-aws-braintrust-data-plane`, `github.com/braintrustdata/terraform-google-braintrust-data-plane`, and `github.com/braintrustdata/terraform-azure-braintrust-data-plane`. Additional documentation identifies Terraform module versions `v5.2.1` and `v6.0` for specific AWS deployments; no Terraform Registry name is stated. ([Braintrust self-hosting deployment](https://www.braintrust.dev/docs/admin/self-hosting/deploy))", "fit_terraform_ref": "terraform-aws-braintrust-data-plane, and Google and Azure equivalents, on GitHub; no Registry name", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "integration_documented", "fit_vercel_ai_sdk_note": "Braintrust documents a Vercel AI SDK integration. Install `braintrust`, `ai`, and `zod` with `pnpm add braintrust ai zod` or `npm install braintrust ai zod`; use `wrapAISDK` for AI SDK v3–v6 or `braintrustAISDKTelemetry()` for AI SDK v7. ([Braintrust Vercel AI SDK integration](https://www.braintrust.dev/docs/integrations/sdk-integrations/vercel))", "fit_vercel_ai_sdk_package": "braintrust wrapAISDK (telemetry wrapper, not a provider)" }, { "slug": "helicone", "name": "Helicone", "company": "Helicone (Mintlify)", "tagline": "Open-source LLM observability platform with an OpenAI-compatible AI gateway attached.", "category": "managed_gateway", "deployment": "both", "license": "Apache-2.0", "founded_year": 2023, "hq_country": "United States", "url_home": "https://www.helicone.ai", "url_docs": "https://docs.helicone.ai/gateway/overview", "url_pricing": "https://www.helicone.ai/pricing", "url_status": "https://status.helicone.ai", "url_github": "https://github.com/Helicone/helicone", "url_changelog": null, "url_models_api": "https://api.helicone.ai/v1/public/model-registry/models", "github_stars": 6109, "model_count": 111, "model_count_note": "Live public model registry returned 111 models spanning 21 provider endpoints on 2026-08-29; docs describe the gateway as 100+ models / 100+ providers. The simpler https://ai-gateway.helicone.ai/v1/models endpoint returns the same 111 models without pricing.", "provider_count": 21, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Bring your own provider keys, or add Helicone credits and pay provider rates with 0% markup while Helicone manages provider keys.", "seat_fee_usd_month": 0, "free_tier": "Hobby: free, 1 seat, 10,000 requests/month, 1 GB storage, 7-day retention, 10 logs/min ingestion.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Pro plan", "amount": "$79/month, unlimited seats, 10k requests + 1 GB free then usage-based" }, { "label": "Team plan", "amount": "$799/month, 5 organizations, 3-month retention" }, { "label": "Usage overage", "amount": "Metered per request and per GB of storage beyond included amounts" } ], "self_host_cost_note": "Main platform repo is Apache-2.0 and self-hostable (infrastructure cost only); the standalone Rust AI gateway repo is GPL-3.0 and was last pushed 2025-11-21, which an independent review flags as effectively stale.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions" ], "sdk_note": "Point the OpenAI SDK at https://ai-gateway.helicone.ai with a Helicone API key; requests are logged automatically.", "cap_failover": true, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": false, "cap_pii_redaction": false, "cap_budgets": null, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No vendor-published managed-gateway overhead. An independent review (Chatforest, researched May 2026) cites a self-hosted proxy overhead of under 1 ms; another (Continuum, 2026-08-19) says the proxy adds a hop and recommends measuring on your own traffic.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": true, "eu_residency": true, "eu_residency_note": "Choice of US or EU data region, which Helicone presents as the route to GDPR compliance.", "zdr": null, "zdr_note": "Not branded as ZDR, but async logging can capture metadata only, and you can decline to send bodies to the backend at all.", "air_gapped": null, "best_for": "Teams whose main need is per-request, per-user LLM observability with a thin gateway bolted on, ideally self-hosted.", "strengths": [ "0% markup on tokens with credits or BYOK", "Apache-2.0 platform that can be self-hosted for full data control", "Strong observability: sessions, user analytics, custom properties, HQL, alerts", "Public model registry endpoint exposing per-endpoint pricing", "US or EU data region choice, SOC 2 Type II and HIPAA listed on plans" ], "weaknesses": [ "Acquired by Mintlify on 2026-03-03 and reported to be in maintenance mode with no new features planned", "Log- and storage-based pricing gets expensive for very high request counts", "Hobby plan drops logs above 10 logs/min, which can silently truncate agent traces", "Observability-first: routing, guardrails and PII redaction are weaker than Portkey or OpenRouter", "Standalone Rust gateway repo (GPL-3.0) has not had a functional commit since July 2025" ], "avoid_if": "You need an actively developed routing-first gateway with guardrails and policy enforcement.", "reviews": [ { "url": "https://clickhouse.com/blog/helicones-migration-from-postgres-to-clickhouse-for-advanced-llm-monitoring", "why": "Third-party engineering writeup of Helicone's proxy/observability architecture and why its request-log workload outgrew Postgres, with the actual migration path described.", "date": "2026-04-17", "type": "written", "title": "Helicone's Migration from Postgres to ClickHouse for Advanced LLM Monitoring", "publisher": "ClickHouse" } ], "sources": { "zdr": "https://docs.helicone.ai/features/advanced-usage/omit-logs", "gdpr": "https://docs.helicone.ai/faq/compliance", "soc2": "https://www.helicone.ai/pricing", "company": "https://chatforest.com/reviews/helicone-llm-observability-gateway/", "license": "https://docs.helicone.ai/references/open-source", "sdk_note": "https://docs.helicone.ai/getting-started/quick-start", "url_docs": "https://docs.helicone.ai/gateway/overview", "zdr_note": "https://docs.helicone.ai/faq/compliance", "cap_evals": "https://www.helicone.ai/pricing", "free_tier": "https://www.helicone.ai/pricing", "hipaa_baa": "https://www.helicone.ai/pricing", "deployment": "https://docs.helicone.ai/faq/compliance", "extra_fees": "https://www.helicone.ai/pricing", "hq_country": "https://www.ycombinator.com/companies/helicone", "url_github": "https://github.com/Helicone/helicone", "url_status": "https://status.helicone.ai", "zdr_status": "https://docs.helicone.ai/features/advanced-usage/omit-logs", "acquired_by": "https://www.mintlify.com/blog/mintlify-acquires-helicone", "model_count": "https://api.helicone.ai/v1/public/model-registry/models", "url_pricing": "https://www.helicone.ai/pricing", "api_surfaces": "https://docs.helicone.ai/gateway/overview", "cap_failover": "https://docs.helicone.ai/gateway/overview", "eu_residency": "https://docs.helicone.ai/faq/compliance", "founded_year": "https://www.ycombinator.com/companies/helicone", "github_stars": "https://api.github.com/repos/Helicone/helicone", "product_fate": "https://www.helicone.ai/blog/joining-mintlify", "regions_note": "https://docs.helicone.ai/faq/compliance", "acquired_date": "https://www.mintlify.com/blog/mintlify-acquires-helicone", "byok_fee_note": "https://docs.helicone.ai/gateway/overview", "vendor_status": "https://www.mintlify.com/blog/mintlify-acquires-helicone", "byok_supported": "https://docs.helicone.ai/getting-started/quick-start", "cap_guardrails": "https://continuumcode.ai/guides/helicone-review/", "certifications": "https://docs.helicone.ai/faq/soc2", "config_as_code": "https://github.com/Helicone/ai-gateway", "provider_count": "https://api.helicone.ai/v1/public/model-registry/models", "retention_days": "https://www.helicone.ai/pricing", "url_models_api": "https://docs.helicone.ai/rest/models/get-v1public-model-registry-models", "cap_rate_limits": "https://docs.helicone.ai/gateway/overview", "logs_by_default": "https://docs.helicone.ai/features/advanced-usage/omit-logs", "cap_cache_simple": "https://docs.helicone.ai/gateway/overview", "helicone_caching": "https://docs.helicone.ai/features/advanced-usage/caching", "helicone_pricing": "https://www.helicone.ai/pricing", "model_count_note": "https://docs.helicone.ai/gateway/overview", "token_markup_pct": "https://docs.helicone.ai/gateway/overview", "cap_observability": "https://docs.helicone.ai/gateway/overview", "cap_pii_redaction": "https://continuumcode.ai/guides/helicone-review/", "eu_residency_note": "https://docs.helicone.ai/faq/compliance", "last_release_date": "https://github.com/Helicone/helicone/releases/tag/v2025.08.21-1", "openai_compatible": "https://docs.helicone.ai/gateway/overview", "seat_fee_usd_month": "https://www.helicone.ai/pricing", "guardrails_location": "https://docs.helicone.ai/faq/compliance", "latency_source_note": "https://chatforest.com/reviews/helicone-llm-observability-gateway/", "requires_vendor_sdk": "https://docs.helicone.ai/getting-started/how-to-integrate", "self_host_cost_note": "https://continuumcode.ai/guides/helicone-review/", "helicone_rate_limits": "https://docs.helicone.ai/features/advanced-usage/custom-rate-limits", "cap_prompt_management": "https://docs.helicone.ai/gateway/overview", "data_export_available": "https://docs.helicone.ai/rest/request/post-v1requestquery-clickhouse", "prompt_transits_vendor": "https://docs.helicone.ai/faq/compliance", "helicone_gateway_overview": "https://docs.helicone.ai/gateway/overview", "helicone_custom_properties": "https://docs.helicone.ai/features/advanced-usage/custom-properties", "helicone_how_we_calculate_cost": "https://docs.helicone.ai/references/how-we-calculate-cost" }, "verified_at": { "zdr": "2026-09-05", "gdpr": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_evals": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_github": "2026-08-29", "url_status": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "eu_residency": "2026-08-29", "founded_year": "2026-08-29", "github_stars": "2026-08-29", "byok_fee_note": "2026-08-29", "byok_supported": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "provider_count": "2026-08-29", "url_models_api": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-05T20:16:27.577312+00:00", "vendor_status": "maintenance", "acquired_by": "Mintlify", "acquired_date": "2026-03-03", "acquisition_terms": "Terms undisclosed", "product_fate": "Mintlify announced the acquisition on 3 March 2026 and put Helicone into maintenance mode: security updates, bug fixes and new models keep shipping, but Mintlify says it will work closely with every customer to support a smooth migration to another platform. Helicone says its services remain live for the foreseeable future. The signup flow still advertises a free trial, so the maintenance status is easy to miss.", "last_release_date": "2025-08-21", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Latest tagged GitHub release v2025.08.21-1. The hosted changelog stops at 26 November 2025 — the stalest product in this catalogue by roughly a year.", "prompt_transits_vendor": "depends", "prompt_path_note": "Hosted cloud in the US or EU, or self-hosted entirely inside your own perimeter.", "logs_by_default": "full_content", "logs_note": "Bodies are stored unless you use Omit Logs. Note the limit of that feature: it stops storage but the request and response are still sent to Helicone's backend.", "retention_days": 7, "retention_note": "Seven days on Hobby, one month on Pro, three months on Team, indefinite on Enterprise.", "zdr_status": "depends", "trains_on_customer_data": "not_published", "training_note": "No statement found in the privacy policy, terms or data-autonomy docs. Get it in writing.", "subprocessors_url": null, "regions_note": "US or EU for the hosted service, or self-host on your own Docker, Kubernetes or cloud.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II, report on request", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "described as GDPR-compliant in a blog post", "status": "claimed" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "detail": "cloud described as HIPAA-compliant; no executable BAA offer found", "status": "claimed" } }, "guardrails_location": "none", "guardrails_note": "No guardrails or PII-redaction feature found. The nearest control is omitting logs.", "acquired_date_basis": "announced", "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Flat per-tier platform fee (Hobby free, Pro $79/mo, Team $799/mo, Enterprise custom) plus usage-based requests/storage. Gateway credits at 0% markup. 50% off first year for startups; $100 credit for OSS; free for students.", "min_commitment": "None stated for Pro/Team. Enterprise not stated.", "overage_note": "Pro/Team/Enterprise all include 10K requests + 1 GB storage with 'usage-based ↑' after — the per-unit overage rates are not published. Calculator implies storage pricing (0.30 GB → $0.97) without a stated rate.", "cache_support": "exact", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact-match gateway cache on Cloudflare's edge; cache key hashes seed, URL, full body, relevant headers and bucket index. Semantic cache not stated. Can combine with provider prompt caching. Helicone charges no cache surcharge, but cached responses count against your storage allowance.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "'Alerts' listed as a feature without tier marks.", "tier": null, "supported": true }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Global, per-user and per-property segments; pre-request. Helicone's rate limit is bounded by your provider's policy.", "tier": null, "supported": true }, "spend_webhooks": { "note": "'Webhooks' listed as a feature without tier marks.", "tier": null, "supported": true }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated as budgets, but rate limits can be denominated in cost — policies control 'request count or cost in cents within a time window' via Helicone-RateLimit-Policy, enforced pre-request with a 429.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Per user via Helicone-User-Id header; per tag/property via Custom Properties (project, feature, workflow stage, ticket). Per key/team not stated. Costs are estimates based on the model returned in the response body, not billed pass-through.", "cost_export": "Query API for filtered retrieval; property-filtered webhooks. CSV, S3 and warehouse export not stated; pricing page 'Data export' line has no format, tier or price.", "product_kind_note": "Observability platform that also proxies/routes: the repo calls it \"Helicone AI Gateway… The fastest, lightest, and easiest-to-integrate AI Gateway on the market\", \"The NGINX of LLMs\", \"Open-source, lightweight, and built on Rust\" ([GitHub Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)); the self-host Docker page calls Helicone an \"LLM observability platform\" whose Jawn service is the \"Jawn API + LLM Proxy\" ([Docker self-host](https://docs.helicone.ai/getting-started/self-host/docker)).", "deployment_modes": [ "saas", "self_host" ], "deployment_note": "Self-hosting options are Manual, Docker Compose, Kubernetes with Helm charts, and cloud infrastructure ([Self-hosting overview](https://docs.helicone.ai/getting-started/self-host/overview)); the all-in-one image is `helicone/helicone-all-in-one:latest` for a \"local machine or server\" or \"Production Setup (Remote Server) — EC2, VPS, etc.\" ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)). Important gating: in the self-hosted version \"Other providers (Vertex AI, AWS Bedrock, Azure OpenAI) are not supported\" — only OpenAI and Anthropic ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "not_documented", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap with \"any OpenAI SDK\" — \"No new SDKs to learn, no integrations to maintain\" — but model names must become provider-qualified (`gpt-4o` → `openai/gpt-4o`) ([GitHub](https://github.com/Helicone/ai-gateway)). Routing selectors are expressed in the model string, e.g. `gpt-4o-mini/openai` for a specific provider or `gpt-4o-mini/azure/clm1a2b3c` for a configured deployment ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "model_count_low": 100, "model_count_high": 100, "model_count_source_note": "\"1 API. 100+ models\" ([GitHub](https://github.com/Helicone/ai-gateway)); \"100+ Models Available\" ([Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)). Both undated.", "provider_count_low": 20, "provider_count_high": 100, "provider_count_source_note": "\"100+\" providers on the gateway docs (undated) ([Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview), [Gateway overview](https://docs.helicone.ai/gateway/overview)); the GitHub README says \"OpenAI, Anthropic, Google, AWS Bedrock, and 20+ more providers\" ([GitHub](https://github.com/Helicone/ai-gateway)); the self-hosted build supports only 2 (OpenAI, Anthropic) ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)). See contradictions.", "first_party_vs_partner": "Third-party routing to \"LLM Providers\" (OpenAI, Anthropic, AWS Bedrock, Google Vertex, \"20+ more\") ([GitHub](https://github.com/Helicone/ai-gateway)); with credits \"we manage provider API keys for you\" so requests still reach those providers ([Gateway overview](https://docs.helicone.ai/gateway/overview)). No Helicone-owned inference hardware claimed.", "custom_endpoint_support": "n.a.: not documented on the pages fetched; the closest is a \"configured deployment\" ID for provider deployments such as Azure ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "byok_mode": "both", "merchant_of_record": "Credits mode: Helicone invoices, at \"0% markup\", \"customers pay exactly provider charges\" ([Gateway overview](https://docs.helicone.ai/gateway/overview), [Provider routing](https://docs.helicone.ai/gateway/provider-routing)). BYOK/self-host mode: the upstream provider, since your own `OPENAI_API_KEY`/`ANTHROPIC_API_KEY` are used ([GitHub](https://github.com/Helicone/ai-gateway)). Explicit invoicing language: not stated.", "key_management_note": "Self-hosted: provider keys stay in your `.env`/environment as `PROVIDER_API_KEY` values, and the gateway itself \"handles API keys\", with the Helicone key only needed when authentication is enabled ([GitHub](https://github.com/Helicone/ai-gateway)). Cloud: keys are added in Provider Settings and BYOK keys are attempted before Helicone-managed keys ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)). Rate limits can be scoped per user, team, per-API-key or globally ([GitHub](https://github.com/Helicone/ai-gateway)). Encryption specifics and whether Helicone can read request bodies in cloud mode: n.a. on pages fetched.", "api_openai_chat_completions_note": "yes: self-hosted path `http://YOUR_IP:8585/v1/gateway/oai/v1/chat/completions`; cloud/router usage is `client.chat.completions.create` against a gateway base URL such as `https://ai-gateway.helicone.ai/ai` or `http://localhost:8080/router/` ([Docker](https://docs.helicone.ai/getting-started/self-host/docker), [GitHub](https://github.com/Helicone/ai-gateway)).", "api_anthropic_messages_note": "yes: `http://YOUR_IP:8585/v1/gateway/anthropic/v1/messages` ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)).", "api_openai_responses_note": "n.a. (not mentioned on the gateway or self-host pages fetched: [GitHub](https://github.com/Helicone/ai-gateway), [Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)).", "api_embeddings_note": "n.a. (not documented on pages fetched: [Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)).", "api_images_note": "n.a. (same pages, not documented).", "api_audio_note": "n.a. (same pages, not documented).", "api_batch_note": "n.a. (same pages, not documented).", "deployment_modes_note": "Hosted SaaS (`https://ai-gateway.helicone.ai/ai`), self-host (local binary via `npx @helicone/ai-gateway@latest`, Docker/Docker Compose, Kubernetes/Helm, cloud deployment on AWS/GCP/Azure) ([GitHub](https://github.com/Helicone/ai-gateway), [Self-hosting overview](https://docs.helicone.ai/getting-started/self-host/overview)); on-prem is implied by the `NEXT_PUBLIC_IS_ON_PREM` flag ([Docker self-host](https://docs.helicone.ai/getting-started/self-host/docker)). Hybrid/VPC and air-gapped: not documented.", "byok_mode_note": "both: BYOK keys are tried first with Helicone-managed keys as fallback, and credits give access to \"100+ other providers without signing up for each one\" at \"0% markup\" ([Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Gateway overview](https://docs.helicone.ai/gateway/overview)).", "access_meta": { "qualified": [], "model_count_kind": "vendor_total", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` as a settable value. Timeout appears only as a failover trigger: `408` \"Timeout errors\" causes the gateway to try the next provider ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)). Retry headers include `Helicone-Retry-Min-Timeout` / `Helicone-Retry-Max-Timeout`, but these bound backoff, not the request ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).", "rel_retry_config": "per_request", "rel_retry_note": "`per_request` via headers: `Helicone-Retry-Enabled`, `Helicone-Retry-Num` (default `5`), `Helicone-Retry-Factor` (default `2`), `Helicone-Retry-Min-Timeout` (default `1000` ms), `Helicone-Retry-Max-Timeout` (default `10000` ms). Backoff is exponential: 1s → 2s → 4s → 8s → 10s, on `429`, `500`, `502`, `503`, `504` ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).", "rel_retry_default_note": "`per_request` via headers: `Helicone-Retry-Enabled`, `Helicone-Retry-Num` (default `5`), `Helicone-Retry-Factor` (default `2`), `Helicone-Retry-Min-Timeout` (default `1000` ms), `Helicone-Retry-Max-Timeout` (default `10000` ms).", "rel_fallback_config": "per_request", "rel_fallback_note": "`per_request` for the cloud gateway: ORDERED. `Helicone-Fallbacks` JSON header with `target-url`, `onCodes` and `bodyKeyOverride` per entry; the response header `Helicone-Fallback-Index` reports which entry served ([Helicone gateway fallbacks](https://docs.helicone.ai/getting-started/integration-method/gateway-fallbacks)). In the AI Gateway the ordered chain is expressed inside the `model` string, e.g. `model: \"gpt-4o-mini/azure,gpt-4o-mini/openai,gpt-4o-mini\"`, \"tried in the exact order you specify\" ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "`not_configurable`, and it is passive unhealthy-provider avoidance rather than a configurable circuit breaker: the gateway is \"always aware of provider uptimes and your rate limits\" and instantly tries the next provider on `429`, `401`, `400`, `408`, `500+` ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway), [Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "`not_documented` as failover. You can pin a specific regional deployment (`model: \"gpt-4o/azure/eu-frankfurt-deployment\"`, \"No data leaves the EU\"), but that is region pinning, not cross-region failover ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights ARE supported. YAML key `routers..load-balance.chat.strategy` with strategies: `model-latency`, provider latency P2C + PeakEWMA, weighted distribution based on model weight, and cost optimization ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)). In the cloud gateway, \"equal-cost providers are load balanced\" and BYOK keys are always tried before Helicone managed keys ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).", "architecture_class": "compiled_binary", "architecture_note": "\"built on Rust\"; repo language breakdown Rust 96.7% ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).", "self_host_data_plane": "yes", "self_host_artifact_note": "`yes`. npm-launched binary `npx @helicone/ai-gateway@latest --config config.yaml`, plus Docker and Kubernetes self-hosting ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).", "latency_claims": [ { "url": "https://github.com/Helicone/ai-gateway", "unit": "ms", "scope": "gateway_overhead", "value": "<5", "metric": "p95", "publisher": "vendor", "conditions": "README comparison table. Whether this is added overhead or full round trip is not stated; no RPS, payload, hardware or cache state given." }, { "url": "https://github.com/Helicone/ai-gateway", "unit": "ms", "scope": "gateway_overhead", "value": "~100", "metric": "cold start", "publisher": "vendor", "conditions": "Same README table, against a \"Typical Setup\" baseline of ~2s. Conditions not stated." } ], "throughput_claims": [ { "url": "https://github.com/Helicone/ai-gateway", "unit": "req/s", "scope": "capacity", "value": "~3,000", "metric": "sustained", "publisher": "vendor", "conditions": "README table against a \"Typical Setup\" baseline of ~500. Hardware not stated." } ], "streaming_support": "yes", "streaming_note": "Supported; the fetched pages do not state a streaming-specific caveat for the AI Gateway ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)). Note the retry headers are documented independently of streaming ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).", "guard_pii": "not_documented", "guard_pii_note": "No PII scanner documented; the privacy control is body omission rather than detection ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))", "guard_injection": "not_documented", "guard_injection_note": "`not_documented` ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))", "guard_content": "sync_block", "guard_content_note": "With the header `Helicone-Moderations-Enabled: true`, Helicone calls OpenAI's moderation endpoint before the chat completion and returns HTTP 400 with `PROMPT_FLAGGED_FOR_MODERATION` when flagged, so the request never reaches the model ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))", "guard_custom": "not_documented", "guard_custom_note": "No customer-defined regex/word-list/JSON-schema policy engine documented ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))", "guard_model_policy": "not_documented", "guard_model_policy_note": "The gateway router config selects load-balance targets and rate limits per key, but no admin allow/deny policy on models is documented ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))", "guard_execution_location": "both", "guard_third_party": [ "OpenAI Moderation" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "No stated behaviour if the moderation call fails or times out ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))", "log_content": "full_request_and_response", "log_content_note": "Request and response bodies (prompt and completion text) are logged unless omission is requested ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))", "log_opt_out": "yes", "log_opt_out_note": "Headers `Helicone-Omit-Request: true` and `Helicone-Omit-Response: true`, or `disable_content_tracing()` in the SDK, keep metrics while dropping bodies ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))", "tracing_support": "otel", "tracing_note": "The AI Gateway repo states \"OpenTelemetry support for logs, metrics, and traces\" alongside built-in Helicone observability; representation of multi-step agent traces is not detailed on the fetched pages ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))", "export_destinations": [ "OpenTelemetry", "Datadog", "Webhook", "PostHog" ], "export_destinations_note": "PostHog via `Helicone-Posthog-Key` / `Helicone-Posthog-Host` headers ([PostHog integration](https://docs.helicone.ai/gateway/integrations/posthog)); OpenTelemetry logs/metrics/traces from the gateway ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))", "feedback_capture": "yes", "feedback_capture_note": "`POST /v1/request/{id}/feedback` with a boolean `rating` ([Feedback](https://docs.helicone.ai/features/advanced-usage/feedback)); arbitrary numeric scores via `POST /v1/request/{requestId}/score` ([Scores](https://docs.helicone.ai/features/advanced-usage/scores))", "eval_hooks": "partial", "eval_hooks_note": "Scores can be pushed against logged requests, but the docs state plainly that \"Helicone doesn't run evaluations for you\" — you compute scores externally and post them ([Scores](https://docs.helicone.ai/features/advanced-usage/scores))", "observability_requires_saas": "The gateway is fully open-source and self-hostable, and the repo presents observability as part of it; the fetched pages do not enumerate which dashboard features are cloud-only ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))", "log_retention_note": "Plan-based and configurable only by upgrading: Hobby 7 days, Pro 1 month, Team 3 months, Enterprise forever ([Helicone pricing](https://www.helicone.ai/pricing))", "production_meta": { "sources": {}, "qualified": [ "guard_content" ], "latency_prose": "README \"Scalable for production\" table: **P95 latency `<5ms`** for Helicone AI Gateway vs `~60-100ms` for a \"Typical Setup\"; **cold start `~100ms`** vs `~2s`. What was measured is not stated (gateway overhead vs round trip is ambiguous), and no RPS, payload, hardware or cache conditions are given — the README defers to `benchmarks/README.md` ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).", "multi_surface": {}, "throughput_prose": "Same table: **`~3,000` requests/sec** vs `~500` for a \"Typical Setup\". Hardware and conditions not stated ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).", "benchmark_independence": "All vendor-published and self-measured. **Flag:** the comparison column (\"Typical Setup\", `~60-100ms`, `~500` req/s) is a vendor figure about unnamed competing setups, with no methodology or named product — treat it as marketing, not a benchmark ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "off_by_default", "guard_custom_default": "not_documented", "rel_notes": "Two distinct products share the brand: the header-driven cloud proxy (`Helicone-Retry-*`, `Helicone-Fallbacks`) and the Rust AI Gateway with YAML routers. Their config surfaces do not overlap, so docs for one do not describe the other ([Helicone gateway fallbacks](https://docs.helicone.ai/getting-started/integration-method/gateway-fallbacks), [Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).", "guard_notes": "Helicone's one true request-path control is moderation, and it is opt-in per request via a header rather than an org-wide policy — a per-call flag is easy for a client to omit ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations)).", "fit_agent_features_note": "Helicone documentation shows tool/function calling, session tracking, and grouping related API calls. The Vercel AI SDK integration documents multi-step orchestration using `Experimental_Agent` with `stopWhen: stepCountIs(5)`, while A2A is not documented. ([Helicone Vercel AI SDK integration](https://docs.helicone.ai/gateway/integrations/vercel-ai-sdk))", "fit_byok_required": "hosted_no_selfhost_yes", "fit_byok_required_note": "No for the hosted gateway (\"Want more control? You can bring your own provider keys instead,\" presented as optional) ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)). Yes in the self-hosted Docker path, whose first call sends `Authorization: Bearer $OPENAI_API_KEY` alongside `Helicone-Auth` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Documented clients and packages include the OpenAI Python package (`openai`, `from openai import OpenAI`), the OpenAI JavaScript package (`openai`), `@helicone/ai-sdk-provider`, `@ai-sdk/gateway`, `@ai-sdk/openai`, `llama-index-llms-helicone`, `boto3`, and `helicone-async`. Bedrock examples also document the classes `BedrockRuntimeClient`, `InvokeModelCommand`, `ConverseCommand`, and `HeliconeAsyncLogger`. ([Helicone client integrations](https://docs.helicone.ai/integrations/openai/python))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "vendor_runtime_only", "fit_cloudflare_workers_note": "Helicone states that its proxy leverages Cloudflare Workers running on Cloudflare’s global network to process requests at the edge. ([Helicone latency reference](https://docs.helicone.ai/references/latency-affect))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated as required for the first call; the signup link is labeled \"Sign up for free\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)) and pricing lists \"Hobby — Free\" plus \"Pro — 7-day free trial\" ([Helicone Pricing](https://www.helicone.ai/pricing))", "fit_ecosystem_notes": "Helicone documents hosted gateways including `https://ai-gateway.helicone.ai`, access to 100+ models or providers, logging and observability, routing, fallbacks, and BYOK through Helicone Providers. It also explicitly documents self-hosting a complete Helicone stack on Kubernetes/Helm, with AWS S3, Aurora, and EKS resources supported through Terraform; some older OpenAI integration pages are described as maintained but no longer actively developed. ([Helicone quick start](https://docs.helicone.ai/getting-started/quick-start))", "fit_eval_prereqs": "A Helicone account plus a Helicone API key. No credit card, cloud account, cluster, or provider key: \"Helicone maintains the keys for you.\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, in TypeScript, Python and curl: `POST https://ai-gateway.helicone.ai/chat/completions` with `Authorization: Bearer $HELICONE_API_KEY` and `{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, world!\"}]}` ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "helicone-core, helicone-infrastructure, helicone-monitoring, helicone-argocd", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Helicone explicitly supports self-hosting on Kubernetes and provides an official Helm chart. The chart repository is the Helicone Helm repository, with components including `helicone-core`, `helicone-infrastructure`, `helicone-monitoring`, and `helicone-argocd`; the Helm deployment uses `helm upgrade --install helicone ./helicone -f values.yaml`. ([Helicone Kubernetes self-hosting](https://docs.helicone.ai/getting-started/self-host/kubernetes))", "fit_mcp": "client_tooling", "fit_mcp_note": "Helicone provides an MCP server available as the npm package `@helicone/mcp`, including the tools `query_requests` and `query_sessions`. It supports MCP-compatible clients such as Claude Desktop, Claude Code, Cursor, and Codex, and `query_sessions` provides session search, filtering, pagination, and session performance metrics. ([Helicone MCP integration](https://docs.helicone.ai/integrations/tools/mcp))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_gateway_platform" ] }, "fit_model_gate": "none", "fit_model_gate_note": "n.a. (not documented). No approval/quota/waitlist step; users can \"Switch between 100+ models by just changing the model name\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))", "fit_onboarding_notes": "Hosted quickstart needs no library install at all (curl works). Self-hosted is materially narrower: \"Self-hosted version supports OpenAI and Anthropic only,\" port 8585 \"has no proxy authentication; firewall restriction is required,\" and email verification must be performed manually after creating an account at `/signup` ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start), [Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Helicone documents OpenAI SDK-compatible integrations by changing the endpoint and authentication. Documented base URLs include `https://oai.helicone.ai/v1`, `https://ai-gateway.helicone.ai`, `https://ai-gateway.helicone.ai/v1`, and `https://vercel.helicone.ai/v1`, depending on the integration. ([Helicone OpenAI Python integration](https://docs.helicone.ai/integrations/openai/python))", "fit_prod_prereqs": "Hosted: credits on the account (\"You simply add credits to your account, and we handle the rest\") ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)). Self-hosted: Docker; ports 3000/8585/9080 reachable; internal PostgreSQL (5432) and ClickHouse (8123); MinIO S3 storage; public IP/domain for remote deployments; Docker volumes for persistence; a reverse proxy for HTTPS; and production env vars `SITE_URL`, `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `NEXT_PUBLIC_APP_URL`, `NEXT_PUBLIC_HELICONE_JAWN_SERVICE`, `NEXT_PUBLIC_IS_ON_PREM=true`, `S3_ENDPOINT` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))", "fit_prod_selfhost_shape": "cli_or_container", "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LlamaIndex" ], "fit_python_frameworks_note": "LlamaIndex is documented with the packages `llama-index-llms-helicone`, `llama_index.core`, and `llama_index.llms.openai`, including the `Helicone` and `OpenAI` classes. LangChain and LlamaIndex are also discussed in a comparison blog post, which mentions `PromptTemplate`, `LLMChain`, `IndexToolConfig`, and `LlamaIndexTool`; the LangChain details are from that comparison post rather than a dedicated integration page. ([Helicone LlamaIndex integration](https://docs.helicone.ai/gateway/integrations/llamaindex))", "fit_quickstart_steps": 2, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.helicone.ai/getting-started/quick-start", "fit_self_host_docs": "command_published", "fit_self_host_install": "`docker pull helicone/helicone-all-in-one:latest` then `docker run -d --name helicone -p 3000:3000 -p 8585:8585 -p 9080:9080 helicone/helicone-all-in-one:latest` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))", "fit_terraform": "shipped_code", "fit_terraform_note": "Helicone documents Terraform directories for creating AWS S3, Aurora, and EKS resources: `terraform/s3`, `terraform/aurora`, and `terraform/eks`. An official Helicone Terraform provider, module, or Terraform Registry name is not stated. ([Helicone Kubernetes self-hosting](https://docs.helicone.ai/getting-started/self-host/kubernetes))", "fit_terraform_ref": "terraform/s3, terraform/aurora and terraform/eks directories in the repo; no provider or module published", "fit_third_party_documented": {}, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get your first LLM request logged with Helicone in under 2 minutes using the AI Gateway.\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Helicone documents Vercel AI SDK integrations using `@helicone/ai-sdk-provider` with `createHelicone`, `@ai-sdk/openai` with `createOpenAI`, and `@ai-sdk/gateway` for gateway-specific functionality. ([Helicone Vercel AI SDK integration](https://docs.helicone.ai/gateway/integrations/vercel-ai-sdk))", "fit_vercel_ai_sdk_package": "@helicone/ai-sdk-provider" }, { "slug": "kong-ai-gateway", "name": "Kong AI Gateway", "company": "Kong Inc.", "tagline": "AI plugins on the Kong API gateway, adding LLM routing, guardrails and token limits to existing API infrastructure.", "category": "managed_gateway", "deployment": "both", "license": "Apache-2.0", "founded_year": 2009, "hq_country": "United States", "url_home": "https://konghq.com/products/kong-ai-gateway", "url_docs": "https://developer.konghq.com/ai-gateway/", "url_pricing": "https://konghq.com/pricing", "url_status": "https://status.konghq.com", "url_github": "https://github.com/Kong/kong", "url_changelog": null, "url_models_api": null, "github_stars": 44100, "model_count": null, "model_count_note": "No total model count is published; the Konnect Plus plan caps the AI Gateway at 5 unique LLM models, with more available on Enterprise.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "You supply your own upstream LLM credentials, stored centrally in Kong's config store; Kong charges for gateways, not tokens.", "seat_fee_usd_month": null, "free_tier": "30-day Konnect free trial with enterprise functionality; separately, Kong Gateway core is Apache-2.0 and free to self-host.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "Kong Gateway core is Apache-2.0 and self-hostable at infra cost only. The Plus plan is billed per gateway per month with the amount not published; fully self-hosted gateways plus audit logs and SSO require the custom-priced, annually billed Enterprise plan. Third-party reviews note the OSS build lacks semantic caching, analytics and compliance features found in Enterprise.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Embeddings", "MCP", "A2A" ], "sdk_note": "No LLM SDK of its own: traffic goes through Kong routes using the AI Proxy / AI Proxy Advanced plugins, exposing a universal OpenAI-style API.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": null, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 12, "latency_source_note": "Vendor-run: Kong's own benchmark (Claudio Acquaviva, Kong, 2025-07-07) compared Kong AI Gateway with Portkey OSS 1.9.19 and LiteLLM 1.63.7 on EKS c5.4xlarge nodes, 12 CPUs each, K6 with 400 VUs against a WireMock LLM, claiming Kong 228% faster than Portkey and 859% faster than LiteLLM with 65%/86% lower latency. A third-party guide summarizes that benchmark as 12 ms p50 gateway overhead and 8,200 RPS, and cautions that independent benchmarks show much smaller gaps (50-100%, not 228%) and that Kong's figures are best-case.", "throughput_rps": 8200, "soc2": true, "hipaa_baa": null, "gdpr": true, "eu_residency": null, "eu_residency_note": "A third-party comparison notes Kong offers an EU control-plane option and that prompt-carrying data planes can stay in customer infrastructure; Kong's own trust center does not state an EU residency guarantee.", "zdr": null, "zdr_note": "No ZDR commitment is published. The practical equivalent comes from the defaults: payload logging is off, and self-hosted logs go to storage you control.", "air_gapped": null, "best_for": "Enterprises already standardized on Kong for API management that want AI traffic governed by the same gateway, plugins and ops tooling.", "strengths": [ "Battle-tested NGINX/OpenResty core with 44k GitHub stars and a decade of production plugin patterns", "Deep AI feature set: semantic caching and routing, RAG injector, prompt templates, MCP and A2A gateways", "Strong data-protection tooling: PII sanitizer across 20 categories and 9 languages, plus Azure/AWS/GCP/Lakera guardrail integrations", "Clear compliance posture: SOC 2 Type 2, GDPR, CCPA, PCI DSS, CSA STAR Level 1, NIST 800-218", "Reuses existing Kong auth, rate limiting, logging and OpenTelemetry pipelines for LLM traffic" ], "weaknesses": [ "No pricing transparency: Plus is quoted per gateway per month with no published amount and Enterprise is custom", "Plus caps the AI Gateway at 5 unique LLM models, pushing broader model access to Enterprise", "Its headline performance advantage comes from a Kong-run benchmark against mock LLMs; a third-party guide says independent tests show far smaller gaps", "Third-party reviews report the free OSS build lacks semantic caching, analytics and compliance features" ], "avoid_if": "You want a lightweight LLM-first proxy with published prices and hundreds of models available out of the box without an enterprise contract.", "reviews": [ { "url": "https://siliconangle.com/2025/04/02/kongs-updated-ai-gateway-helps-secure-ai-model-production-deployments/", "why": "Independent coverage of AI Gateway 3.10's RAG injector and PII-sanitization plugins and what they mean for teams putting model traffic through Kong in production.", "date": "2025-04-02", "type": "written", "title": "Kong's updated AI Gateway helps to secure AI model production deployments", "publisher": "SiliconANGLE" }, { "url": "https://techcrunch.com/2024/02/15/kongs-new-open-source-ai-gateway-makes-building-multi-llm-apps-easier/", "why": "Trade-press analysis of the AI Gateway launch, explaining how Kong bolted multi-LLM routing, credential management and request/response transformers onto its existing gateway.", "date": "2024-02-15", "type": "written", "title": "Kong's new open source AI Gateway makes building multi-LLM apps easier", "publisher": "TechCrunch" }, { "url": "https://www.youtube.com/watch?v=sWI4drhs0lc", "why": "Independent engineer configures Kong AI Gateway from scratch, wires up OpenAI/Anthropic/Azure providers, and shows load balancing, the prompt-decorator plugin and analytics.", "date": "2025-12-14", "type": "video", "title": "KONG AI GATEWAY | MODEL ABSTRACTION", "publisher": "YouTube - Rajib Deb" }, { "url": "https://www.reddit.com/r/LLMDevs/comments/1v2mbno/litellm_alternatives_after_prod_outage_anyone/", "why": "9-comment thread from teams evaluating Kong as a LiteLLM replacement after a multi-hour routing outage, with operators reporting real cluster sizes and upgrade pain.", "date": "2026-07-21", "type": "community", "title": "LiteLLM alternatives after prod outage, anyone running TrueFoundry or Kong for 3+ months?", "publisher": "Reddit - r/LLMDevs" }, { "url": "https://www.reddit.com/r/kubernetes/comments/1themkz/migrating_away_from_the_kong_enterprise_stack/", "why": "15-comment postmortem of a Kong Enterprise/Kong Mesh exit: shockingly high CPU/memory under API-key rate limiting and ~$70k/year saved after a month-long migration to Envoy Gateway.", "date": "2026-05-19", "type": "community", "title": "Migrating Away from the Kong Enterprise Stack", "publisher": "Reddit - r/kubernetes" } ], "sources": { "gdpr": "https://trust.konghq.com/", "soc2": "https://trust.konghq.com/", "cap_mcp": "https://konghq.com/pricing", "company": "https://konghq.com/about", "license": "https://github.com/Kong/kong", "sdk_note": "https://developer.konghq.com/ai-gateway/", "free_tier": "https://konghq.com/pricing", "hq_country": "https://konghq.com/about", "url_status": "https://status.konghq.com", "weaknesses": "https://dev.to/pranay_batta/best-open-source-ai-gateway-in-2026-2flb", "api_surfaces": "https://developer.konghq.com/ai-gateway/", "cap_failover": "https://developer.konghq.com/ai-gateway/", "founded_year": "https://konghq.com/about", "github_stars": "https://github.com/Kong/kong", "kong_pricing": "https://konghq.com/pricing", "product_fate": "https://developer.konghq.com/gateway/changelog/", "regions_note": "https://developer.konghq.com/konnect-platform/geos/", "vendor_status": "https://konghq.com/company/press-room/press-release/kong-to-unveil-ai-connectivity-vision-and-roadmap", "byok_supported": "https://developer.konghq.com/ai-gateway/", "cap_embeddings": "https://developer.konghq.com/ai-gateway/", "cap_guardrails": "https://developer.konghq.com/ai-gateway/", "certifications": "https://trust.konghq.com/", "config_as_code": "https://developer.konghq.com/deck/file/format/", "retention_days": "https://konghq.com/pricing", "sla_uptime_pct": "https://konghq.com/pricing", "throughput_rps": "https://apiscout.dev/guides/portkey-vs-kong-ai-gateway-llm-routing-2026", "cap_rate_limits": "https://konghq.com/pricing", "logs_by_default": "https://docs.jp.konghq.com/hub/kong-inc/ai-proxy/configuration/", "model_count_note": "https://konghq.com/pricing", "cap_observability": "https://developer.konghq.com/ai-gateway/", "cap_pii_redaction": "https://developer.konghq.com/ai-gateway/", "eu_residency_note": "https://opper.ai/blog/best-european-ai-gateways", "last_release_date": "https://developer.konghq.com/gateway/changelog/", "openai_compatible": "https://developer.konghq.com/ai-gateway/", "subprocessors_url": "https://konghq.com/legal/subprocessors", "cap_cache_semantic": "https://developer.konghq.com/ai-gateway/", "cap_load_balancing": "https://developer.konghq.com/ai-gateway/", "guardrails_location": "https://developer.konghq.com/plugins/ai-sanitizer/", "latency_overhead_ms": "https://apiscout.dev/guides/portkey-vs-kong-ai-gateway-llm-routing-2026", "latency_source_note": "https://konghq.com/blog/engineering/ai-gateway-benchmark-kong-ai-gateway-portkey-litellm", "requires_vendor_sdk": "https://developer.konghq.com/plugins/ai-proxy/", "self_host_cost_note": "https://konghq.com/pricing", "cap_prompt_management": "https://developer.konghq.com/ai-gateway/", "data_export_available": "https://developer.konghq.com/plugins/http-log/", "prompt_transits_vendor": "https://developer.konghq.com/gateway/deployment-topologies/", "cap_conditional_routing": "https://developer.konghq.com/ai-gateway/" }, "verified_at": { "gdpr": "2026-08-29", "soc2": "2026-08-29", "cap_mcp": "2026-08-29", "company": "2026-08-29", "license": "2026-09-14", "sdk_note": "2026-08-29", "free_tier": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "weaknesses": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "github_stars": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "throughput_rps": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_note": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-14T20:21:32.795242+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed, with weekly gateway releases and new MCP governance features. Kong opened Paris and Milan offices in July 2026.", "last_release_date": "2026-08-26", "sla_uptime_pct": 99.9, "sla_url": "https://konghq.com/legal/service-level-agreement", "claim_basis": {}, "last_release_note": "Kong Gateway 3.14.0.14. The open-source Kong/kong repository moves more slowly: latest tag 3.9.3 on 17 June 2026.", "prompt_transits_vendor": "depends", "prompt_path_note": "The data plane is yours to run, or Kong-hosted as serverless or dedicated cloud gateways. Konnect's control plane is Kong-hosted in a geography you pick.", "logs_by_default": "none", "logs_note": "The AI Proxy plugin ships with statistics and payload logging both set to false, so nothing about your prompts is recorded until you turn it on and point it at a sink you own.", "retention_days": 30, "retention_note": "Thirty days for Konnect analytics and seven for audit logs. Prompt bodies are not retained at all unless you enable payload logging.", "zdr_status": "not_published", "trains_on_customer_data": "not_published", "training_note": "Nothing on the trust center or subprocessor page addresses training.", "subprocessors_url": "https://konghq.com/legal/subprocessors", "regions_note": "Konnect control-plane geographies in Australia, the EU, the Middle East, the US, India and Singapore, with dedicated data planes across roughly 17 AWS, 13 Azure and 11 Google regions. Self-hosted and on-premises data planes are supported.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type 2, via trust center", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "listed on the trust center", "status": "certified" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "self_hosted", "guardrails_note": "The PII sanitizer calls an anonymizer service you run yourself, typically in a Docker container, so redaction happens inside your boundary.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "platform_fee", "pricing_model_note": "Flat per-gateway platform fee (Konnect Plus, no dollar figure published) + enterprise-only annual quote. Kong does not resell inference. Plans billed on a calendar month basis in arrears.", "min_commitment": "Enterprise 'billed annually' with duration set out in the individual Order Form; no minimum amount stated. Plus is monthly.", "overage_note": "No metered overage billing published. Plus enforces soft limits via email + in-app notifications prompting you to reduce usage or upgrade.", "cache_support": "semantic", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Semantic caching included in Plus under 'Cost Control'. Exact-match and provider passthrough not stated. Kong prices no tokens, so semantic cache hits save on the customer's provider bill; cached requests may still count toward Plus request/analytics limits.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated (only resource-limit email/in-app notices).", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Token-based rate limiting, enforced pre-request, included in Plus.", "tier": "plus", "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated. Nearest published control is 'LLM access control and auth' plus a 5-unique-model Plus ceiling.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Plus includes 'AI Observability: token-level tracking and real-time cost analytics'. Per key/user/team/tag/customer splits not stated.", "cost_export": "None stated on the pricing page.", "product_kind_note": "A routing/governance proxy layered on an API gateway: \"Connectivity and governance layer for modern AI-native applications built on top of Kong Gateway\" that \"routes AI requests to various providers through a provider-agnostic API\"; the AI Proxy plugin \"lets you transform and proxy requests to a number of AI providers and models\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "deployment_modes": [ "saas", "self_host", "hybrid_vpc", "on_prem" ], "deployment_note": "Konnect provides \"a unified control plane\" while customers \"Decide where you want to host your Data Plane nodes, and whether you want Kong to host them or host them yourself\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)) — i.e. customer-cloud data planes with a SaaS control plane, plus fully self-managed Kong Gateway. Feature gating is by Kong Gateway version rather than deployment (retries/fallback in load balancing require v3.10+; REST-style responses/files/assistants/batches require v3.11+; native `llm_format` pass-through requires v3.10+) ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Provider availability also varies: \"Some providers may not be available depending on your Kong Gateway version\" ([AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Not a base-URL-swap SaaS: you configure the plugin on your own Kong routes, then call your gateway with an OpenAI-compatible SDK (\"You can combine these parameters with an OpenAI-compatible SDK…\") or a provider-native SDK in native `llm_format` mode ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Native formats supported: `anthropic`, `bedrock`, `cohere`, `gemini`, `huggingface` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Caveats: \"some providers don't support all route types\"; provider-specific params must go in `extra_body`; streaming is listed as a supported capability area without further detail ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a.: \"Number of models: not mentioned\" on the plugin page ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)); Kong ships no model catalog of its own on pages fetched.", "provider_count_low": 17, "provider_count_high": 17, "provider_count_source_note": "No vendor total stated; the AI Proxy plugin page lists 17 providers (OpenAI, Azure OpenAI, Amazon Bedrock, Anthropic, Gemini, Vertex AI, Cohere, Mistral, Hugging Face, Llama, xAI, Alibaba Cloud DashScope, Cerebras, DeepSeek, Ollama, Databricks, vLLM) and the providers index lists the same 17 names; both pages undated ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).", "first_party_vs_partner": "Entirely third-party/self-hosted upstreams: Kong routes to \"various providers\" and \"upstream LLMs\", including self-hosted models fulfilled \"using select supported format transformations\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). No Kong-operated models.", "custom_endpoint_support": "yes in practice: Ollama, vLLM, Llama and Hugging Face are first-class provider options and the plugin explicitly fulfils requests to \"self-hosted models\" ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). A generic \"register any private URL\" flow is not spelled out on the pages fetched.", "byok_mode": "byok_only", "merchant_of_record": "The upstream provider or cloud, since the customer's own keys/IAM are used (Azure Managed Identity, AWS \"IAM credentials or assumed roles\") ([AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). Kong itself is positioned to help you bill your own consumers: \"Meter, bill, and monetize the entire AI connectivity data path… automate invoicing with Stripe or ERP integrations\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).", "key_management_note": "Keys are stored/referenced through Konnect Config Store, or replaced by cloud identity (Azure `azure_use_managed_identity`, `azure_client_id`; AWS IAM credentials/assumed roles); request-level API keys or bearer tokens can override static plugin values when `config.auth.allow_override` is true ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Because the data plane can be customer-hosted — \"Running privately and self-hosted for full control and compliance\" — request content need not leave the customer's infrastructure ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).", "api_openai_chat_completions_note": "yes in effect: the plugin accepts requests in \"defined and standardized OpenAI formats\", uses \"the OpenAI format by default\", and exposes route type `llm/v1/chat` mapped to OpenAI Chat completions; the literal `/v1/chat/completions` path is the customer's own Kong route, not a fixed vendor path ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_anthropic_messages_note": "yes as native pass-through: setting `config.llm_format` to `anthropic` proxies requests upstream \"without payload format conversion\"; the literal `/v1/messages` path is not stated ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_openai_responses_note": "yes: route type `llm/v1/responses`, mapped to OpenAI \"Responses\" (v3.11+ for the REST-based full-text responses set) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_embeddings_note": "yes: route type `llm/v1/embeddings` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_images_note": "yes: `image/v1/images/generations` and `image/v1/images/edits` (plus `video/v1/videos/generations`) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_audio_note": "yes, both: `audio/v1/audio/speech` (TTS), `audio/v1/audio/transcriptions` (STT), `audio/v1/audio/translations` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "api_batch_note": "yes: `llm/v1/batches` and `llm/v1/files` support \"asynchronous bulk LLM requests\" with CRUD via POST/GET/DELETE (v3.11+) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).", "deployment_modes_note": "Hosted control plane (Konnect SaaS), self-hosted traditional, hybrid, DB-less, and Kubernetes via Kong Ingress Controller — \"AI plugins are supported in all deployment modes, including Konnect, self-hosted traditional, hybrid, and DB-less, and on Kubernetes via the Kong Ingress Controller\"; an on-prem demo instance is launched with `curl -Ls https://get.konghq.com/ai/v1 | bash` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)). Air-gapped: not documented on pages fetched.", "byok_mode_note": "byok_only: Kong sells software/subscription, not model tokens; credentials are the customer's own provider keys, managed centrally (\"Use Konnect Config Store to store and reference your LLM provider API keys\", \"Centralized AI provider credential management\") ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). The term \"BYOK\" itself is not used.", "access_meta": { "qualified": [ "api_openai_chat_completions", "api_anthropic_messages", "api_audio" ], "model_count_kind": null, "provider_count_kind": "enumerated" }, "rel_timeout_config": "config_file", "rel_timeout_note": "Upstream/service timeouts `connect_timeout`, `read_timeout`, `write_timeout` are configurable, and timeout is a first-class failover condition ([Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).", "rel_retry_config": "config_file", "rel_retry_note": "Key `retries`, which retries on both error and timeout. Default retry count and backoff strategy are not stated on the fetched pages: `n.a.` ([Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).", "rel_retry_default_note": null, "rel_fallback_config": "config_file", "rel_fallback_note": "ORDERED and WEIGHTED are both available — the AI Proxy Advanced plugin's target list supports `weight` (documented example 70/25/5), and failover is controlled by `failover_criteria`, which defaults to error and timeout and can add `http_429`, `http_500` and `non_idempotent` ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/), [Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).", "rel_fallback_shape": "weighted", "rel_health_check_config": "config_file", "rel_health_check_note": "`config_file`, and it is a real circuit breaker: from v3.13+ `config.balancer.max_fails` with `config.balancer.fail_timeout` ejects a failing upstream for a period ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)). Kong Gateway itself lists \"health checking\" as configurable through the admin API or declarative config ([Kong/kong](https://github.com/Kong/kong)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "`not_documented` as a named feature. You self-host the data plane wherever you like (Linux, Docker, Kubernetes, or Konnect's managed control plane), but no cross-region failover config is documented ([Kong Gateway install](https://developer.konghq.com/gateway/install/)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights supported (`weight` on upstream targets). Strategies include lowest-latency with `latency_strategy` (`tpot` default, or `e2e`) and peak-EWMA, plus consistent hashing via `hash_on_header` (default header `X-Kong-LLM-Request-ID`) ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)).", "architecture_class": "interpreted_proxy", "architecture_note": "Repo language breakdown: Lua 89.2%, Perl 5.2%, Raku 3.1% ([Kong/kong](https://github.com/Kong/kong)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Prebuilt `deb` and `rpm` packages, official Docker Hub images including a distroless image (AMD64 and ARM64), and Helm charts for Kong Gateway and Kong Ingress Controller ([Kong Gateway install](https://developer.konghq.com/gateway/install/)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported, including WebSocket realtime traffic. Documented caveat: the lowest-latency balancing algorithm is \"less suitable for long-lived connections like WebSockets\" ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)).", "guard_pii": "sync_block", "guard_pii_note": "The AI PII Sanitizer plugin sends payloads to a customer-run `ai-pii-service` anonymiser covering ~20 categories, replaces matches with placeholders or synthetic values, and can restore originals on the response path; audit logs record `ai.sanitizer.pii_identified`, `pii_sanitized`, and per-entity original/redacted pairs ([AI Sanitizer plugin](https://developer.konghq.com/plugins/ai-sanitizer/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))", "guard_injection": "sync_block", "guard_injection_note": "Kong itself ships regex prompt guarding; injection/jailbreak classification comes from plugged-in services such as Lakera Guard, whose `input_block_reason` / `output_block_reason` audit fields record blocked requests ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "guard_content": "sync_block", "guard_content_note": "The AI Azure Content Safety plugin \"analyzes the request against configured moderation categories and allows or blocks the request\", and does the same for responses; GCP Model Armor input/output block reasons are also logged ([AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))", "guard_custom": "sync_block", "guard_custom_note": "`ai-prompt-guard` takes PCRE allow and deny pattern lists and returns HTTP 400 on a deny match; Azure Content Safety blocklist IDs and per-category thresholds are also configurable ([AI Prompt Guard](https://developer.konghq.com/plugins/ai-prompt-guard/), [AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages — routing and plugin scoping are per route/service/consumer, but an explicit model allow/deny list feature is not documented in what was fetched ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy reference](https://developer.konghq.com/plugins/ai-proxy/reference/))", "guard_execution_location": "both", "guard_third_party": [ "AWS Bedrock Guardrails", "Azure Content Safety", "Google Model Armor", "Lakera" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` — no fail-open/fail-closed statement on the fetched guardrail plugin pages ([AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/), [AI AWS Guardrails reference](https://developer.konghq.com/plugins/ai-aws-guardrails/reference/))", "log_content": "configurable", "log_content_note": "AI plugin logging of statistics and payloads is controlled per plugin; Kong's AI audit log records guardrail verdicts and, for the sanitizer, the original and redacted entity values ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))", "log_opt_out": "yes", "log_opt_out_note": "Payload logging is a plugin config toggle, so metrics can be kept without bodies ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))", "tracing_support": "otel", "tracing_note": "OpenTelemetry is native to Kong: the `opentelemetry` plugin exports OTLP-over-HTTP spans (with AI span attributes and metrics in the AI Gateway) and supports full request-lifecycle tracing ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "export_destinations": [ "OpenTelemetry", "Prometheus", "Datadog", "StatsD", "HTTP log", "Kafka", "Loki" ], "export_destinations_note": "OpenTelemetry collectors/OTLP-HTTP endpoints for traces and metrics; Kong's AI audit log is emitted through Kong's logging plugins ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/)). A full enumerated destination list was `n.a.` on the fetched pages.", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — No feedback/score API documented ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` — No eval/online-scoring feature documented ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "observability_requires_saas": "No — Kong runs in the customer's infrastructure and exports telemetry to the customer's own OTel/metrics stack ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "log_retention_note": "`n.a.` — Retention is a property of whatever sink the customer ships to; not stated ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` No gateway latency figure appears on the fetched Kong pages ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).", "multi_surface": {}, "throughput_prose": "`n.a.` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).", "benchmark_independence": "No figures published ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))." }, "guard_pii_default": "sync_block", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Kong is the only product here where the LLM-specific reliability config inherits a mature general-purpose proxy stack (upstream entities, retries, circuit breaker, consistent hashing), and the hybrid control-plane/data-plane deployment model is documented ([Kong/kong](https://github.com/Kong/kong)).", "guard_notes": "Kong is the clearest example of guardrails as first-class request-path plugins with an audit trail designed for them: every guardrail writes structured block-reason fields into the AI audit log, which is exactly what an auditor wants ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/)).", "fit_agent_features_note": "Kong documents function calling and custom tool use, including Gemini function declarations, OpenAI `llm/v1/chat` function calling, and LangChain `bind_tools` usage. AI Gateway also provides an A2A traffic gateway for securing, governing, and observing agent-to-agent traffic. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))", "fit_byok_required": "yes", "fit_byok_required_note": "Yes for the documented walkthrough: prerequisites include \"OpenAI account and API key\" ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/)); keys can be centralised via \"Konnect Config Store to store and reference your LLM provider API keys\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "fit_client_sdk_langs": [], "fit_client_sdks_note": "The documentation explicitly names the **OpenAI SDK** and **Gemini SDK**. The LangChain guide documents Python usage of `langchain-openai` and `ChatOpenAI`; official Kong client-library names and a broader supported-language list are not documented. ([Kong AI provider documentation](https://developer.konghq.com/ai-gateway/ai-providers/))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Kong documents Azure authentication through an Azure Compute resource’s Managed Identity or User-Assigned Identity, configured with `config.auth.azure_use_managed_identity` and optionally `config.auth.azure_client_id`. Amazon Bedrock authentication can use AWS IAM credentials or assumed roles. ([Kong AI provider documentation](https://developer.konghq.com/ai-gateway/ai-providers/))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented). No credit-card, free-tier or trial wording appears on the AI Gateway landing page ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)); the get-started page refers to \"conserve your free trial credits or avoid unnecessary charges\" without stating a payment requirement ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/))", "fit_ecosystem_notes": "Kong AI Gateway supports Konnect, self-hosted traditional, hybrid, and DB-less deployment modes, and Kubernetes deployment through the Kong Ingress Controller. Kong also documents fully self-hosted deployments on cloud, bare metal, containers, or Kubernetes, alongside Konnect’s managed SaaS control plane. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))", "fit_eval_prereqs": "Konnect personal access token; Kong Gateway Enterprise; decK v1.65.2+; an OpenAI account and API key; a licence for Kong Gateway Enterprise (for the self-managed path). No credit card, cloud account or cluster stated ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/))", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes on the get-started page: `curl -X POST \"$KONNECT_PROXY_URL/chat\" ... --json '{\"model\":\"gpt-5-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Say this is a test!\"}]}'` ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/)). The AI Gateway landing page itself shows no completion request, only `curl -Ls https://get.konghq.com/ai/v1 | bash` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Kong AI Gateway is implemented through Kong Gateway plugins, including **AI Proxy** and **AI Proxy Advanced**, with a minimum Kong Gateway version of `3.6`. The **AI MCP Proxy** is an additional Kong Gateway plugin for connecting Kong-managed services to MCP, available from Kong Gateway `3.12` and supporting hybrid, DB-less, and traditional topologies. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))", "fit_helm_chart": "kong/ingress and kong/kong from https://charts.konghq.com", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Kong provides official Kubernetes Helm charts from `https://charts.konghq.com`, including `kong/ingress` and `kong/kong`. Kong’s Kubernetes integration uses the Kong Ingress Controller; `kong/ingress` is recommended for new installations, while `kong/kong` supports hybrid and unmanaged Kong instances. ([Kong Helm charts](https://charts.konghq.com/))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "Kong’s AI MCP Proxy bridges MCP and HTTP, supports MCP tool discovery and invocation, proxies MCP requests, aggregates tools, and can convert REST API paths into MCP tools. Kong also provides `mcp-konnect`, an MCP server with tools for querying analytics and managing Konnect resources, usable with MCP clients such as Claude Desktop and Cursor. ([Kong AI MCP Proxy plugin](https://developer.konghq.com/plugins/ai-mcp-proxy/))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_credit_card", "fit_model_gate", "fit_vercel_ai_sdk" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented). No model approval, enablement, quota, or waitlist step appears ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "fit_onboarding_notes": "AI features are Kong Gateway plugins: \"You can enable the AI Gateway features through a set of modern and specialized plugins,\" and the landing page instructs \"Run the Kong Gateway quickstart and enable the AI Proxy plugin.\" Configuration is applied with decK state piped to `deck gateway apply`; Kong Gateway version shown is 3.15.0.5. Config tooling spans AI Gateway editor, decK, Terraform, KIC, Admin API, and Control Plane Config API ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/))", "fit_openai_dropin": "partial", "fit_openai_dropin_note": "Kong documents connecting the LangChain OpenAI adapter without code changes. The documented setting is `base_url=f'{kong_url}/{kong_route}'`, with `kong_url=\"http://127.0.0.1:8000\"`, `kong_route=\"gpt4o\"`, and `api_key=\"NONE\"` or a Kong consumer key. ([Kong LangChain integration guide](https://docs.jp.konghq.com/gateway/latest/ai-gateway/llm-library-integration-guides/langchain/))", "fit_prod_prereqs": "A Konnect personal access token and a Konnect Control Plane plus local Data Plane for the hybrid path, or Kong Gateway Enterprise with a licence exported as `KONG_LICENSE_DATA` for self-managed; decK v1.65.2+ and cURL. The page refers production users to the Install page and does not itself specify database/Redis/cluster/load-balancer requirements ([How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/)). Planning topics listed include \"AI Gateway resource sizing guidelines,\" \"Deployment topologies,\" and \"Hosting options\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain" ], "fit_python_frameworks_note": "Kong documents LangChain integration using the `langchain-openai` package and the `ChatOpenAI` class. Custom tools use `langchain_core.tools` and its `tool` decorator; LlamaIndex is not documented. ([Kong LangChain integration guide](https://docs.jp.konghq.com/gateway/latest/ai-gateway/llm-library-integration-guides/langchain/))", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://developer.konghq.com/ai-gateway/get-started/", "fit_self_host_docs": "command_published", "fit_self_host_install": "`curl -Ls https://get.konghq.com/quickstart | bash -s -- -e KONG_LICENSE_DATA` (self-managed Enterprise); destroy with `curl -Ls https://get.konghq.com/quickstart | bash -s -- -d`; the AI-specific demo is `curl -Ls https://get.konghq.com/ai/v1 | bash` ([How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))", "fit_terraform": "official_provider", "fit_terraform_note": "Kong documents the official Terraform providers `konnect`, `konnect-beta`, `kong-gateway`, and `kong-mesh`. The Terraform Registry provider source for the Kong Gateway provider is `Kong/kong-gateway`; the registry documentation states that Terraform 0.13+ is required. ([Kong Terraform documentation](https://developer.konghq.com/terraform/))", "fit_terraform_ref": "Kong/kong-gateway, konnect, konnect-beta and kong-mesh", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "n.a. (not documented)", "fit_vercel_ai_sdk_package": null }, { "slug": "merge-gateway", "name": "Merge Gateway", "company": "Merge API, Inc.", "tagline": "Hosted-only LLM gateway from the Merge unified-API company, sold on intelligent cost routing, spend governance and per-customer controls; launched 31 March 2026.", "category": "managed_gateway", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2020, "hq_country": "United States", "url_home": "https://www.merge.dev/gateway", "url_docs": "https://docs.merge.dev/merge-gateway/get-started", "url_pricing": "https://www.merge.dev/pricing/gateway", "url_status": "https://status.merge.dev", "url_github": null, "url_changelog": null, "url_models_api": "https://api-gateway.merge.dev/v1/models", "github_stars": null, "model_count": 3, "model_count_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 24, "token_markup_pct": 5, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "BYOK is gated to the Pro and Enterprise plans; one credential per provider per organisation, stored encrypted at rest, with an optional \"Fall back to Merge managed credentials\" toggle. No separate BYOK per-token or platform fee is published on the pricing page ([BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "seat_fee_usd_month": null, "free_tier": "\"Get $10 in free credits for your first month\", described in the plan table as \"Free, capped at $10 LLM usage\", with access to all major models, automatic fallback, cost optimisation and consolidated billing ([Gateway pricing](https://www.merge.dev/pricing/gateway)). The docs give a different number for the same thing: \"Free tier default: $15 spend cap for new organizations\" ([Cost governance and savings](https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings)).", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Pro usage fee on model spend", "amount": "5% of LLM cost" }, { "label": "Pro monthly credit grant", "amount": "$10/month, issued on the 1st, expires monthly, no roll-over, no cash value" }, { "label": "Launch offer", "amount": "Sign up before 30 June for zero fees for 12 months (ACH: unlimited zero-fee spend; card: zero fees on the first ~$40,000 of annual spend)" } ], "self_host_cost_note": "n.a. There is no self-hosted artifact to cost: no Docker image, Helm chart, binary, npm package or install command appears anywhere in the 403-URL Gateway documentation sitemap, and the only self-hosting language is a single Enterprise sales bullet, \"Deploy to your own VPC or on-prem environment\" ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Get started](https://docs.merge.dev/merge-gateway/get-started)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "Requests are portable in both directions by base-URL swap, since the gateway speaks OpenAI Chat Completions and Anthropic Messages natively and Merge publishes migration guides from OpenRouter, LiteLLM, Azure OpenAI, Bedrock and the direct SDKs ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Install skills](https://docs.merge.dev/merge-gateway/install-skills)). Configuration is not portable: routing policies, budgets, DLP and prompt-injection settings live only in Merge's dashboard and management API ([API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "api_surfaces": [ "Merge Gateway Responses API (native)", "OpenAI Chat Completions", "Anthropic Messages", "Embeddings", "Images", "Audio (text-to-speech)", "Video", "Management API" ], "sdk_note": "First-party SDKs are Python `merge-gateway-python` (`from merge_gateway import MergeGateway`), JavaScript/TypeScript `merge-gateway-sdk`, and a Vercel AI SDK provider `merge-gateway-ai-sdk-provider` (with a `/v5` subpath; AI SDK v4 is unsupported). None is required: the OpenAI, Anthropic and LangChain clients work by base-URL swap ([Get started](https://docs.merge.dev/merge-gateway/get-started)).", "cap_failover": true, "cap_load_balancing": null, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Two vendor-published figures exist, on different things, and neither is a single gateway-overhead number. A Merge blog post measuring ten \"lane combinations\" states \"[t]he median Gateway overhead is subsecond across all ten lane combinations, landing between roughly 90 and 650 milliseconds\" ([The cost of a gateway](https://www.merge.dev/blog/cost-of-gateway)), while the routing docs claim intelligent routing \"adds ~1-4ms of latency\" for the classification step alone ([Intelligent routing](https://docs.merge.dev/merge-gateway/routing/intelligent)). Both are vendor self-published with no percentile beyond \"median\", no RPS and no hardware; no independent benchmark of Gateway was found.", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Not answerable from Gateway's own pages. Geo-location routing lets you restrict which *vendors* Gateway may use by ISO country code and region (NA/EU/APAC/CN), for example \"add `EU` to your org's `allowed_regions`\" for GDPR-covered traffic, but that governs the upstream provider, not where Merge stores Gateway metadata and logs ([Geo-location routing](https://docs.merge.dev/merge-gateway/security/geo-location-routing), [Provider terms](https://docs.merge.dev/merge-gateway/security/provider-terms)). The US/EU/APAC hosting choice on Merge's security page is written about integration data in Merge Unified, not about Gateway ([Merge security](https://www.merge.dev/security)).", "zdr": true, "zdr_note": "A real, self-serve, org-level toggle in Configuration → Controls, effective on the next request and restricted to org admins. When on, Gateway will only route to vendors with a contractual zero-data-retention agreement, and fails closed: HTTP 400 containing \"Zero Data Retention\", or `zdr_vendor_unavailable` when no compliant vendor serves the model. Eighteen vendors are listed as covered, BYOK vendors count as ZDR because the traffic runs under your own agreement, and `GET /v1/models` exposes a per-model `zero_data_retention` boolean as the authoritative source ([Zero data retention](https://docs.merge.dev/merge-gateway/security/zero-data-retention), [Errors](https://docs.merge.dev/merge-gateway/errors)).", "air_gapped": null, "best_for": "Teams whose LLM bill has become a margin problem and who want routing, budgets and per-customer controls as a managed service — especially B2B products that need to hand each of their own customers separate routing policies, credentials and spend limits without building that plumbing.", "strengths": [ "Genuinely deep spend governance: org credit, project budgets over five periods, per-key spend limits with resets, per-customer budgets, 50/80/90% alert thresholds, a 10x-daily-average velocity alert, and HTTP 402 enforcement before any provider is called", "Embedded Routing Stack gives each of your customers their own routing policies, BYOK credentials, budgets and API keys, with pass-through usage reporting - unusual among gateways", "Security controls documented to an unusual level of specificity: Presidio-based DLP, a DeBERTa v3 injection classifier with published thresholds calibrated to 1% FPR, an org-level ZDR toggle that fails closed across 18 vendors, geo restriction by vendor country and region, and a ~60-event append-only audit trail with 29 permissions over 16 resources", "Intelligent routing with real cost claims (40-60% for Cost Optimized) and a Build Your Own Router mode where you weight benchmarks yourself, plus one named customer saving \"more than $10,000 per month\"", "Low-friction adoption: OpenAI, Anthropic, AI SDK and LangChain all work by base-URL swap, first-party Python and TypeScript SDKs, and migration skills for OpenRouter, LiteLLM, Bedrock and the direct SDKs" ], "weaknesses": [ "Hosted only: no self-host artifact, image, chart or architecture page exists, and Enterprise \"VPC or on-prem\" is a single unelaborated pricing bullet", "DLP and prompt-injection protection both fail open by default, so a configured blocking policy is telemetry until you set pi_fail_closed and account for DLP's documented fail-open behaviour", "Semantic caching was announced at launch as shipping \"in the next few weeks\" and, five months later, appears on no documentation page; Gateway's only caching is provider passthrough", "Observability is a closed loop: proprietary header-based tracing with ~24h retention, no OpenTelemetry or OTLP export, no log drain, and no published log-retention window", "Five months old with company-level compliance claims that are written about Merge Unified integration data rather than Gateway, no SLA page, no published uptime figure and no Gateway component on the status page", "The pricing page contradicts itself (\"Credit card required\" versus \"Start without a credit card\") and contradicts the docs on the free cap ($10 versus a documented $15 spend cap for new organisations)" ], "avoid_if": "You need to self-host or run in your own VPC with documented artifacts, you need OpenTelemetry export into an existing observability stack, or you are early enough that a five-month-old proprietary gateway with no changelog and no status-page component is an unacceptable dependency.", "reviews": [ { "url": "https://techsy.io/en/blog/best-llm-gateway-tools", "why": "Ranks Merge Gateway second of its shortlist as \"Best for enterprise-scale routing and spend control\" and is blunt about the trade-offs: not open source, \"heavier than it needs to be for prototypes\", and a thin community because it \"[l]aunched March 2026\".", "date": "2026-07-25", "type": "written", "title": "Best LLM Gateway Tools", "publisher": "Techsy" }, { "url": "https://www.synthszr.com/en/rankings/merge-merge-gateway", "why": "Independent profile that reaches the same structural conclusions - \"[f]ully hosted cloud gateway (no self-hosting option documented)\" and \"[p]roprietary, commercial SaaS product... not open source\" - and cites the ~1-4 ms routing figure. It also repeats the product page's wrong base URL, which is a useful demonstration of that error propagating.", "date": null, "type": "written", "title": "Merge Gateway ranking profile", "publisher": "Synthszr" }, { "url": "https://entelligence.ai/blogs/9-best-llm-routers-and-model-routing-tools-in-2026", "why": "Places Merge Gateway sixth as \"[b]est for embedded customer routing controls\" and correctly cautions that the Windmill savings figure is \"one customer case study, not a controlled benchmark\". Its claim of \"$0.05 per million tokens\" for BYOK could not be found on the pricing page.", "date": "2026-08-03", "type": "written", "title": "9 Best LLM Routers and Model Routing Tools in 2026", "publisher": "Entelligence" }, { "url": "https://www.testingcatalog.com/merge-launches-gateway-for-teams-managing-llms-on-prod/", "why": "The launch-day writeup, useful for the launch date, the founding year and the initial provider list - but it is explicitly marked SPONSORED, and its \"$75 million Series B\" contradicts Merge's own $55M Series B announcement, so it is not independent coverage.", "date": "2026-03-31", "type": "written", "title": "Merge launches Gateway for teams managing LLMs on prod", "publisher": "TestingCatalog" } ], "sources": { "zdr": "https://docs.merge.dev/merge-gateway/security/zero-data-retention", "name": "https://www.merge.dev/gateway", "company": "https://www.merge.dev/legal/gateway-terms", "license": "https://www.merge.dev/legal/gateway-terms", "reviews": "https://techsy.io/en/blog/best-llm-gateway-tools", "tagline": "https://www.merge.dev/gateway", "avoid_if": "https://www.merge.dev/pricing/gateway", "best_for": "https://docs.merge.dev/merge-gateway/embedded-routing-stack/overview", "category": "https://www.merge.dev/gateway", "url_docs": "https://docs.merge.dev/merge-gateway/get-started", "url_home": "https://www.merge.dev/gateway", "api_audio": "https://docs.merge.dev/merge-gateway/capabilities/text-to-speech", "api_batch": "https://docs.merge.dev/merge-gateway/api-overview", "byok_mode": "https://docs.merge.dev/merge-gateway/capabilities/byok/overview", "cap_audio": "https://docs.merge.dev/merge-gateway/capabilities/text-to-speech", "cap_evals": "https://docs.merge.dev/merge-gateway/routing/build-your-own-router", "cap_video": "https://docs.merge.dev/merge-gateway/capabilities/video-generation", "free_tier": "https://www.merge.dev/pricing/gateway", "guard_pii": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "rel_notes": "https://docs.merge.dev/merge-gateway/rate-limits", "strengths": "https://docs.merge.dev/merge-gateway/embedded-routing-stack/overview", "api_images": "https://docs.merge.dev/merge-gateway/capabilities/multimodal/image-generation", "deployment": "https://docs.merge.dev/merge-gateway/get-started", "eval_hooks": "https://docs.merge.dev/merge-gateway/routing/build-your-own-router", "extra_fees": "https://www.merge.dev/pricing/gateway", "hq_country": "https://www.testingcatalog.com/merge-launches-gateway-for-teams-managing-llms-on-prod/", "url_status": "https://status.merge.dev/", "weaknesses": "https://www.merge.dev/pricing/gateway", "zdr_status": "https://docs.merge.dev/merge-gateway/security/zero-data-retention", "cap_budgets": "https://docs.merge.dev/merge-gateway/cost/budgets", "cost_export": "https://docs.merge.dev/merge-gateway/api-overview", "guard_notes": "https://docs.merge.dev/merge-gateway/security/prompt-injection-protection", "log_content": "https://www.merge.dev/legal/gateway-terms", "log_opt_out": "https://www.merge.dev/legal/gateway-terms", "model_count": "https://api-gateway.merge.dev/v1/models", "url_pricing": "https://www.merge.dev/pricing/gateway", "api_surfaces": "https://docs.merge.dev/merge-gateway/api-overview", "cap_failover": "https://docs.merge.dev/merge-gateway/routing/deterministic-strategies", "founded_year": "https://www.testingcatalog.com/merge-launches-gateway-for-teams-managing-llms-on-prod/", "guard_custom": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "product_fate": "https://www.merge.dev/blog/gateway-announcement", "cache_support": "https://docs.merge.dev/merge-gateway/capabilities/prompt-caching", "cap_image_gen": "https://docs.merge.dev/merge-gateway/capabilities/multimodal/image-generation", "cost_controls": "https://docs.merge.dev/merge-gateway/cost/budgets", "fit_cloud_iam": "https://docs.merge.dev/merge-gateway/capabilities/byok/amazon-bedrock", "fit_terraform": "https://docs.merge.dev/merge-gateway/automation/projects-api/guardrails", "guard_content": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "pricing_model": "https://www.merge.dev/pricing/gateway", "vendor_status": "https://www.merge.dev/blog/gateway-announcement", "api_embeddings": "https://docs.merge.dev/merge-gateway/api-overview", "byok_supported": "https://docs.merge.dev/merge-gateway/capabilities/byok/overview", "cap_embeddings": "https://docs.merge.dev/merge-gateway/api-overview", "cap_guardrails": "https://docs.merge.dev/merge-gateway/security/prompt-injection-protection", "certifications": "https://www.merge.dev/security", "config_as_code": "https://docs.merge.dev/merge-gateway/automation/projects-api/guardrails", "fit_eval_shape": "https://docs.merge.dev/merge-gateway/get-started", "fit_kubernetes": "https://docs.merge.dev/merge-gateway/get-started", "fit_model_gate": "https://www.merge.dev/pricing/gateway", "fit_prod_shape": "https://docs.merge.dev/merge-gateway/get-started", "latency_claims": "https://www.merge.dev/blog/cost-of-gateway", "min_commitment": "https://www.merge.dev/legal/gateway-terms", "provider_count": "https://docs.merge.dev/merge-gateway/models/catalog", "url_models_api": "https://docs.merge.dev/merge-gateway/api-overview/models/list", "fit_credit_card": "https://www.merge.dev/pricing/gateway", "guard_injection": "https://docs.merge.dev/merge-gateway/security/prompt-injection-protection", "logs_by_default": "https://www.merge.dev/legal/gateway-terms", "model_count_low": "https://api-gateway.merge.dev/v1/models", "tracing_support": "https://docs.merge.dev/merge-gateway/observability/tracing", "cost_attribution": "https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings", "deployment_modes": "https://docs.merge.dev/merge-gateway/get-started", "feedback_capture": "https://docs.merge.dev/merge-gateway/observability/tracing", "fit_eval_prereqs": "https://docs.merge.dev/merge-gateway/get-started", "fit_eval_snippet": "https://docs.merge.dev/merge-gateway/get-started", "fit_prod_prereqs": "https://www.merge.dev/pricing/gateway", "model_count_high": "https://api-gateway.merge.dev/v1/models", "model_count_note": "https://api-gateway.merge.dev/v1/models", "rel_retry_config": "https://docs.merge.dev/merge-gateway/routing/deterministic-strategies", "token_markup_pct": "https://www.merge.dev/pricing/gateway", "cap_observability": "https://docs.merge.dev/merge-gateway/observability/tracing", "cap_pii_redaction": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "fit_byok_required": "https://docs.merge.dev/merge-gateway/capabilities/byok/overview", "fit_openai_dropin": "https://docs.merge.dev/merge-gateway/get-started", "fit_vercel_ai_sdk": "https://docs.merge.dev/merge-gateway/get-started", "guard_pii_default": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "guard_third_party": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "last_release_date": "https://api.github.com/repos/merge-api/merge-gateway-ai-sdk-provider", "openai_compatible": "https://docs.merge.dev/merge-gateway/get-started", "streaming_support": "https://docs.merge.dev/merge-gateway/streaming", "subprocessors_url": "https://www.merge.dev/legal/data-subprocessors", "architecture_class": "https://docs.merge.dev/merge-gateway/get-started", "fit_quickstart_url": "https://docs.merge.dev/merge-gateway/get-started", "fit_self_host_docs": "https://www.merge.dev/pricing/gateway", "guard_failure_mode": "https://docs.merge.dev/merge-gateway/security/prompt-injection-protection", "guard_model_policy": "https://docs.merge.dev/merge-gateway/security/per-customer-restrictions", "merchant_of_record": "https://www.merge.dev/pricing/gateway", "provider_count_low": "https://docs.merge.dev/merge-gateway/get-started", "rel_fallback_shape": "https://docs.merge.dev/merge-gateway/routing/deterministic-strategies", "rel_timeout_config": "https://docs.merge.dev/merge-gateway/streaming", "fit_ecosystem_notes": "https://docs.merge.dev/merge-gateway/install-skills", "guardrails_location": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "provider_count_high": "https://docs.merge.dev/merge-gateway/models/catalog", "rel_fallback_config": "https://docs.merge.dev/merge-gateway/routing/using-policies", "requires_vendor_sdk": "https://docs.merge.dev/merge-gateway/get-started", "api_openai_responses": "https://docs.merge.dev/merge-gateway/api-overview", "fit_client_sdk_langs": "https://docs.merge.dev/merge-gateway/get-started", "fit_gateway_platform": "https://docs.merge.dev/merge-gateway/get-started", "fit_onboarding_notes": "https://docs.merge.dev/merge-gateway/get-started", "fit_quickstart_steps": "https://docs.merge.dev/merge-gateway/get-started", "guard_custom_default": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "self_host_data_plane": "https://www.merge.dev/pricing/gateway", "data_export_available": "https://docs.merge.dev/merge-gateway/api-overview", "fit_python_frameworks": "https://docs.merge.dev/merge-gateway/get-started", "guard_content_default": "https://docs.merge.dev/merge-gateway/errors", "api_anthropic_messages": "https://docs.merge.dev/merge-gateway/api-overview", "first_party_vs_partner": "https://www.merge.dev/legal/gateway-terms", "fit_cloudflare_workers": "https://docs.merge.dev/merge-gateway/get-started", "prompt_transits_vendor": "https://docs.merge.dev/merge-gateway/get-started", "cap_conditional_routing": "https://docs.merge.dev/merge-gateway/routing/using-policies", "custom_endpoint_support": "https://docs.merge.dev/merge-gateway/capabilities/byok/overview", "guard_injection_default": "https://docs.merge.dev/merge-gateway/security/prompt-injection-protection", "model_count_source_note": "https://api-gateway.merge.dev/v1/models", "rel_health_check_config": "https://docs.merge.dev/merge-gateway/routing/deterministic-strategies", "rel_multi_region_config": "https://docs.merge.dev/merge-gateway/security/geo-location-routing", "trains_on_customer_data": "https://www.merge.dev/legal/gateway-terms", "guard_execution_location": "https://docs.merge.dev/merge-gateway/security/data-loss-prevention", "fit_vercel_ai_sdk_package": "https://docs.merge.dev/merge-gateway/get-started", "rel_load_balancing_config": "https://docs.merge.dev/merge-gateway/routing/deterministic-strategies", "api_openai_chat_completions": "https://docs.merge.dev/merge-gateway/api-overview", "fit_quickstart_steps_caveat": "https://docs.merge.dev/merge-gateway/get-started", "observability_requires_saas": "https://docs.merge.dev/merge-gateway/observability/tracing" }, "verified_at": { "zdr": "2026-09-02", "name": "2026-09-02", "company": "2026-09-02", "license": "2026-09-02", "reviews": "2026-09-02", "tagline": "2026-09-02", "avoid_if": "2026-09-02", "best_for": "2026-09-02", "category": "2026-09-02", "url_docs": "2026-09-02", "url_home": "2026-09-02", "api_audio": "2026-09-02", "api_batch": "2026-09-02", "byok_mode": "2026-09-02", "cap_audio": "2026-09-02", "cap_evals": "2026-09-02", "cap_video": "2026-09-02", "free_tier": "2026-09-02", "guard_pii": "2026-09-02", "rel_notes": "2026-09-02", "strengths": "2026-09-02", "api_images": "2026-09-02", "deployment": "2026-09-02", "eval_hooks": "2026-09-02", "extra_fees": "2026-09-02", "hq_country": "2026-09-02", "url_status": "2026-09-02", "weaknesses": "2026-09-02", "zdr_status": "2026-09-02", "cap_budgets": "2026-09-02", "cost_export": "2026-09-02", "guard_notes": "2026-09-02", "log_content": "2026-09-02", "log_opt_out": "2026-09-02", "model_count": "2026-09-26", "url_pricing": "2026-09-02", "api_surfaces": "2026-09-02", "cap_failover": "2026-09-02", "founded_year": "2026-09-02", "guard_custom": "2026-09-02", "product_fate": "2026-09-02", "cache_support": "2026-09-02", "cap_image_gen": "2026-09-02", "cost_controls": "2026-09-02", "fit_cloud_iam": "2026-09-02", "fit_terraform": "2026-09-02", "guard_content": "2026-09-02", "pricing_model": "2026-09-02", "vendor_status": "2026-09-02", "api_embeddings": "2026-09-02", "byok_supported": "2026-09-02", "cap_embeddings": "2026-09-02", "cap_guardrails": "2026-09-02", "certifications": "2026-09-02", "config_as_code": "2026-09-02", "fit_eval_shape": "2026-09-02", "fit_kubernetes": "2026-09-02", "fit_model_gate": "2026-09-02", "fit_prod_shape": "2026-09-02", "latency_claims": "2026-09-02", "min_commitment": "2026-09-02", "provider_count": "2026-09-02", "url_models_api": "2026-09-02", "fit_credit_card": "2026-09-02", "guard_injection": "2026-09-02", "logs_by_default": "2026-09-02", "model_count_low": "2026-09-26", "tracing_support": "2026-09-02", "cost_attribution": "2026-09-02", "deployment_modes": "2026-09-02", "feedback_capture": "2026-09-02", "fit_eval_prereqs": "2026-09-02", "fit_eval_snippet": "2026-09-02", "fit_prod_prereqs": "2026-09-02", "model_count_high": "2026-09-26", "model_count_note": "2026-09-26", "rel_retry_config": "2026-09-02", "token_markup_pct": "2026-09-02", "cap_observability": "2026-09-02", "cap_pii_redaction": "2026-09-02", "fit_byok_required": "2026-09-02", "fit_openai_dropin": "2026-09-02", "fit_vercel_ai_sdk": "2026-09-02", "guard_pii_default": "2026-09-02", "guard_third_party": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "subprocessors_url": "2026-09-02", "architecture_class": "2026-09-02", "fit_quickstart_url": "2026-09-02", "fit_self_host_docs": "2026-09-02", "guard_failure_mode": "2026-09-02", "guard_model_policy": "2026-09-02", "merchant_of_record": "2026-09-02", "provider_count_low": "2026-09-02", "rel_fallback_shape": "2026-09-02", "rel_timeout_config": "2026-09-02", "fit_ecosystem_notes": "2026-09-02", "guardrails_location": "2026-09-02", "provider_count_high": "2026-09-02", "rel_fallback_config": "2026-09-02", "requires_vendor_sdk": "2026-09-02", "api_openai_responses": "2026-09-02", "fit_client_sdk_langs": "2026-09-02", "fit_gateway_platform": "2026-09-02", "fit_onboarding_notes": "2026-09-02", "fit_quickstart_steps": "2026-09-02", "guard_custom_default": "2026-09-02", "self_host_data_plane": "2026-09-02", "data_export_available": "2026-09-02", "fit_python_frameworks": "2026-09-02", "guard_content_default": "2026-09-02", "api_anthropic_messages": "2026-09-02", "first_party_vs_partner": "2026-09-02", "fit_cloudflare_workers": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "cap_conditional_routing": "2026-09-02", "custom_endpoint_support": "2026-09-02", "guard_injection_default": "2026-09-02", "model_count_source_note": "2026-09-26", "rel_health_check_config": "2026-09-02", "rel_multi_region_config": "2026-09-02", "trains_on_customer_data": "2026-09-02", "guard_execution_location": "2026-09-02", "fit_vercel_ai_sdk_package": "2026-09-02", "rel_load_balancing_config": "2026-09-02", "api_openai_chat_completions": "2026-09-02", "fit_quickstart_steps_caveat": "2026-09-02", "observability_requires_saas": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:17:00.469341+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively shipping but very young and narrowly proven. Gateway launched 31 March 2026 ([Introducing Merge Gateway](https://www.merge.dev/blog/gateway-announcement), [TestingCatalog](https://www.testingcatalog.com/merge-launches-gateway-for-teams-managing-llms-on-prod/)), making it roughly five months old at the observation date, and it is the third SKU of an independent company that raised a $55M Series B led by Accel in 2022 ([Announcing Merge's $55M Series B](https://www.merge.dev/blog/announcing-merges-55-million-series-b)). Three signals argue for caution rather than for abandonment risk: there is no changelog or release-notes page in the docs at all, Merge's status page tracks only \"Merge Docs\" and \"Merge Unified Common Model Data Syncing\" with no Gateway component ([Merge status](https://status.merge.dev/)), and one feature announced at launch — semantic caching, \"this feature will launch in the next few weeks\" — is still absent from every documentation page five months later. A third-party review makes the same point from the outside: \"Launched March 2026, so the community, integrations, and third-party tutorials are thin next to LiteLLM or OpenRouter\" ([Techsy](https://techsy.io/en/blog/best-llm-gateway-tools), last updated 2026-07-25).", "last_release_date": "2026-08-11", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Not a product release: Merge publishes no changelog or release notes for Gateway, and no changelog URL exists in the docs sitemap. 2026-08-11 is the last public code push to `merge-api/merge-gateway-ai-sdk-provider`, the only actively pushed public Gateway repository; the skills pack was last pushed 2026-04-13 and neither repo has a tagged release ([merge-gateway-ai-sdk-provider](https://api.github.com/repos/merge-api/merge-gateway-ai-sdk-provider), [merge-gateway-skills](https://api.github.com/repos/merge-api/merge-gateway-skills)).", "prompt_transits_vendor": "yes", "prompt_path_note": "Always. Every request crosses Merge's hosted endpoint before reaching a provider, and there is no self-hosted or in-VPC data plane documented; the pipeline (validate, meter, filter, route, forward) runs entirely on Merge's side ([How it works](https://docs.merge.dev/merge-gateway/how-it-works), [Get started](https://docs.merge.dev/merge-gateway/get-started)).", "logs_by_default": "metadata_only", "logs_note": "Payload logging is off by default and is a setting you turn on: \"Except to the extent Customer enables payload logging via settings, Merge will not retain Inputs or Outputs and will only retain metadata regarding each request and response. When payload logging is enabled, Merge will store Inputs and Outputs for all requests, which can be viewed in the Logs detail view\" ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms), last updated 2026-04-02). Metadata is still rich: routing decision, cost, latency and policy outcome per request ([Introducing Merge Gateway](https://www.merge.dev/blog/gateway-announcement)).", "retention_days": null, "retention_note": "No log-retention window is published. What is stated: metadata only unless payload logging is enabled, and \"[w]ithin thirty (30) days of termination or expiration of this Agreement for any reason, Merge will, upon written request, delete all Customer Data in Merge's possession\" ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). Traces are the one figure given — retained about 24 hours ([Tracing](https://docs.merge.dev/merge-gateway/observability/tracing)).", "zdr_status": "yes", "trains_on_customer_data": "not_published", "training_note": "Merge makes no statement about training on Gateway customer data. The Gateway Terms grant only a licence to use Customer Data \"to provide and maintain the Service and develop Usage Data\", where Usage Data is defined as analytics and performance information that \"does not consist of personal data\" — silence, not a commitment ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). The only training language on any Gateway page is about third-party vendors under ZDR ([Zero data retention](https://docs.merge.dev/merge-gateway/security/zero-data-retention)).", "subprocessors_url": "https://www.merge.dev/legal/data-subprocessors", "regions_note": "Vendor-side region control is strong and customer-configurable: every vendor carries a `country_code` and a `geo_region` (NA, EU, APAC, CN), and an organisation sets `allowed_regions`/`ignored_regions` and `allowed_vendors`/`ignored_vendors`, which also apply to BYOK traffic and are recorded in the audit trail as `ORG_SETTINGS_UPDATED` ([Geo-location routing](https://docs.merge.dev/merge-gateway/security/geo-location-routing)). Merge's own hosting regions for Gateway are not stated on any Gateway page.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Company-level claim only: \"Merge adheres to industry-standard compliance frameworks, including SOC 2 Type II, ISO 27001, HIPAA, GDPR, and CCPA\" on merge.dev/security, a page written around Merge Unified integration data (Linked Accounts, selective sync). No Gateway page, the Gateway Terms or the pricing page repeats it, and no trust portal or report is named.", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "Merge publishes a Data Processing Agreement and a GDPR page in its legal index and lists CCPA/GDPR in the company compliance sentence; the Gateway Terms do not incorporate the DPA by reference.", "status": "claimed" }, "iso27001": { "detail": "Same company-level sentence on merge.dev/security; not restated for Gateway.", "status": "claimed" }, "hipaa_baa": { "detail": "A Business Associate Agreement is published in Merge's legal index and HIPAA appears in the company compliance sentence; neither is scoped to Gateway, and the Gateway Terms do not reference a BAA.", "status": "claimed" } }, "guardrails_location": "vendor_saas", "guardrails_note": "All enforcement runs inside Merge's hosted pipeline, at step 3 of five (validate, meter, filter, route, forward), before any provider is called; there is no self-hosted or in-VPC option in which the checks would run on your own infrastructure ([How it works](https://docs.merge.dev/merge-gateway/how-it-works), [Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention)).", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "markup_only", "pricing_model_note": "Pro is \"Pay LLM cost plus a 5% fee\" with no platform fee, no seat fee and no request fee; Free is \"Free, capped at $10 LLM usage\" and Enterprise is custom ([Gateway pricing](https://www.merge.dev/pricing/gateway)). Credits are prepaid and consumed per request, expire one year after purchase, are non-refundable, and can be auto-replenished ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms), last updated 2026-04-02).", "min_commitment": "None stated for Free or Pro; both are self-serve and Pro is \"Pay as you go\". Enterprise is custom-quoted with custom MSAs. Prepaid Service Credits expire one year after purchase and are non-refundable ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)).", "overage_note": "There is no request or seat meter to overrun; the meter is model spend. Free is hard-capped ($10 on the pricing page, $15 in the docs) and Pro \"scale[s] without a spending cap\" ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Cost governance and savings](https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings)). Exhausted org credit, project budgets and per-key limits return HTTP 402 with `budget_exceeded`, `project_budget_exceeded` or `api_key_limit_exceeded` before any provider is called ([Errors](https://docs.merge.dev/merge-gateway/errors), [How it works](https://docs.merge.dev/merge-gateway/how-it-works)).", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Gateway operates no cache of its own; it passes provider prompt caching through. Models are classed `explicit` (Anthropic, Claude and Amazon Nova on Bedrock — you place the markers), `automatic` (OpenAI, Gemini, DeepSeek, Qwen, Mistral, Z.AI, xAI) or `none`, and an `X-Session-Id` header (256 chars, namespaced per organisation) improves cache affinity; unsupported markers are stripped rather than erroring ([Prompt caching](https://docs.merge.dev/merge-gateway/capabilities/prompt-caching)). No cached-token discount or write premium is published, and no gateway-side exact or semantic cache exists — semantic caching was announced at launch as arriving \"in the next few weeks\" ([Introducing Merge Gateway](https://www.merge.dev/blog/gateway-announcement)) and, as of 2026-09-02, appears on no documentation page.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Threshold alerts default to 50%, 80% and 90% of budget, plus a velocity alert when spend hits 10x the daily average.", "tier": null, "supported": true }, "virtual_keys": { "note": "Not called virtual keys, but the object exists: API keys minted per project, environment or customer via the dashboard or Management API `POST /v1/keys`, each with its own spend limit and reset period.", "tier": null, "supported": true }, "rate_limiting": { "note": "Not stated as a customer-configurable control. `X-RateLimit-*` headers read 0 unless a per-key limit is set, and the only published caps are provider-facing: 100 requests and 100,000 tokens per minute per organisation per provider on managed credentials, which BYOK traffic bypasses. No concurrency or body-size cap is published.", "tier": null, "supported": null }, "spend_webhooks": { "note": "n.a. No webhook or subscription mechanism for budget or alert events appears on the budgets, cost-governance or projects pages.", "tier": null, "supported": null }, "model_allowlists": { "note": "Per-customer block and pin rules over provider/model combinations (maximum 100 rules per organisation, rejections as HTTP 403 `customer_blocked`, `customer_pinned`, `provider_blocked` or `model_blocked`), plus organisation-level `allowed_vendors`/`ignored_vendors` and region restrictions. Dashboard-configured, with an audited `reason`.", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "Per-key spend limits with `limit_reset` of daily, weekly or monthly; exhaustion returns HTTP 402 `api_key_limit_exceeded` and `X-Key-Limit-Remaining-USD` tracks the headroom.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Projects are the team/workload unit and carry budgets over daily, weekly, monthly, quarterly or yearly periods, with Soft Limit (alert) or Hard Limit (HTTP 402) enforcement; per-customer budgets exist separately for embedded tenants.", "tier": null, "supported": true } }, "cost_attribution": "Strong: `usage.cost` is returned on every response (null rather than 0 when a route is unpriced), and the dashboard breaks spend down by model, project, tag, API key, provider and customer, with consolidated invoicing across providers ([Cost governance and savings](https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings), [Projects](https://docs.merge.dev/merge-gateway/cost/projects), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "cost_export": "Programmatic read is available through the Management API `GET /v1/organization/usage`, alongside per-customer usage reporting for embedded tenants and an `AUDIT_LOG_EXPORTED` audit event implying a UI export path ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Embedded Routing Stack](https://docs.merge.dev/merge-gateway/embedded-routing-stack/overview), [Audit trail](https://docs.merge.dev/merge-gateway/security/audit-trail)). CSV, webhook, S3 or warehouse cost export: n.a. on the fetched pages.", "product_kind_note": "A hosted routing, cost-governance and security proxy in front of other vendors' models: \"The control plane for production AI. Access all LLMs through a single API, with intelligent routing, cost management, and security built-in\" ([Merge Gateway product page](https://www.merge.dev/gateway)). Its own terms call it \"Merge's proprietary large language model aggregator\" ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). It is a distinct SKU from Merge Unified (HRIS/CRM/ATS integrations) and Merge Agent Handler, launched on 31 March 2026 ([Introducing Merge Gateway](https://www.merge.dev/blog/gateway-announcement), [TestingCatalog](https://www.testingcatalog.com/merge-launches-gateway-for-teams-managing-llms-on-prod/), marked SPONSORED).", "deployment_modes": [ "saas" ], "deployment_note": "Nothing to install and nothing to operate: you swap a base URL and send a request ([Get started](https://docs.merge.dev/merge-gateway/get-started)). Searching the full Gateway documentation sitemap (403 URLs, fetched 2026-09-02) returns no page for Docker, Helm, Kubernetes, Terraform or a self-hosted data plane, so the Enterprise VPC/on-prem bullet is an unelaborated sales claim rather than a documented deployment mode ([Gateway pricing](https://www.merge.dev/pricing/gateway)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "partial", "api_embeddings": "yes", "api_images": "yes", "api_audio": "partial", "api_batch": "not_documented", "api_surface_note": "Base URL `https://api-gateway.merge.dev/v1` with `Authorization: Bearer`. Any SDK that accepts a custom base URL works, and Merge publishes per-SDK shims: `/v1/openai`, `/v1/anthropic`, `/v1/ai-sdk` and `/v1/openai` for LangChain ([Get started](https://docs.merge.dev/merge-gateway/get-started)). Two documented traps: `client.responses.*` against the bare `/v1` base URL reaches Gateway's own Responses API rather than OpenAI's, and the Anthropic SDK's `models.list()` against the bare host returns Gateway's native model listing. Note the product page instead tells you to \"Change the base URL to gateway.merge.dev/v1\", which is the dashboard host, not the API ([Merge Gateway product page](https://www.merge.dev/gateway)).", "model_count_low": 3, "model_count_high": 3, "model_count_source_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": 4, "provider_count_high": 24, "provider_count_source_note": "Merge publishes no provider total, and its docs use two different senses of the word. The catalogue's model IDs carry 24 model-family prefixes (ai21, alibaba, amazon, anthropic, arcee-ai, bland, bytedance, cohere, deepseek, google, meta, minimax, mistral, moonshot, morph, nvidia, openai, qwen, sakana, thinkingmachines, writer, xai, xiaomimimo, zai), while the separate \"vendor\" column is the execution host and is queried through `GET /v1/vendors` ([Model catalog](https://docs.merge.dev/merge-gateway/models/catalog), [Vendors list](https://docs.merge.dev/merge-gateway/api-overview/vendors/list)). Only four hosts are named in the quickstart ([Get started](https://docs.merge.dev/merge-gateway/get-started)); the zero-data-retention page names 18 ([Zero data retention](https://docs.merge.dev/merge-gateway/security/zero-data-retention)).", "first_party_vs_partner": "All third-party routed. Merge hosts no models; the Gateway Terms define the service as \"Merge's proprietary large language model aggregator, which allows users to route, monitor, manage, and optimize API requests to a variety of third-party artificial intelligence model providers\" ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)), and customers must comply with each AI Provider's own terms ([Provider terms](https://docs.merge.dev/merge-gateway/security/provider-terms)).", "custom_endpoint_support": "n.a. No mechanism for registering a custom, self-hosted or OpenAI-compatible third-party endpoint (vLLM, Ollama, SageMaker or a private base URL) appears on the pages fetched: BYOK covers named vendors only ([BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview), [Amazon Bedrock BYOK](https://docs.merge.dev/merge-gateway/capabilities/byok/amazon-bedrock)), and the routable set is the Merge-curated catalogue ([Model catalog](https://docs.merge.dev/merge-gateway/models/catalog), [Vendors list](https://docs.merge.dev/merge-gateway/api-overview/vendors/list)).", "byok_mode": "both", "merchant_of_record": "Split by mode. On managed credentials Merge is the merchant of record: you prepay Service Credits, Merge bills LLM cost plus 5% and \"consolidate[s] all provider invoices into one\" ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). On BYOK the provider bills you directly — \"BYOK traffic goes to the vendor under your own account and agreement\" ([Zero data retention](https://docs.merge.dev/merge-gateway/security/zero-data-retention)).", "key_management_note": "Gateway API keys are minted in the dashboard or through the Management API, which uses a separate `mgmt_` key that cannot call model endpoints; keys carry optional spend limits with daily, weekly or monthly resets, and can be scoped per project or per customer ([API keys](https://docs.merge.dev/merge-gateway/automation/api-keys), [API overview](https://docs.merge.dev/merge-gateway/api-overview)). Upstream BYOK secrets are encrypted at rest, limited to one per provider per organisation, and gated behind a manage-credentials permission; Bedrock accepts either a bearer token or IAM access keys ([BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview), [Amazon Bedrock BYOK](https://docs.merge.dev/merge-gateway/capabilities/byok/amazon-bedrock)).", "api_openai_chat_completions_note": "Yes: `POST /chat/completions` on the OpenAI wire format, with `/v1/openai` as the drop-in base URL for the OpenAI SDK and unprefixed model names accepted ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Get started](https://docs.merge.dev/merge-gateway/get-started)).", "api_anthropic_messages_note": "Yes: `POST /messages` plus `POST /messages/count_tokens`, with `/v1/anthropic` as the Anthropic SDK base URL. The Anthropic surface omits the `cost` field on stream terminal frames and sends no `ping` events ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Streaming](https://docs.merge.dev/merge-gateway/streaming)).", "api_openai_responses_note": "Partial, and easy to misread: there is a native `POST /responses`, but it is Gateway's own shape (snapshot `response.stream` events terminated by `response.done`), explicitly not OpenAI's Responses API. The docs tell you to use the `/v1/openai` base URL when you want OpenAI-shaped Responses calls ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Get started](https://docs.merge.dev/merge-gateway/get-started)).", "api_embeddings_note": "Yes: `POST /embeddings` is documented on the API surface ([API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "api_images_note": "Yes: `POST /v1/images/generations`, with `openai/gpt-image-2`, `xai/grok-imagine-image` and Gemini hybrid image models documented ([Image generation](https://docs.merge.dev/merge-gateway/capabilities/multimodal/image-generation), [API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "api_audio_note": "Partial: text-to-speech only, `POST /v1/audio/speech` with `openai/tts-1`, `tts-1-hd` and MiniMax voices, billed per character. No speech-to-text or realtime endpoint appears in the docs sitemap ([Text to speech](https://docs.merge.dev/merge-gateway/capabilities/text-to-speech), [API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "api_batch_note": "n.a. No batch or async bulk-inference endpoint exists on the fetched API surface, and the string \"batch\" does not appear in any of the 403 Gateway documentation URLs ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Rate limits](https://docs.merge.dev/merge-gateway/rate-limits)). The one asynchronous job type is video generation ([Video generation](https://docs.merge.dev/merge-gateway/capabilities/video-generation)).", "deployment_modes_note": "hosted SaaS only, at `https://api-gateway.merge.dev/v1` ([Get started](https://docs.merge.dev/merge-gateway/get-started)). Self-host, hybrid VPC, on-prem and air-gapped: n.a. as documented modes. The sole self-hosting statement anywhere is one Enterprise sales bullet, \"Deploy to your own VPC or on-prem environment\", with no accompanying architecture, artifact or requirements page ([Gateway pricing](https://www.merge.dev/pricing/gateway)); a third-party review reaches the same conclusion, \"Fully hosted cloud gateway (no self-hosting option documented)\" ([Synthszr](https://www.synthszr.com/en/rankings/merge-merge-gateway)).", "byok_mode_note": "both: Merge-managed credentials are the default and are billed at LLM cost plus 5%, while Pro and Enterprise organisations can attach their own provider credentials per vendor, with an optional fallback to Merge's keys and a `BYOK_ONLY` strict mode for embedded customers that never falls back ([BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview), [Embedded Routing Stack](https://docs.merge.dev/merge-gateway/embedded-routing-stack/overview), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "access_meta": { "qualified": [ "api_audio", "api_openai_responses" ], "model_count_kind": "enumerated", "provider_count_kind": "counted" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "not_documented as a user-settable value: no timeout header, SDK parameter or config key appears on the fetched pages. What is published is a fixed stall watchdog — a first chunk taking more than 120 seconds, or a 120-second gap between chunks, abandons the upstream and yields a 408/502 or a failover — plus the warning that a client disconnect is still billed ([Streaming](https://docs.merge.dev/merge-gateway/streaming), [API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "rel_retry_config": "not_documented", "rel_retry_note": "No retry counter, backoff strategy or retryable-status list is published; the documented failure behaviour is failover to the next model in the policy, not a retry against the same one. Provider 429s, 5xx and timeouts trigger failover, while client errors 400-404 do not. Default retry count: n.a. Backoff: n.a. ([Deterministic strategies](https://docs.merge.dev/merge-gateway/routing/deterministic-strategies), [Errors](https://docs.merge.dev/merge-gateway/errors)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED. A Priority strategy holds an ordered list of models and walks it on provider failure (`strategy: \"PRIORITY\"` over the API); the policy is created in the dashboard or via `POST /v1/routing-policies` and then selected per request with `routing_policy_id`, with precedence request > customer default > project > org default > raw `model` ([Deterministic strategies](https://docs.merge.dev/merge-gateway/routing/deterministic-strategies), [Using policies](https://docs.merge.dev/merge-gateway/routing/using-policies)). Mid-stream failover is visible to the client as a `{\"fallback_restart\": true}` frame ([Streaming](https://docs.merge.dev/merge-gateway/streaming)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "The feature exists but exposes no knobs: Gateway tracks provider health and automatically skips routes that are failing, and a tripped circuit breaker surfaces as HTTP 502 `model_vendor_unhealthy`. No probe interval, failure threshold, ejection window or half-open policy is published ([Deterministic strategies](https://docs.merge.dev/merge-gateway/routing/deterministic-strategies), [Errors](https://docs.merge.dev/merge-gateway/errors)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "not_documented as failover. Geo-location routing is a compliance restriction that narrows the vendor set by country and region; it is not described as a cross-region failover mechanism, and Merge publishes nothing about its own regional topology for Gateway ([Geo-location routing](https://docs.merge.dev/merge-gateway/security/geo-location-routing), [How it works](https://docs.merge.dev/merge-gateway/how-it-works)).", "rel_load_balancing_config": "dashboard_only", "rel_load_balancing_note": "Weights are not user-settable for traffic splitting. Least Latency and Lowest Cost are dashboard-only strategies with no exposed weights, and the API accepts only `PRIORITY` and `INTELLIGENT`. The one weighted surface is Build Your Own Router, where you weight *benchmarks* (summing to 1.0) to score models, not traffic shares — and it too is dashboard-only ([Deterministic strategies](https://docs.merge.dev/merge-gateway/routing/deterministic-strategies), [Build your own router](https://docs.merge.dev/merge-gateway/routing/build-your-own-router)).", "architecture_class": "vendor_saas", "architecture_note": "vendor_saas: closed multi-tenant service reached at `https://api-gateway.merge.dev/v1`, with no published source, runtime or language for the data plane. The only public Merge Gateway repositories are a Claude Code skills pack and the Vercel AI SDK provider (TypeScript, MIT), neither of which is the gateway itself ([merge-gateway-ai-sdk-provider](https://api.github.com/repos/merge-api/merge-gateway-ai-sdk-provider), [merge-gateway-skills](https://api.github.com/repos/merge-api/merge-gateway-skills)).", "self_host_data_plane": "not_documented", "self_host_artifact_note": "No Docker image, Helm chart, binary, npm package or install command for a self-hosted data plane appears on any fetched page, and no such page exists in the 403-URL Gateway docs sitemap ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Gateway pricing](https://www.merge.dev/pricing/gateway)). Enterprise \"Deploy to your own VPC or on-prem environment\" is a pricing-page bullet with no artifact behind it.", "latency_claims": [ { "url": "https://www.merge.dev/blog/cost-of-gateway", "unit": "ms", "scope": "gateway_overhead", "value": "90-650", "metric": "median", "publisher": "vendor", "conditions": "Vendor blog post measuring ten \"lane combinations\"; described as \"subsecond median TTFT\". No percentile beyond median, no RPS, no payload size, no hardware, no cache state." }, { "url": "https://docs.merge.dev/merge-gateway/routing/intelligent", "unit": "ms", "scope": "routing_decision", "value": "~1-4", "metric": "not stated", "publisher": "vendor", "conditions": "Docs claim for the intelligent-routing classification step only (prompt embedding plus complexity scoring), not end-to-end gateway overhead. Conditions not stated." } ], "throughput_claims": null, "streaming_support": "yes", "streaming_note": "Supported on every surface with `stream: true`, and unusually well documented. Usage and a `cost` figure arrive on the terminal frame (omitted on the Anthropic and AI SDK surfaces); a mid-stream failover emits `{\"fallback_restart\": true}` so clients must be prepared to discard partial output; the 120-second stall watchdogs apply to both first byte and inter-chunk gaps; and a client disconnect is still billed ([Streaming](https://docs.merge.dev/merge-gateway/streaming)).", "guard_pii": "sync_block", "guard_pii_note": "A Presidio-based DLP sidecar scans both prompts and completions against Global, USA and Custom category sets (15 seeded entity types) with a per-rule action of log, redact or block; blocking returns HTTP 400 (the aggregate page states 422 `blocked_by_dlp_policy`) and redaction means the redacted text is what gets logged. Vendor-stated cost: \"a few milliseconds\" ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention)).", "guard_injection": "sync_block", "guard_injection_note": "A fine-tuned DeBERTa v3 classifier with modes off, alert and block, calibrated to a 1% false-positive rate: `pi_block_threshold` 0.57, `pi_pass_threshold` 0.30, `pi_input_action` block and `pi_output_action` redact (from observe, redact, route, block, escalate), plus tiered indirect-injection handling and an allowlist of regexes (50 patterns of 200 characters at org level, up to 20 more per project, unioned). Blocks return 400/422 `pi_blocked` and every log carries a `pi_score` ([Prompt injection protection](https://docs.merge.dev/merge-gateway/security/prompt-injection-protection)).", "guard_content": "not_documented", "guard_content_note": "n.a. No toxicity, moderation or content-category filter is offered; the security suite is DLP, prompt injection, ZDR, geo routing and per-customer blocklists, and content refusals come from the provider's own policy as `content_policy_violation` with `source: \"provider\"` ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention), [Prompt injection protection](https://docs.merge.dev/merge-gateway/security/prompt-injection-protection), [Errors](https://docs.merge.dev/merge-gateway/errors)).", "guard_custom": "sync_block", "guard_custom_note": "Custom DLP categories accept your own Python-flavoured regexes (500 characters) and keyword lists (50 entries) with the same log/redact/block actions, and a built-in tester accepts up to 50,000 characters of sample text ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention)). A webhook or arbitrary-code guardrail hook is not offered; custom routing classifiers are a separate, dashboard-only centroid scorer requiring at least three simple and three complex examples ([Custom classifiers](https://docs.merge.dev/merge-gateway/routing/custom-classifiers)).", "guard_model_policy": "yes", "guard_model_policy_note": "Yes, and per-customer rather than only per-organisation: block or pin specific provider/model combinations for a given customer (up to 100 rules per organisation, rejected as HTTP 403 with an audited reason), on top of organisation-level `allowed_vendors`/`ignored_vendors` and region allowlists that also apply to BYOK traffic ([Per-customer restrictions](https://docs.merge.dev/merge-gateway/security/per-customer-restrictions), [Geo-location routing](https://docs.merge.dev/merge-gateway/security/geo-location-routing)).", "guard_execution_location": "vendor_saas", "guard_third_party": [ "Microsoft Presidio" ], "guard_failure_mode": "configurable", "guard_failure_mode_note": "Read this one carefully before trusting the controls: \"By default, DLP fails open and the request proceeds\", and prompt-injection checks also fail open unless you set `pi_fail_closed: true`. So a freshly configured blocking policy will let traffic through if the classifier or sidecar errors, until you change that ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention), [Prompt injection protection](https://docs.merge.dev/merge-gateway/security/prompt-injection-protection)). ZDR is the exception and fails closed with HTTP 400 ([Zero data retention](https://docs.merge.dev/merge-gateway/security/zero-data-retention)).", "log_content": "configurable", "log_content_note": "Configurable, and off by default: metadata only unless payload logging is enabled in settings, at which point inputs and outputs are stored for all requests and visible in the Logs detail view ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). Where DLP redaction applies, the redacted text is what gets logged ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention)).", "log_opt_out": "yes", "log_opt_out_note": "Yes, and it is the default state rather than an opt-out you have to request: payload logging is an account setting that starts disabled ([Merge Gateway Terms](https://www.merge.dev/legal/gateway-terms)). A per-request suppression header is not documented ([API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "tracing_support": "proprietary", "tracing_note": "Proprietary and header-based: pass `X-Merge-Trace-Id`, `X-Merge-Parent-Span-Id`, `X-Merge-Span-Name` and `X-Merge-Thread-Id` to stitch multi-step agent runs, with cost rolled up from the billing pipeline and Fusion runs generating child spans automatically. Traces are retained about 24 hours. No OpenTelemetry or OTLP export is offered, and `observability/tracing` is the only observability page in the 403-URL docs sitemap ([Tracing](https://docs.merge.dev/merge-gateway/observability/tracing)).", "export_destinations": null, "export_destinations_note": "n.a. No OTLP endpoint, log-drain, S3/warehouse sink or named third-party destination (Datadog, Grafana, LangSmith and so on) appears on any fetched page; the docs sitemap contains no OpenTelemetry page at all ([Tracing](https://docs.merge.dev/merge-gateway/observability/tracing), [API overview](https://docs.merge.dev/merge-gateway/api-overview)). The nearest thing is that `usage.cost` is returned inline on every response, which third-party tools such as Langfuse pick up automatically ([Cost governance and savings](https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings)).", "feedback_capture": "no", "feedback_capture_note": "n.a. — recorded as no because no score, rating or feedback ingestion endpoint appears on the fetched pages ([API overview](https://docs.merge.dev/merge-gateway/api-overview), [Tracing](https://docs.merge.dev/merge-gateway/observability/tracing)); the trace surface is write-time headers only, with no companion endpoint for attaching outcomes after the fact.", "eval_hooks": "partial", "eval_hooks_note": "Partial and routing-bound: Build Your Own Router can \"Run evals\" against a judge model and rubric, will auto-evaluate newly added models, and can source scores from published benchmarks, your own eval runs or uploaded results — and `EVAL_*` events appear in the audit trail. There is no eval API, dataset object or CI hook, and evals exist to weight routing rather than to test your application ([Build your own router](https://docs.merge.dev/merge-gateway/routing/build-your-own-router), [Audit trail](https://docs.merge.dev/merge-gateway/security/audit-trail)).", "observability_requires_saas": "Entirely. There is no self-hosted mode, so logs, traces, cost breakdowns and the audit trail live only in Merge's control plane, and there is no export path to bring them into your own stack ([Tracing](https://docs.merge.dev/merge-gateway/observability/tracing), [Cost governance and savings](https://docs.merge.dev/merge-gateway/cost/cost-governance-and-savings)).", "log_retention_note": "n.a. as a numeric window. Neither the tracing page nor the pricing page states a log-retention period or a per-tier retention difference; the only published figure is roughly 24 hours for traces ([Tracing](https://docs.merge.dev/merge-gateway/observability/tracing), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "production_meta": { "sources": {}, "qualified": [ "api_audio", "api_openai_responses" ], "latency_prose": "Two vendor figures, neither of them a clean overhead number. The blog post *The cost of a gateway* states \"[t]he median Gateway overhead is subsecond across all ten lane combinations, landing between roughly 90 and 650 milliseconds\" and that \"the overhead itself is nearly free with a subsecond median TTFT\" - metric is median TTFT overhead, but no p95/p99, no RPS, no payload size, no hardware and no cache state are given ([The cost of a gateway](https://www.merge.dev/blog/cost-of-gateway)). Separately the docs claim intelligent routing \"adds ~1-4ms of latency\", which covers only the classification step ([Intelligent routing](https://docs.merge.dev/merge-gateway/routing/intelligent)). Conditions not stated in either case.", "multi_surface": {}, "throughput_prose": "No throughput, RPS, QPS or tokens-per-day claim appears on any fetched Merge page, including the product page, pricing page, launch announcement, the cost-of-a-gateway blog post and the rate-limits documentation - conditions not stated because no claim exists ([The cost of a gateway](https://www.merge.dev/blog/cost-of-gateway), [Rate limits](https://docs.merge.dev/merge-gateway/rate-limits)).", "benchmark_independence": "Both latency figures are vendor self-published with no released methodology or raw data, and no independent benchmark of Merge Gateway was found. The same blog post also carries a cost comparison - \"total model spend was $8.17 for the fixed-Opus lane against $2.87 for the router, which represents a 65% reduction in cost\" - which is a vendor claim about its own routing, not a third-party test ([The cost of a gateway](https://www.merge.dev/blog/cost-of-gateway)). The one customer figure, Windmill's \"more than $10,000 per month\" saving, is a vendor-published case study ([Windmill case study](https://www.merge.dev/case-studies/windmill-gateway))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "not_documented", "guard_custom_default": "choose_at_setup", "rel_notes": "The reliability surface is deliberately narrow: failover order is the one thing you control, and it is configured as a policy object rather than in code. Timeouts, retries and health-check thresholds are all Merge's to set, which is the trade you accept for a managed control plane. Two operational details worth knowing before production: managed-credential traffic is capped at 100 requests and 100,000 tokens per minute per organisation per provider (BYOK traffic skips these caps), and `Retry-After` is only returned on gateway-generated 429s, not provider ones ([Rate limits](https://docs.merge.dev/merge-gateway/rate-limits), [Errors](https://docs.merge.dev/merge-gateway/errors)).", "guard_notes": "**The most important default here is the failure mode, not the thresholds**: DLP fails open by default and prompt-injection protection fails open unless `pi_fail_closed: true` is set, so both need explicit hardening before they can be treated as controls rather than telemetry ([Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention), [Prompt injection protection](https://docs.merge.dev/merge-gateway/security/prompt-injection-protection)). The compensating strength is scope: policies can be overridden per project through `PUT /v1/projects/{id}/pi-settings` and `/dlp-settings` (sparse, with DELETE restoring inheritance), and every change lands in an append-only audit trail of roughly 60 event types ([Projects guardrails API](https://docs.merge.dev/merge-gateway/automation/projects-api/guardrails), [Audit trail](https://docs.merge.dev/merge-gateway/security/audit-trail)).", "fit_agent_features_note": "Good coverage of the primitives, thin on orchestration. Documented: tool calling, structured outputs (`json_object` and `json_schema` with tri-state `strict`), reasoning via `thinking` plus provider-native `reasoning_effort`, a server-side web-search tool `merge:web_search` (Exa, You.com or Parallel, `auto` by default, up to 25 results per call and 6 tool iterations), and Fusion, which fans a request across at least two analysis models and has a judge model synthesise the answer (non-streaming only). Multi-turn state exists but is weak: `previous_response_id` uses an in-memory store that is off by default, expires after an hour, is best-effort and drops thinking blocks ([Tool calling](https://docs.merge.dev/merge-gateway/capabilities/tool-calling), [Structured outputs](https://docs.merge.dev/merge-gateway/capabilities/structured-outputs), [Web search](https://docs.merge.dev/merge-gateway/capabilities/web-search), [Fusion](https://docs.merge.dev/merge-gateway/capabilities/fusion), [Multi-turn conversations](https://docs.merge.dev/merge-gateway/capabilities/multi-turn-conversations)).", "fit_byok_required": "optional", "fit_byok_required_note": "Optional. Managed credentials are the default path and the quickstart needs no provider key at all; BYOK is an upgrade for Pro and Enterprise organisations, with a fallback-to-Merge toggle and a strict `BYOK_ONLY` mode for embedded tenants ([BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview), [Get started](https://docs.merge.dev/merge-gateway/get-started)).", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "First-party: Python `merge-gateway-python` (`from merge_gateway import MergeGateway`) and JavaScript/TypeScript `merge-gateway-sdk` (`import { MergeGateway } from \"merge-gateway-sdk\"`), plus the AI SDK provider `merge-gateway-ai-sdk-provider`. Third-party clients are supported by base URL: OpenAI, Anthropic and LangChain are named in a table of base URLs ([Get started](https://docs.merge.dev/merge-gateway/get-started)). No Go, Java, Ruby, .NET or Rust SDK is published.", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "upstream_credentials_only: AWS credentials appear solely as a way to attach your own Bedrock account — a bearer token or IAM access keys with `bedrock:InvokeModel`, `bedrock:InvokeModelWithResponseStream` or `AmazonBedrockLimitedAccess` — and Vertex AI has its own provider-specific credential shape. There is no IAM, OIDC or workload-identity path for authenticating *to* Gateway, which uses its own bearer keys ([Amazon Bedrock BYOK](https://docs.merge.dev/merge-gateway/capabilities/byok/amazon-bedrock), [BYOK overview](https://docs.merge.dev/merge-gateway/capabilities/byok/overview)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. Cloudflare Workers are neither a documented deployment target nor a documented runtime for Gateway; no edge or Workers page exists in the docs sitemap ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Coding agents](https://docs.merge.dev/merge-gateway/coding-agents/overview)).", "fit_credit_card": "required_for_model_access", "fit_credit_card_note": "The pricing page contradicts itself on the same card: the Free plan is badged \"Credit card required\" while its feature list includes the bullet \"Start without a credit card\" ([Gateway pricing](https://www.merge.dev/pricing/gateway)). Recorded as required because the badge is the explicit payment statement, but treat this as unresolved.", "fit_ecosystem_notes": "The ecosystem effort is aimed squarely at coding agents. Merge ships a Claude Code plugin (`claude plugin marketplace add merge-api/merge-gateway-skills`, then `claude plugin install merge-gateway`) exposing skills such as `/merge-gateway:gateway-implement`, `build-agent`, `gateway-features` and migration skills for the AI SDK, OpenRouter, direct SDKs and Bedrock; Cursor, Zed, Continue.dev, OpenCode, Claude Code, Codex, Pi, Factory Droid and Claude Desktop are documented as clients ([Install skills](https://docs.merge.dev/merge-gateway/install-skills), [Coding agents](https://docs.merge.dev/merge-gateway/coding-agents/overview)). Written migration guides cover OpenRouter, LiteLLM, Azure OpenAI, Bedrock, the Vercel AI SDK and direct SDKs. Beyond that the ecosystem is thin, as a third-party review notes: \"the community, integrations, and third-party tutorials are thin next to LiteLLM or OpenRouter\" ([Techsy](https://techsy.io/en/blog/best-llm-gateway-tools)).", "fit_eval_prereqs": "A Merge Gateway account and an API key from the dashboard (or minted through the Management API). No cloud account, cluster or upstream provider key is needed, because managed credentials are the default. Payment is the ambiguity: the Free plan carries the badge \"Credit card required\" while its own bullet list says \"Start without a credit card\" ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, runnable snippets in Python, TypeScript and the AI SDK on the first page: `pip install merge-gateway-python`, then `client = MergeGateway(api_key=\"YOUR_API_KEY\")` and `client.responses.create(model=\"openai/gpt-5.2\", input=[...])`, plus the OpenAI-SDK and Anthropic-SDK base-URL variants ([Get started](https://docs.merge.dev/merge-gateway/get-started)).", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented). Gateway is a standalone hosted endpoint; no integration with an API-gateway platform or service mesh (Kong, APISIX, Envoy, Istio) appears on the fetched pages ([Get started](https://docs.merge.dev/merge-gateway/get-started), [API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. There is nothing to run in a cluster: no Helm chart, manifest, operator or container image is published, and \"helm\" and \"kubernetes\" return no hits across the 403-URL Gateway docs sitemap ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Gateway pricing](https://www.merge.dev/pricing/gateway)).", "fit_mcp": null, "fit_mcp_note": "n.a. Gateway documents no MCP support of any kind — no MCP gateway, no hosted MCP server, no MCP tools in the API. The string \"mcp\" does not appear in any of the 403 Gateway documentation URLs, and the Claude Code integration ships skills rather than an MCP server ([Install skills](https://docs.merge.dev/merge-gateway/install-skills), [Coding agents](https://docs.merge.dev/merge-gateway/coding-agents/overview), [API overview](https://docs.merge.dev/merge-gateway/api-overview)). MCP is Merge's separate Agent Handler product, not Gateway ([Introducing Merge Gateway](https://www.merge.dev/blog/gateway-announcement)).", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_gateway_platform", "fit_kubernetes", "fit_terraform", "fit_mcp" ] }, "fit_model_gate": "none", "fit_model_gate_note": "None: the Free plan promises \"Access all major LLM models\" and the quickstart routes to any catalogue model by ID with no enablement, approval, waitlist or quota step ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Get started](https://docs.merge.dev/merge-gateway/get-started)). Gating is something you impose, not something Merge imposes: ZDR, geo rules and per-customer blocklists all narrow the reachable set ([Per-customer restrictions](https://docs.merge.dev/merge-gateway/security/per-customer-restrictions)).", "fit_onboarding_notes": "Onboarding is genuinely short — install an SDK or change a base URL, send a request — and the docs are unusually candid about the sharp edges you meet next: the `model` field becomes optional once a routing policy exists (omit it or pass the sentinel `\"default_routing\"`), streams can restart mid-flight with `{\"fallback_restart\": true}`, managed credentials are capped at 100 requests and 100,000 tokens per minute per organisation per provider, and inline media is limited to 20 MB (25 MB for audio) before a 413 ([Get started](https://docs.merge.dev/merge-gateway/get-started), [Streaming](https://docs.merge.dev/merge-gateway/streaming), [Rate limits](https://docs.merge.dev/merge-gateway/rate-limits)).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes, and documented as a one-line change: point the OpenAI SDK at `https://api-gateway.merge.dev/v1/openai` (or the bare `/v1`, which \"matches the OpenAI SDK's default base URL shape\") and call `chat.completions.create` with unprefixed model names ([Get started](https://docs.merge.dev/merge-gateway/get-started)). One caveat: `client.responses.*` against the bare `/v1` base URL hits Gateway's native Responses API instead of OpenAI's.", "fit_prod_prereqs": "A funded account (prepaid credits, or a card for Pro's LLM-cost-plus-5%), and for anything beyond a prototype: a routing policy, a project per workload with a budget, and DLP/prompt-injection settings hardened away from their fail-open defaults. BYOK and SSO/SAML require Pro; VPC or on-prem requires Enterprise ([Gateway pricing](https://www.merge.dev/pricing/gateway), [Data loss prevention](https://docs.merge.dev/merge-gateway/security/data-loss-prevention), [Using policies](https://docs.merge.dev/merge-gateway/routing/using-policies)).", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain" ], "fit_python_frameworks_note": "LangChain is the only Python framework named, and it is supported by base-URL swap rather than a dedicated package: \"LangChain — `https://api-gateway.merge.dev/v1/openai`\" ([Get started](https://docs.merge.dev/merge-gateway/get-started)). LlamaIndex, CrewAI, DSPy and Haystack: n.a. on the fetched pages.", "fit_quickstart_steps": 2, "fit_quickstart_steps_caveat": "Two steps on the SDK path (\"Install the SDK\", \"Send a request\"); the page then repeats the same two-step shape for the OpenAI SDK, the Vercel AI SDK provider and a URL shim, so there is no single canonical numbered list.", "fit_quickstart_url": "https://docs.merge.dev/merge-gateway/get-started", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. No Terraform provider or module is published and \"terraform\" returns no hits in the docs sitemap. Infrastructure-as-code, if you want it, means driving the REST Management API yourself — keys, projects, routing policies and guardrail settings are all API-addressable ([Projects guardrails API](https://docs.merge.dev/merge-gateway/automation/projects-api/guardrails), [API overview](https://docs.merge.dev/merge-gateway/api-overview)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Official first-party provider package `merge-gateway-ai-sdk-provider` with `createMergeGateway` and typed Gateway options under `providerOptions.mergeGateway`; `generateObject`/`streamObject` map to Gateway's native `json_schema` structured output. AI SDK v6 uses the root import, v5 the `/v5` subpath, and v4 is not supported (use the `/v1/ai-sdk` URL shim). The package source is public and MIT-licensed ([Get started](https://docs.merge.dev/merge-gateway/get-started), [merge-gateway-ai-sdk-provider](https://api.github.com/repos/merge-api/merge-gateway-ai-sdk-provider)).", "fit_vercel_ai_sdk_package": "merge-gateway-ai-sdk-provider" }, { "slug": "orq-ai-router", "name": "Orq.ai Router", "company": "Orq.ai", "tagline": "Managed EU-hosted AI gateway and router bundled with evaluation, observability and governance tooling.", "category": "managed_gateway", "deployment": "both", "license": "Proprietary", "founded_year": 2022, "hq_country": "Netherlands", "url_home": "https://orq.ai/", "url_docs": "https://docs.orq.ai/docs/ai-gateway", "url_pricing": "https://orq.ai/pricing", "url_status": "https://status.orq.ai", "url_github": null, "url_changelog": null, "url_models_api": null, "github_stars": null, "model_count": 500, "model_count_note": "Vendor-stated 500+ models from 30+ providers on the pricing page; the docs cite 300+ models and a competitor comparison cites 400+ from 28+ providers, so the number moves by page.", "provider_count": 30, "token_markup_pct": 0, "credit_fee_pct": 4.5, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "BYOK free for the first 1M requests/month, then 4%; no markup on model traffic itself.", "seat_fee_usd_month": null, "free_tier": "Pay-as-you-go starts free: 100k spans, 1 GB processed data, 500 agent runs, unlimited AI Gateway seats, 100 req/min limit, 30-day retention.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Orq-managed model credits", "amount": "4.5% fee on credits" }, { "label": "BYOK above 1M requests/month", "amount": "4%" }, { "label": "AI Studio seats", "amount": "EUR 35 per seat per month" }, { "label": "Knowledge Bases and Agent Memory", "amount": "EUR 500 per month" }, { "label": "Teams", "amount": "EUR 300 per month" }, { "label": "Spans beyond 100k", "amount": "EUR 7 per 100k" }, { "label": "Processed data beyond 1 GB", "amount": "EUR 3 per GB" }, { "label": "Agent runs beyond 500", "amount": "EUR 0.01 per run" } ], "self_host_cost_note": "No open-source or free self-host option. Enterprise-only sovereign deployments: customer-owned VPC on AWS or Azure via marketplace, or fully on-prem/air-gapped via a single Helm chart, priced custom and billed annually.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": false, "portability_note": "Telemetry query API is in private preview.", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Responses", "Anthropic Messages", "MCP" ], "sdk_note": "OpenAI- and Anthropic-compatible endpoints plus first-party SDKs (orq-ai-sdk for Python, @orq-ai/node for JavaScript).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No gateway proxy-overhead benchmark is published; the vendor states only that its Smart Router classifies each request in under 40 ms before model selection.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": true, "eu_residency": true, "eu_residency_note": "Vendor states all data is processed and stored in EU data centers on Google Cloud, with EU-based sovereign VPC and on-prem options on Enterprise. A third-party review of European gateways cautions that Orq's privacy policy describes storage on cloud database servers in the United States and that EU-only residency looks like a configuration or enterprise discussion rather than the default.", "zdr": true, "zdr_note": "Implemented as a routing filter rather than a storage setting: switch it on and the model pool narrows to providers that guarantee no request retention. Included on every plan.", "air_gapped": true, "best_for": "European teams that want an EU-resident managed gateway with governance, evaluations and observability in one platform rather than a bare proxy.", "strengths": [ "Strongest EU/sovereignty story here: EU data centers, sovereign VPC, on-prem and air-gapped Helm deployment", "Zero markup on model traffic, with free BYOK up to 1M requests/month", "Zero-data-retention routing filter and PII redaction built into the gateway", "SOC 2 Type II, GDPR, HIPAA BAA availability and detailed audit logs with SIEM export", "Broad multimodal reach (image, video, audio, embeddings, rerank) plus 40+ evaluators and prompt versioning" ], "weaknesses": [ "Closed source with no free self-host path; sovereign deployment is enterprise-only at unpublished prices", "Line-item pricing in euros across spans, processed data, agent runs, seats, teams and knowledge bases is hard to forecast", "Free tier is rate-limited to 100 requests/minute with only 30-day retention", "An independent European-gateway review disputes how EU-only the default data storage really is, and notes ISO 27001 is still in progress" ], "avoid_if": "You just need a thin high-throughput proxy, or you want a self-hostable open-source gateway without an enterprise contract.", "reviews": [ { "url": "https://woutersligter.medium.com/scaling-ai-with-confidence-our-real-world-experience-with-orq-ai-9ce394d32c70", "why": "Consultant's hands-on account of running Orq.ai's model garden, routing, deployments, RAG and evaluators on client projects, with specific module-level criticism.", "date": "2025-04-24", "type": "written", "title": "Scaling AI with Confidence: Our Real-World Experience with Orq.ai", "publisher": "Medium - Wouter Sligter (KODIFY)" }, { "url": "https://www.reddit.com/r/devops/comments/1ue3pzq/compared_openrouter_portkey_and_orqs_gateway_for/", "why": "Operator who ran all three gateways in production reports where Orq's routing and per-team cost breakdowns fall short (tier-gated), with commenters adding dashboard critiques.", "date": "2026-06-24", "type": "community", "title": "Compared OpenRouter, Portkey, and Orq's gateway for routing across providers", "publisher": "Reddit - r/devops" } ], "sources": { "zdr": "https://orq.ai/security", "gdpr": "https://orq.ai/security", "soc2": "https://orq.ai/security", "cap_mcp": "https://docs.orq.ai/docs/ai-gateway", "sdk_note": "https://docs.orq.ai/docs/ai-gateway", "url_docs": "https://docs.orq.ai/docs/ai-gateway", "zdr_note": "https://orq.ai/security", "cap_audio": "https://docs.orq.ai/docs/ai-gateway", "cap_evals": "https://orq.ai/", "cap_video": "https://docs.orq.ai/docs/ai-gateway", "free_tier": "https://orq.ai/pricing", "hipaa_baa": "https://docs.orq.ai/docs/enterprise/sovereign-ai", "air_gapped": "https://docs.orq.ai/docs/enterprise/sovereign-ai", "extra_fees": "https://orq.ai/pricing", "hq_country": "https://orq.ai/about-us", "url_status": "https://status.orq.ai", "zdr_status": "https://docs.orq.ai/docs/enterprise/eu-regions-faq", "cap_budgets": "https://orq.ai/pricing", "model_count": "https://orq.ai/pricing", "orq_budgets": "https://docs.orq.ai/docs/ai-gateway/budgets", "orq_pricing": "https://orq.ai/pricing", "api_surfaces": "https://docs.orq.ai/docs/ai-gateway", "cap_failover": "https://docs.orq.ai/docs/ai-gateway", "eu_residency": "https://orq.ai/security", "founded_year": "https://docs.orq.ai/docs/enterprise/sovereign-ai", "product_fate": "https://docs.orq.ai/changelog", "regions_note": "https://docs.orq.ai/docs/enterprise/eu-regions-faq", "byok_fee_note": "https://orq.ai/pricing", "cap_image_gen": "https://docs.orq.ai/docs/ai-gateway", "vendor_status": "https://orq.ai/press-releases", "byok_supported": "https://orq.ai/pricing", "cap_embeddings": "https://docs.orq.ai/docs/ai-gateway", "cap_guardrails": "https://orq.ai/", "certifications": "https://docs.orq.ai/docs/enterprise/deployment-options", "config_as_code": "https://docs.orq.ai/docs/deployments/routing", "credit_fee_pct": "https://orq.ai/pricing", "provider_count": "https://orq.ai/pricing", "retention_days": "https://orq.ai/legal/security", "cap_rate_limits": "https://orq.ai/pricing", "logs_by_default": "https://docs.orq.ai/docs/administer/data-compliance", "cap_cache_simple": "https://docs.orq.ai/docs/ai-gateway", "model_count_note": "https://opper.ai/blog/best-european-ai-gateways", "token_markup_pct": "https://orq.ai/pricing", "cap_observability": "https://orq.ai/", "cap_pii_redaction": "https://orq.ai/security", "eu_residency_note": "https://opper.ai/blog/best-european-ai-gateways", "last_release_date": "https://docs.orq.ai/changelog", "openai_compatible": "https://docs.orq.ai/docs/ai-gateway", "cap_load_balancing": "https://docs.orq.ai/docs/ai-gateway", "guardrails_location": "https://docs.orq.ai/docs/enterprise/deployment-options", "latency_source_note": "https://docs.orq.ai/docs/ai-gateway", "requires_vendor_sdk": "https://docs.orq.ai/docs/ai-gateway/features/openai-compatible-api", "self_host_cost_note": "https://docs.orq.ai/docs/enterprise/sovereign-ai", "cap_prompt_management": "https://orq.ai/", "data_export_available": "https://docs.orq.ai/docs/ai-studio/observability/telemetry-query-api", "prompt_transits_vendor": "https://docs.orq.ai/docs/enterprise/deployment-options", "cap_conditional_routing": "https://docs.orq.ai/docs/ai-gateway", "trains_on_customer_data": "https://docs.orq.ai/docs/administer/data-compliance" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "soc2": "2026-08-29", "cap_mcp": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "cap_evals": "2026-08-29", "cap_video": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "air_gapped": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "eu_residency": "2026-08-29", "founded_year": "2026-08-29", "byok_fee_note": "2026-08-29", "cap_image_gen": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-08-31T04:37:45.482322+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed with no sunset notice. Release 4.14 added an MCP gateway plus caching and tracing in routing rules. Raised a EUR 5M seed in December 2025.", "last_release_date": "2026-08-26", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Orq.ai prints no dates on its changelog, so this is the last-modified timestamp of the Release 4.14 page rather than a published release date.", "prompt_transits_vendor": "depends", "prompt_path_note": "EU multi-tenant cloud on GCP, your own AWS or Azure VPC where platform data never leaves it, or an on-premises Helm install with every component inside your environment.", "logs_by_default": "full_content", "logs_note": "Hosted deployments log by default for the plan's retention period. Variables flagged as PII and masked responses are the exception: they reach the model but are not stored or shown in logs.", "retention_days": null, "retention_note": "No number is published. Versions, evaluation logs and audit logs are deleted according to your plan's retention policy, which is not stated publicly.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "No data flowing through Orq.ai is ever used to train or fine-tune any model by Orq.ai.", "subprocessors_url": null, "regions_note": "EU only for the hosted platform — there is no US region, and EU residency is included on every plan. Customer AWS or Azure VPC and on-premises or air-gapped installs are offered.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type 2, monitored via Vanta", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA on request", "status": "available" }, "iso27001": { "detail": "proceeding towards formal certification", "status": "in_progress" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "both", "guardrails_note": "PII masking is available at platform level on all tiers, and VPC or on-premises deployment puts the whole platform — masking included — in your environment.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Models billed at provider list prices with 0% Orq markup. Routing on Orq credits adds a 4.5% top-up fee; BYOK is free up to 1M req/mo then 4%. Seat, module and data overages layer on top. Enterprise annual.", "min_commitment": "None stated for Pay-as-you-go. Enterprise agreements run on annual terms.", "overage_note": "Spans 100k/mo included then €7 per 100k. Processed data 1 GB/mo then €3/GB. Agent runs 500/mo then €0.01/run. Knowledge Bases include 2.5 GB document processing then €0.10/MB. Retention 30 days, no published price for extending. Usage beyond allowance bills as separate line items — nothing blocks.", "cache_support": "none", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Pricing page lists caching in the platform comparison but does not name a mechanism; AI Gateway docs page timed out on fetch. Because model traffic is billed at provider rates with no Orq markup, any provider-side cache discount flows to the customer, but Orq publishes no cached-token pricing of its own.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Documented example of a $100 budget with an alert configured at 90% ($90).", "tier": null, "supported": true }, "virtual_keys": { "note": "Not stated as virtual keys, but budgets attach to API keys.", "tier": null, "supported": null }, "rate_limiting": { "note": "Budgets include a per-minute rate cap; plan-level 100 req/min.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated as webhooks.", "tier": null, "supported": null }, "model_allowlists": { "note": "Budgets can be scoped per model; a policy-style allowlist is not stated separately.", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "Budgets scoped to workspace, project, identity, API key, provider or model. Pre-request hard cap — requests are blocked with HTTP 429 until the period resets.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Workspace / project / identity budgets supported; a 'team' scope not stated but workspace covers most team use.", "tier": null, "supported": true } }, "cost_attribution": "Attribution by workspace, project, identity, API key, provider and model. Tag/customer splits not stated.", "cost_export": "CSV/JSON of traces, experiments and datasets; Dashboards & Reporting API; SIEM export to Splunk/Datadog — all with no stated additional cost. S3/warehouse export not stated.", "product_kind_note": "Vendor calls it the \"AI Gateway\" / Router inside a broader generative-AI collaboration platform, positioned as a \"Sovereign AI Gateway and Platform\"; requests go to a router endpoint such as `POST https://my.orq.ai/v3/router/responses` ([Orq AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction), page dated 2026-08-19).", "deployment_modes": [ "saas", "self_host", "hybrid_vpc", "on_prem" ], "deployment_note": "Three documented options — Cloud, VPC, On-Premise — all with \"sovereign AI controls\"; on-premise supports model endpoints reachable from inside the environment ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)). Clients point at a custom host via `server_url` / `base_url` / `ORQ_BASE_URL` ([Base URLs](https://docs.orq.ai/reference/base-urls)). Tier gating: not stated (deployment options live under \"enterprise\" docs) ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "OpenAI-shaped paths but under an Orq prefix (`/v3/router/...`), so it is a base-URL swap plus path-prefix change; SDKs allow overriding the base URL ([Base URLs](https://docs.orq.ai/reference/base-urls)). Models are addressed provider-qualified, e.g. `\"model\": \"openai/gpt-5.4\"` ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Streaming caveats: n.a.", "model_count_low": 500, "model_count_high": 500, "model_count_source_note": "\"500+ models\" ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)), page undated.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "n.a. as a number on pages fetched; private-model sources number 3 (Azure AI Foundry, Google Vertex AI, LiteLLM) ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). Pages undated.", "first_party_vs_partner": "Third-party routing: the customer connects providers such as OpenAI via BYOK and Orq routes to them ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); private/fine-tuned/self-hosted models are imported from Azure AI Foundry, Vertex AI or LiteLLM ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). No Orq-owned inference hardware claimed.", "custom_endpoint_support": "yes: \"Bring Your Own Model\" supports fine-tuned, self-hosted and privately deployed models — choose Azure AI Foundry, Vertex AI or LiteLLM, enter the endpoint and credentials, then import and enable the deployment ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)).", "byok_mode": "byok_only", "merchant_of_record": "Not stated on pages fetched; because provider keys are the customer's own (BYOK), the upstream provider bills for tokens, while Orq shows \"latency, token usage, and cost\" in Traces ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Formal invoicing language: n.a.", "key_management_note": "Provider keys are connected per provider in the BYOK section of the workspace ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); private-model credentials are entered with the endpoint at import time ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). Encryption/KMS details and vendor visibility of request content: n.a. on pages fetched; VPC and on-premise deployments keep the router in the customer's environment ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)).", "api_openai_chat_completions_note": "yes: `/v3/router/chat/completions` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models), [Base URLs](https://docs.orq.ai/reference/base-urls)). Path prefix is `/v3/router`, not `/v1`.", "api_anthropic_messages_note": "n.a. (not documented on the router pages fetched: [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models), [Base URLs](https://docs.orq.ai/reference/base-urls)).", "api_openai_responses_note": "yes: `/v3/router/responses`, the endpoint used in the quickstart ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction), [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).", "api_embeddings_note": "yes: `/v3/router/embeddings` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).", "api_images_note": "yes: `/v3/router/images/generations`, `/images/edits`, `/images/variations` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).", "api_audio_note": "yes, both: `/v3/router/audio/transcriptions` (STT) and `/v3/router/audio/speech` (TTS) ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).", "api_batch_note": "n.a. (not documented on pages fetched: [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).", "deployment_modes_note": "Hosted SaaS (\"managed cloud\"), VPC, and on-premise; self-hosted and on-premise deployments \"serve `/v3/router` under their own hostname\" ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options), [Base URLs](https://docs.orq.ai/reference/base-urls), [AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Air-gapped: not documented.", "byok_mode_note": "byok_only on the evidence fetched: the quickstart requires \"Open BYOK, choose OpenAI, select Connect, and add your provider API key\" ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); no Orq-sold inference credits appear on the pages fetched.", "access_meta": { "qualified": [ "api_audio" ], "model_count_kind": "vendor_total", "provider_count_kind": null }, "rel_timeout_config": "per_request", "rel_timeout_note": "Request-body key `timeout.call_timeout` in milliseconds (documented examples `10000` and `15000`) ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).", "rel_retry_config": "per_request", "rel_retry_note": "`retry.count` accepts 1–5 with `retry.on_codes` defaulting to `[429]`. Backoff is exponential with jitter: 1s / 2s / 4s / 8s / 16s ±25%, for a documented maximum of roughly 31s of retry delay ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).", "rel_retry_default_note": "`retry.count` accepts 1–5 with `retry.on_codes` defaulting to `[429]`.", "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED: `fallbacks[].model`, tried in order, with a documented recommendation of at most 3 entries ([Orq.ai fallbacks](https://docs.orq.ai/docs/ai-gateway/features/fallbacks)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": "`not_documented` as an explicit health check or circuit breaker. What exists is latency-history-driven selection: an in-memory latency store with 5-minute freshness, 10% exploration traffic and a 0.5 ms near-tie threshold ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Self-hosted/on-prem deployments serve `/v3/router`, but no cross-region failover config is documented ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).", "rel_load_balancing_config": "per_request", "rel_load_balancing_note": "Weights supported: `load_balancer.type` of `weight_based`, `round_robin` or `latency_based`, with `models[].weight` in the range 0.001–1.0, normalized ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).", "architecture_class": "vendor_saas", "architecture_note": "Runtime is never disclosed on the fetched pages; only the hosted/self-hosted `/v3/router` endpoint is documented ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).", "self_host_data_plane": "yes", "self_host_artifact_note": "`yes`, per the docs' statement that self-hosted and on-prem deployments serve `/v3/router`; no specific artifact (image, chart, binary) is named on the fetched pages ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported; no streaming-specific caveat is stated on the fetched pages ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).", "guard_pii": "sync_block", "guard_pii_note": "Guardrail rules include PII detection, and non-compliant generations are blocked; however \"If the timeout is reached, the guardrail is skipped\" ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules), [Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails)). Separately, input variables flagged as PII are sent to the model but never stored or shown in logs ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))", "guard_injection": "sync_block", "guard_injection_note": "A jailbreak evaluator is available as a guardrail rule; detection is evaluator/model-based ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules))", "guard_content": "sync_block", "guard_content_note": "Guardrails are built on evaluators including LLM-as-judge, which can block non-compliant generations ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))", "guard_custom": "sync_block", "guard_custom_note": "Custom policies take the form of Python code guardrails, regex, JSON-schema validation, and HTTP calls, all able to block ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "fail_open", "guard_failure_mode_note": "**Fail open, documented**: \"If the timeout is reached, the guardrail is skipped\" ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules))", "log_content": "configurable", "log_content_note": "Logs and traces retain input/output for observability, but PII-flagged variables and masked outputs are excluded from storage ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance), [Observability overview](https://docs.orq.ai/docs/observability/overview))", "log_opt_out": "yes", "log_opt_out_note": "PII flags on input variables and full output masking prevent storage while requests still run; a global body-logging switch is not documented ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))", "tracing_support": "otel", "tracing_note": "OpenTelemetry natively: OTLP endpoint `https://api.orq.ai/v2/otel`, with agent traces and conversation threads represented ([Observability overview](https://docs.orq.ai/docs/observability/overview))", "export_destinations": [ "Webhook" ], "export_destinations_note": "`n.a.` — Orq documents OTel ingest into Orq, not outbound shipping ([Observability overview](https://docs.orq.ai/docs/observability/overview))", "feedback_capture": "yes", "feedback_capture_note": "Yes — `orq.feedback.create(field=..., value=..., trace_id=...)` submits feedback for an LLM transaction via the API, keyed on the trace ID returned by invoke/get_config ([Feedback SDK reference](https://github.com/orq-ai/orq-python/blob/main/docs/sdks/feedback/README.md))", "eval_hooks": "yes", "eval_hooks_note": "Yes, on Orq's SaaS — the same evaluator library (LLM-as-judge, Python, HTTP) that powers guardrails is used for scoring, and guardrail rules carry sample rates for partial online evaluation ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules), [Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))", "observability_requires_saas": null, "log_retention_note": "Configurable retention period, after which data is automatically deleted; the default duration is not stated ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` for benchmark figures. The only numbers are routing-internal thresholds: 5-minute latency freshness window and the 0.5 ms near-tie band ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).", "multi_surface": {}, "throughput_prose": "`n.a.` ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).", "benchmark_independence": "No figures published ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing))." }, "guard_pii_default": "sync_block", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Orq publishes unusually specific retry math (jitter percentage and cumulative worst-case delay), which matters because a 31s retry envelope plus a 15s `call_timeout` can exceed most client-side timeouts ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).", "guard_notes": "Sample rates mean a guardrail can be configured to evaluate only a fraction of traffic, which converts an apparent blocking control into a partial one; combined with skip-on-timeout, Orq's guardrails are best treated as best-effort enforcement ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules)).", "fit_agent_features_note": "Orq.ai supports function, HTTP, MCP, and built-in tools, tool-choice modes, conversation continuation with `previous_response_id`, and multi-agent orchestration using sub-agents and `team_of_agents`. Its MCP integration also supports tool filtering with `allowed_tools`; A2A is not documented. ([Orq.ai run agents](https://docs.orq.ai/docs/ai-studio/ai-engineering/run-agents))", "fit_byok_required": "yes", "fit_byok_required_note": "Yes in this quickstart: it instructs adding an OpenAI API key through AI Gateway > BYOK before enabling `gpt-4o`, creating the agent, and calling it ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))", "fit_client_sdk_langs": [ "Python", "Node.js" ], "fit_client_sdks_note": "Official SDKs are documented for Python and Node.js: Python package `orq-ai-sdk` with class `Orq`, and Node.js package `@orq-ai/node` with class `Orq`. The `orq` CLI command is also documented. ([Orq.ai client libraries reference](https://docs.orq.ai/reference/client-libraries))", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Partially: for Google Vertex AI, Orq.ai documents pasting a GCP service-account key JSON and granting the Service Account Token Creator and Vertex AI User roles; AWS SigV4/IAM roles and Azure Entra/managed identity are not documented. ([Orq.ai Vertex AI provider integration docs](https://docs.orq.ai/docs/ai-studio/integrations/providers/vertex-ai))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated; the account is described as free ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent)). No credit-card or credits wording on the deployment page ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))", "fit_ecosystem_notes": "Orq.ai documents managed cloud, self-hosted, VPC, and on-premise deployment options. Managed Cloud is hosted in EU data centers on Google Cloud Platform; VPC runs in a customer AWS or Azure account and is available through those marketplaces; Enterprise on-premise runs in Kubernetes on customer or self-managed infrastructure and can operate air-gapped after images are cached locally, provided configured model endpoints are reachable. ([Orq.ai deployment options](https://docs.orq.ai/docs/enterprise/deployment-options))", "fit_eval_prereqs": "None stated before step 1; step 1 itself is signing up for a free orq.ai account and creating a workspace. However the flow requires adding an OpenAI API key under AI Gateway > BYOK. No credit card, cloud account, or cluster mentioned ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `curl --request POST --url 'https://my.orq.ai/v3/router/responses' --header \"Authorization: Bearer $ORQ_API_KEY\" --data '{\"model\":\"agent/my-assistant\",\"input\":\"What is the capital of France?\"}'` ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "single Orq chart, Kubernetes 1.28+", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Enterprise on-premise deployment is supported as a single Helm chart in a Kubernetes cluster; Kubernetes 1.28 or later is required, and the chart uses Helm hooks for database and data migrations. The optional agent sandbox uses the upstream `kubernetes-sigs/agent-sandbox` controller version `v0.4.6` with its own Helm chart. ([Orq.ai deployment options](https://docs.orq.ai/docs/enterprise/deployment-options))", "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "Yes. Orq.ai documents hosted MCP servers at `https://my.orq.ai/v2/mcp` for workspace access with 38 specialized tools, and supports MCP-compatible servers in agents through inline `server_url` or saved tools, with Streamable HTTP and SSE. A separate documentation MCP server is hosted at `https://docs.orq.ai/mcp`. ([Orq.ai MCP server integration](https://docs.orq.ai/docs/ai-studio/code-assistants/orq-mcp))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_gateway_platform" ] }, "fit_model_gate": "enablement_step", "fit_model_gate_note": "No approval gate, but a per-model enablement action inside the product: the quickstart requires enabling `gpt-4o` after adding a BYOK key ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))", "fit_onboarding_notes": "The path is agent-centric rather than a raw model call: create an agent, enable `gpt-4o`, configure Web Search and Web Scraper tools, publish, call, then view traces; the guide is billed as \"Beginner-friendly, no AI experience needed.\" Optional SDKs: `pip install orq-ai-sdk`, `npm install @orq-ai/node`. Every agent call is automatically traced ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. OpenAI-compatible clients can use the exact base URL `https://my.orq.ai/v3/router` with an Orq.ai API key in `Authorization: Bearer $ORQ_API_KEY`; another proxy page documents `https://api.orq.ai/v2/router`. ([Orq.ai base URLs reference](https://docs.orq.ai/reference/base-urls))", "fit_prod_prereqs": "VPC: the customer's own AWS or Azure account, deployed through the AWS or Azure marketplaces. On-premise: Kubernetes 1.28+, at least 3 worker nodes with 4 vCPU and 16 GB RAM each, PostgreSQL 14–18, a MongoDB-compatible database, and S3-protocol object storage; \"All components run inside the customer environment.\" ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "cloud_console", "fit_python_frameworks": [ "LlamaIndex" ], "fit_python_frameworks_note": "LlamaIndex integration is documented through the packages `llama-index` and `llama-index-llms-openai-like`, using the `OpenAILike` class. LlamaIndex Agents examples use `ReActAgent` and `FunctionTool`; LangChain is not documented. ([Orq.ai LlamaIndex Agents integration](https://docs.orq.ai/docs/ai-studio/integrations/frameworks/llamaindex-agents))", "fit_quickstart_steps": 7, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.orq.ai/docs/quickstarts/build-your-first-agent", "fit_self_host_docs": "documented_no_command", "fit_self_host_install": "No literal install command is published; the deployment-options page states the requirements and that \"container images must be downloaded or cached in a local registry\" but gives no docker/helm command ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))", "fit_terraform": "official_provider", "fit_terraform_note": "An official Terraform provider is documented as `orq-ai/orq`, version `0.1.0`. ([Orq.ai Terraform provider guardrail rule](https://registry.terraform.io/providers/orq-ai/orq/latest/docs/resources/guardrail_rule))", "fit_terraform_ref": "orq-ai/orq 0.1.0", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Yes. Orq.ai documents the official TypeScript package `@orq-ai/vercel-provider`, installed with `npm install @orq-ai/vercel-provider ai`, and configured with `createOrqAiProvider({ apiKey: process.env.ORQ_API_KEY })`. ([Orq.ai Vercel AI SDK integration](https://docs.orq.ai/docs/ai-studio/integrations/frameworks/vercel-ai))", "fit_vercel_ai_sdk_package": "@orq-ai/vercel-provider" }, { "slug": "portkey", "name": "Portkey", "company": "Portkey, Inc. (Palo Alto Networks)", "tagline": "Open-core AI gateway with a hosted control plane for observability, prompt management and governance.", "category": "managed_gateway", "deployment": "both", "license": "MIT", "founded_year": 2023, "hq_country": "United States", "url_home": "https://portkey.ai", "url_docs": "https://portkey.ai/docs", "url_pricing": "https://portkey.ai/pricing", "url_status": "https://status.portkey.ai", "url_github": "https://github.com/Portkey-AI/gateway", "url_changelog": "https://portkey.ai/docs/changelog", "url_models_api": null, "github_stars": 12848, "model_count": 250, "model_count_note": "Docs and the Gateway 2.0 announcement state a unified interface to 250+ models; the gateway's own /v1/models endpoint requires an API key, so no public catalog endpoint exists.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Requests use your own provider credentials via provider integrations; enterprise deployments can also bring their own KMS encryption keys.", "seat_fee_usd_month": null, "free_tier": "Developer plan free: 10,000 recorded logs/month, 3-day log and 30-day metric retention, universal API, fallbacks, load balancing, retries, 3 prompt templates, simple caching, deterministic guardrails.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Production plan platform fee", "amount": "$49/month (100k recorded logs, 30-day log retention)" }, { "label": "Production log/request overage", "amount": "+$9 per additional 100k requests" } ], "self_host_cost_note": "The gateway core is MIT-licensed and free to self-host (you pay only your own infrastructure and model providers); the Gateway 2.0 release moved circuit breakers, semantic cache, budget limits, model catalog, metadata governance and config management into open source. Self-hosting drops the hosted dashboards/control plane, and an On-Prem Enterprise Gateway (gRPC, SSO, SCIM, AWS KMS, RBAC, JWT, audit logs, multi-workspace) is sold separately.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions", "Portkey unified API", "Admin/Control-plane API" ], "sdk_note": "Three-line integration via Portkey SDKs or by pointing the OpenAI SDK at https://api.portkey.ai/v1.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": null, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Portkey docs state benchmarks estimate a total added latency of 20-40 ms versus direct API calls, along with 99.99% uptime and 25M+ requests served daily; no single figure is published. A third-party comparison describes Portkey's SaaS hop as similar to OpenRouter's without a number.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": true, "eu_residency": null, "eu_residency_note": null, "zdr": true, "zdr_note": "Zero retention, PII scrubbing and a metrics-only mode are all offered, but enabled on request rather than self-service.", "air_gapped": null, "best_for": "Teams that want governance, guardrails and prompt management in one control plane, with the option to self-host the routing engine.", "strengths": [ "MIT-licensed gateway core covering routing, fallbacks, conditional routing, semantic cache, budgets and guardrails", "Full governance stack: RBAC, SSO/SCIM, audit logs, budget and rate limits, PII-redaction guardrails", "SOC 2, HIPAA, GDPR and ISO 27001 compliance claims plus custom BAAs", "MCP registry with OAuth 2.1 (PKCE) in the open-source gateway", "Cheap entry point: free Developer tier, $49/month Production" ], "weaknesses": [ "Pricing is metered on recorded logs, not tokens, so high-request workloads hit $9 per extra 100k requests", "No public model-catalog endpoint - the /v1/models call requires an API key", "Self-hosting loses the hosted dashboards, prompt management and guardrails UI", "Third-party reviews note slower development cadence and a smaller self-hosting community than LiteLLM" ], "avoid_if": "You mainly need a thin, zero-config marketplace router and do not want to run or pay for a control plane.", "reviews": [ { "url": "https://www.infoworld.com/article/3835182/portkey-an-open-source-ai-gateway-for-easy-llm-orchestration.html", "why": "InfoWorld's hands-on review of the open-source gateway, exercising Portkey's unified API across OpenAI, Anthropic, Azure, Groq, Fireworks and Ollama plus its hosted tier.", "date": "2025-03-06", "type": "written", "title": "Portkey: An open-source AI gateway for easy LLM orchestration", "publisher": "InfoWorld" }, { "url": "https://www.youtube.com/watch?v=-rSbvS0qLqY", "why": "Latent Space's hosts interrogate Portkey's CEO on gateway design tradeoffs - routing, guardrails, logging overhead and observability - on an established AI-engineering podcast.", "date": "2025-02-05", "type": "video", "title": "Why every AI Engineer needs an AI Gateway (ft Portkey.ai CEO)", "publisher": "YouTube - Latent Space" }, { "url": "https://www.youtube.com/watch?v=TpUwSmGfMrQ", "why": "Independent hands-on build using Portkey's gateway: virtual keys, configs, a Bun/Node project querying OpenAI and Perplexity, then inspecting logs and cache hits.", "date": "2024-01-18", "type": "video", "title": "AI Gateway: Enhancing LLM Integrations for Application Development", "publisher": "YouTube - Developers Digest" }, { "url": "https://www.reddit.com/r/devops/comments/1ue3pzq/compared_openrouter_portkey_and_orqs_gateway_for/", "why": "Same operator thread covers Portkey's routing behaviour and cost-attribution dashboards after running it in production alongside two rivals.", "date": "2026-06-24", "type": "community", "title": "Compared OpenRouter, Portkey, and Orq's gateway for routing across providers", "publisher": "Reddit - r/devops" } ], "sources": { "zdr": "https://portkey.ai/docs", "gdpr": "https://portkey.ai/docs", "soc2": "https://portkey.ai/docs", "cap_mcp": "https://portkey.ai/blog/gateway-2-0", "license": "https://github.com/Portkey-AI/gateway", "sdk_note": "https://portkey.ai/docs", "url_docs": "https://portkey.ai/docs", "zdr_note": "https://portkey.ai/docs", "cap_evals": "https://portkey.ai/pricing", "free_tier": "https://portkey.ai/pricing", "hipaa_baa": "https://portkey.ai/docs", "deployment": "https://portkey.ai/docs", "extra_fees": "https://portkey.ai/pricing", "hq_country": "https://www.linkedin.com/company/portkey-ai", "url_github": "https://github.com/Portkey-AI/gateway", "url_status": "https://status.portkey.ai", "zdr_status": "https://portkey.ai/docs/enterprise/security", "acquired_by": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents", "cap_budgets": "https://docs.portkey.ai/docs/product/enterprise-offering", "model_count": "https://portkey.ai/docs", "url_pricing": "https://portkey.ai/pricing", "api_surfaces": "https://portkey.ai/docs", "cap_failover": "https://portkey.ai/pricing", "github_stars": "https://api.github.com/repos/Portkey-AI/gateway", "product_fate": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents", "regions_note": "https://portkey.ai/docs/enterprise/security", "acquired_date": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents", "byok_fee_note": "https://portkey.ai/blog/gateway-2-0", "url_changelog": "https://portkey.ai/docs/changelog", "vendor_status": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents", "byok_supported": "https://portkey.ai/docs", "cap_guardrails": "https://portkey.ai/pricing", "certifications": "https://docs.portkey.ai/docs/product/enterprise-offering/security-portkey", "config_as_code": "https://docs.portkey.ai/docs/api-reference/config-object", "retention_days": "https://portkey.ai/docs/enterprise/security", "retention_note": "https://portkey.ai/docs/enterprise/security", "sla_uptime_pct": "https://portkey.ai/docs/product/product-feature-comparison", "cap_rate_limits": "https://docs.portkey.ai/docs/product/enterprise-offering", "logs_by_default": "https://docs.portkey.ai/docs/introduction/what-is-portkey", "portkey_pricing": "https://portkey.ai/pricing", "cap_cache_simple": "https://portkey.ai/pricing", "model_count_note": "https://portkey.ai/blog/gateway-2-0", "acquisition_terms": "https://economictimes.com/tech/startups/palo-alto-networks-to-acquire-elevation-capital-backed-ai-app-infra-startup-portkey/articleshow/130668555.cms", "cap_observability": "https://portkey.ai/pricing", "cap_pii_redaction": "https://docs.portkey.ai/docs/product/enterprise-offering", "last_release_date": "https://portkey.ai/docs/changelog/enterprise", "openai_compatible": "https://portkey.ai/docs", "cap_cache_semantic": "https://portkey.ai/pricing", "cap_load_balancing": "https://portkey.ai/pricing", "log_retention_note": "https://portkey.ai/docs/enterprise/security", "guardrails_location": "https://portkey.ai/docs/enterprise/security", "latency_source_note": "https://portkey.ai/docs", "requires_vendor_sdk": "https://docs.portkey.ai/docs/integrations/llms/openai", "self_host_cost_note": "https://portkey.ai/blog/gateway-2-0", "cap_prompt_management": "https://portkey.ai/pricing", "data_export_available": "https://docs.portkey.ai/docs/product/observability/logs-export", "portkey_budget_limits": "https://docs.portkey.ai/docs/product/ai-gateway/virtual-keys/budget-limits", "prompt_transits_vendor": "https://portkey.ai/docs/enterprise/security", "cap_conditional_routing": "https://portkey.ai/blog/gateway-2-0", "portkey_feature_comparison": "https://portkey.ai/docs/product/product-feature-comparison" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "soc2": "2026-08-29", "cap_mcp": "2026-08-29", "license": "2026-09-14", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "zdr_note": "2026-08-29", "cap_evals": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "url_github": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "github_stars": "2026-08-29", "byok_fee_note": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "retention_days": "2026-09-05", "retention_note": "2026-09-05", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "log_retention_note": "2026-09-05", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-14T20:21:32.499616+00:00", "vendor_status": "acquired", "acquired_by": "Palo Alto Networks, Inc.", "acquired_date": "2026-05-29", "acquisition_terms": "Terms undisclosed; The Economic Times reported $120-140M citing sources", "product_fate": "Palo Alto Networks closed its acquisition on 29 May 2026 and is establishing Portkey as the core AI Gateway for its Prisma AIRS platform. Palo Alto Networks said it will continue to support existing and new Portkey customers, and the commercial Enterprise Gateway is still shipping. The open-source gateway repository, however, has had no tagged release since 12 January 2026, so if you were planning to self-host the free version, check that cadence before committing.", "last_release_date": "2026-08-21", "sla_uptime_pct": 99.9, "sla_url": "https://portkey.ai/docs/product/product-feature-comparison", "claim_basis": {}, "last_release_note": "Enterprise Gateway v2.19.0. The open-source repository is far staler: last tag v1.15.2 on 12 January 2026.", "prompt_transits_vendor": "depends", "prompt_path_note": "Requests hit the nearest regional edge gateway on the hosted product, or you run the gateway in your own VPC or air-gapped, where prompts, responses and logs never leave. One caveat holds in every mode: operational metrics are always sent to Portkey's control plane.", "logs_by_default": "full_content", "logs_note": "Hosted logging is on by default. Suppressing body storage is not a setting you can flip — it is enabled on request.", "retention_days": null, "retention_note": "Plan and deployment dependent: Developer logs 3 days; Production logs 30 days ([Logs](https://portkey.ai/docs/product/observability/logs)). SaaS Enterprise logs 90 days and metrics 365 days; Hybrid logs follow customer storage lifecycle rules ([Security](https://portkey.ai/docs/enterprise/security)). No single default applies to all plans.", "zdr_status": "yes", "trains_on_customer_data": "not_published", "training_note": "Nothing in the privacy policy, terms, enterprise security page or security overview addresses training.", "subprocessors_url": null, "regions_note": "Region pinning with data stored in your selected region and requests served in-region across a network of about 310 data centres, plus customer-VPC and fully disconnected air-gapped deployment.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II said to be available via an unnamed trust portal", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "described as GDPR-compliant", "status": "claimed" }, "iso27001": { "detail": "asserted in docs; no report or portal named", "status": "claimed" }, "hipaa_baa": { "detail": "described as HIPAA-compliant; no BAA offer found", "status": "claimed" } }, "guardrails_location": "both", "guardrails_note": "PII scrubbing is a gateway feature, and the gateway runs either at Portkey's edge or inside your VPC or air-gapped environment.", "acquired_date_basis": "closed", "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Free OSS self-host + free Dev tier + flat monthly platform fee ($49/mo Pro per docs) + request overage + Enterprise quote. Public pricing page shows tiers without dollar amounts and describes the free allowance as '10k recorded logs' per month; docs comparison lists 'Pro $49/Month' with '10k requests/month'. These two framings contradict.", "min_commitment": "Not stated.", "overage_note": "Dev 10K requests/mo with 'No Overage Allowed' (docs) — but pricing page says exceeding the log limit only stops recording, not requests. Pro 100K then $9/mo per additional 100K up to 3M requests. Enterprise custom. Retention-overage price and export cost not stated.", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact/'simple' caching on all tiers; semantic caching from the paid tier (Production/Pro). No cached-token pricing published. Semantic cache savings are paid-tier-only.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "'Alerts' in Production observability; spend-specific alerts not stated.", "tier": null, "supported": true }, "virtual_keys": { "note": "Virtual keys are the object budgets attach to.", "tier": null, "supported": true }, "rate_limiting": { "note": "'Granular Budget & Rate Limits' is an Enterprise bullet.", "tier": "enterprise", "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Docs contradict on availability: 'currently only available to Enterprise' vs 'Available on Enterprise plan and select Pro customers'. Pricing page lists 'Granular Budget & Rate Limits' as Enterprise-only. Minimum $1 cost limit or 100 tokens. Pre-request; key auto-expires on exhaustion. No automatic reset by default.", "tier": "enterprise", "supported": true }, "budget_caps_per_team": { "note": "Per-workspace limits supported. Team-level not stated.", "tier": null, "supported": true } }, "cost_attribution": "Custom metadata and filters on all tiers. Per key/user/team/tag/customer cost splits not explicitly stated.", "cost_export": "'Data Export to Data Lakes' on Enterprise only. CSV/API/webhook/S3 not stated; price not stated.", "product_kind_note": "Portkey calls itself a \"unified interface for interacting with over 250 AI models\" and an \"AI Gateway\" that sits in front of provider APIs, adding routing, fallbacks, guardrails and observability ([Portkey: what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). It is a routing/observability proxy in front of many providers, available as an open-source gateway and as managed SaaS ([portkey-ai/gateway on GitHub](https://github.com/portkey-ai/gateway)).", "deployment_modes": [ "saas", "self_host", "hybrid_vpc" ], "deployment_note": "The OSS gateway runs locally with `npx @portkey-ai/gateway` exposing `http://localhost:8787/v1` ([portkey-ai/gateway](https://github.com/portkey-ai/gateway)). Enterprise hybrid uses the `portkey/gateway-enterprise:latest` Docker image, Helm v3.0+ on Kubernetes v1.24+, deployed into the customer's own VPC: \"Deploy Portkey's data plane in your VPC… All prompts and responses stay in your VPC. Only metrics leave (no sensitive data)\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01). Hybrid/VPC is presented as the enterprise offering ([Portkey hybrid on GCP](https://portkey.ai/docs/self-hosting/hybrid-deployments/gcp)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap: point an existing OpenAI-compatible client at `https://api.portkey.ai/v1` and pass `x-portkey-api-key`; a first-party `portkey_ai` SDK also exists but is not required ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Requests select the upstream through a virtual key / provider slug, and sensitive headers can be forwarded \"without processing\" ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Streaming caveats: n.a.", "model_count_low": 250, "model_count_high": 2300, "model_count_source_note": "Vendor numbers disagree: \"over 250 AI models\" ([Portkey docs](https://portkey.ai/docs/introduction/what-is-portkey), undated), \"1,600+ LLMs\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01), \"2,300+ models\" ([portkey-ai/gateway](https://github.com/portkey-ai/gateway), undated).", "provider_count_low": 40, "provider_count_high": 48, "provider_count_source_note": "Vendor numbers disagree: \"45+ providers\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01) and \"2,300+ models across 40+ providers\" ([portkey-ai/gateway](https://github.com/portkey-ai/gateway), undated); the integrations index lists 48 provider cards ([Portkey LLM integrations](https://portkey.ai/docs/integrations/llms), undated).", "first_party_vs_partner": "All third-party routed: Portkey owns no models and proxies to provider APIs plus privately hosted models ([Portkey LLM integrations](https://portkey.ai/docs/integrations/llms), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).", "custom_endpoint_support": "Yes: add a \"Local/Privately hosted provider\" in Model Catalog → Add Provider, pick a compatible provider API specification and enter a \"Custom Host\", or pass `customHost` per request ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Named support for vLLM/Ollama/SageMaker specifically: n.a.", "byok_mode": "byok_only", "merchant_of_record": "n.a. Portkey's pricing page does not state who invoices model usage, and lists no token markup or credit top-up fee ([Portkey pricing](https://portkey.ai/pricing)); with BYOK the provider key is the customer's own ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys)).", "key_management_note": "Upstream credentials live in Portkey's \"secure vault\" as virtual keys, \"stored securely, never exposed in code\", with organization-level management and sharing across workspaces ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Data is AES-256 encrypted in transit and at rest, storage of request/response bodies can be disabled, and Portkey holds ISO 27001 and SOC 2 ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). In hybrid/VPC mode \"all prompts and responses stay in your VPC. Only metrics leave (no sensitive data)\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01).", "api_openai_chat_completions_note": "Yes: `POST https://api.portkey.ai/v1/chat/completions`, usable by pointing the OpenAI SDK's base URL at Portkey ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)).", "api_anthropic_messages_note": "Yes: `/messages` is listed among the endpoints Portkey exposes for provider integrations ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).", "api_openai_responses_note": "n.a. (not stated on the Portkey pages fetched in this session).", "api_embeddings_note": "Yes: `/embeddings` documented alongside `/chat/completions` and `/completions` ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).", "api_images_note": "Yes: image generation is listed among the modalities the gateway supports ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Endpoint path: n.a.", "api_audio_note": "Yes, both directions: text-to-speech and speech-to-text (plus realtime) are listed as supported modalities ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Endpoint paths: n.a.", "api_batch_note": "n.a. (no batch endpoint stated on the pages fetched).", "deployment_modes_note": "hosted SaaS ([Portkey docs](https://portkey.ai/docs/introduction/what-is-portkey)); self-host / open-source gateway via npx or Docker ([portkey-ai/gateway](https://github.com/portkey-ai/gateway)); hybrid / customer-VPC data plane with Docker + Helm on AWS, Azure, GCP and OpenShift ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), page dated 2024-10-01; [Portkey hybrid on GCP](https://portkey.ai/docs/self-hosting/hybrid-deployments/gcp)). On-prem and air-gapped: n.a. (not documented on the pages fetched).", "byok_mode_note": "byok_only on the evidence fetched: Portkey requests run on the customer's own provider credentials held as virtual keys ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)); the pricing page states no token markup or credit purchase model ([Portkey pricing](https://portkey.ai/pricing)).", "access_meta": { "qualified": [ "api_audio" ], "model_count_kind": "vendor_total", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "per_request", "rel_timeout_note": "`per_request` and `config_file`. Header `x-portkey-request-timeout`, SDK parameter `requestTimeout`, and config key `request_timeout` in milliseconds, settable at strategy level or per target with nested inheritance ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts)).", "rel_retry_config": "config_file", "rel_retry_note": "`retry.attempts` up to `5`, default retried status codes `[429, 500, 502, 503, 504]` overridable via `on_status_codes`; exponential backoff 1s / 2s / 4s / 8s / 16s ([Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).", "rel_retry_default_note": "`retry.attempts` up to `5`, default retried status codes `[429, 500, 502, 503, 504]` overridable via `on_status_codes`; exponential backoff 1s / 2s / 4s / 8s / 16s ([Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).", "rel_fallback_config": "config_file", "rel_fallback_note": "ORDERED: `strategy.mode: fallback` with an ordered `targets` array; default trigger is any non-2xx response ([Portkey fallbacks](https://docs.portkey.ai/docs/product/ai-gateway/fallbacks)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": "No upstream health check, circuit breaker or provider-ejection key appears on the fetched configs/fallback/load-balancing pages ([Portkey configs](https://docs.portkey.ai/docs/product/ai-gateway/configs)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "You can list targets that happen to be in different regions, but no cross-region failover feature is documented ([Portkey fallbacks](https://docs.portkey.ai/docs/product/ai-gateway/fallbacks)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights supported: `strategy.mode: loadbalance` with per-target `weight` (default `1`, `0` disables a target, weights are normalized) ([Portkey load balancing](https://docs.portkey.ai/docs/product/ai-gateway/load-balancing)).", "architecture_class": "interpreted_proxy", "architecture_note": "`interpreted_proxy` (JS runtime). Repo is TypeScript 96.0% and ships as an npm-launched service ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).", "self_host_data_plane": "yes", "self_host_artifact_note": "`npx @portkey-ai/gateway` (npm package `@portkey-ai/gateway`), with documented Docker, Cloudflare, AWS, Azure, GCP and Kubernetes deployments ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).", "latency_claims": [ { "url": "https://github.com/Portkey-AI/gateway", "unit": "ms", "scope": "gateway_overhead", "value": "<1", "metric": "not stated", "publisher": "vendor", "conditions": "README headline. No percentile, no RPS, no payload, no hardware, no cache state." } ], "throughput_claims": [ { "url": "https://github.com/Portkey-AI/gateway", "unit": "tokens/day", "scope": "marketing", "value": "over 10B", "metric": "fleet total", "publisher": "vendor", "conditions": "Aggregate traffic across all customers, not a measured per-instance capacity." } ], "streaming_support": "yes", "streaming_note": "Supported, with an important documented caveat: the request timeout does not trigger if at least one chunk has already arrived, so a stalled mid-stream response is not cut off by `request_timeout` ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts)).", "guard_pii": "sync_block", "guard_pii_note": "\"Detect PII\" is one of 20+ deterministic checks; guardrail actions default to `async: TRUE` (log only, no added latency), and only when a check is set synchronous **and** `Deny: TRUE` does Portkey return HTTP 446 and block. With `Deny: FALSE` the request passes through with a 246 status and the verdict recorded ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))", "guard_injection": "sync_block", "guard_injection_note": "Injection/jailbreak detection comes from partner guardrail vendors rather than a built-in classifier, and inherits the same async/sync + Deny semantics ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))", "guard_content": "sync_block", "guard_content_note": "\"Moderate Content\" is a built-in check subject to the same async default ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))", "guard_custom": "sync_block", "guard_custom_note": "Policies take the form of Regex Match, Contains / word lists, JSON Schema and JSON Keys validation, Model Whitelist, and Webhook (custom classifier hook) ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks))", "guard_model_policy": "yes", "guard_model_policy_note": "Model allowlists and denylists are configured per Integration and cascade to AI Providers, with workspace provisioning deciding which teams reach which provider; a Model Whitelist guardrail check exists as well ([Model Catalog](https://portkey.ai/docs/product/model-catalog), [Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks))", "guard_execution_location": "both", "guard_third_party": [ "Acuvity", "Aporia", "AWS Bedrock Guardrails", "Azure Content Safety", "Javelin", "Lasso", "Mistral", "Pangea", "Palo Alto Prisma AIRS", "Patronus AI", "Pillar Security", "Qualifire" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "The docs describe deny/allow verdict semantics but not behaviour when the guardrail service itself errors or times out ([Guardrails](https://docs.portkey.ai/docs/product/guardrails))", "log_content": "configurable", "log_content_note": "Full Logging stores \"Complete request payloads\", \"Full response content\", and all metrics; Metrics Only (Privacy Mode) stores usage, metadata, and errors \"without sensitive content\" ([Request logging settings](https://portkey.ai/docs/product/administration/configuring-request-logging)). Clicking a log entry reveals \"the entire raw data with the request and response objects\" ([Logs](https://docs.portkey.ai/docs/product/observability/logs))", "log_opt_out": "yes", "log_opt_out_note": "Org/workspace-level Metrics Only mode ([Request logging settings](https://portkey.ai/docs/product/administration/configuring-request-logging)), or per request the `DO NOT TRACK` control: set `debug: false` on the client or send header `x-portkey-debug: false`, keeping tokens, cost, and latency ([Logs](https://docs.portkey.ai/docs/product/observability/logs))", "tracing_support": "otel", "tracing_note": "OpenTelemetry export is supported in two modes: analytics-only spans without prompts, and complete logs including full prompt/completion (marked experimental); trace IDs group multi-step requests ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel), [Feedback](https://docs.portkey.ai/docs/product/observability/feedback))", "export_destinations": [ "OpenTelemetry", "Datadog", "Grafana", "LangSmith" ], "export_destinations_note": "Any OTLP-compatible backend, with Datadog, Grafana, and LangSmith named explicitly ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel))", "feedback_capture": "yes", "feedback_capture_note": "Yes — `POST https://api.portkey.ai/v1/feedback` with `trace_id`, `value` (−10…10), optional `weight` and `metadata`; the trace ID is returned in the `x-portkey-trace-id` response header ([Feedback](https://docs.portkey.ai/docs/product/observability/feedback))", "eval_hooks": "partial", "eval_hooks_note": "Feedback data feeds Analytics and a Prompt Eval tab inside Portkey; the same guardrail checks can run asynchronously over live traffic as scoring. Both require Portkey's platform ([Feedback](https://docs.portkey.ai/docs/product/observability/feedback), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))", "observability_requires_saas": "Enterprise deployments can run the gateway in customer infrastructure and export telemetry via OTel, but the log/analytics UI is Portkey's control plane ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel))", "log_retention_note": "Plan and deployment dependent: Developer logs 3 days; Production logs 30 days ([Logs](https://portkey.ai/docs/product/observability/logs)). SaaS Enterprise logs 90 days and metrics 365 days; Hybrid logs follow customer storage lifecycle rules ([Security](https://portkey.ai/docs/enterprise/security)). No single default applies to all plans.", "production_meta": { "sources": {}, "qualified": [ "guard_pii", "guard_injection", "guard_content", "guard_custom" ], "latency_prose": "README claims **`<1ms` latency** and a `122kb` footprint. Metric type (p50/p99/mean) is not stated, what was measured is not stated beyond \"latency\", and no RPS, payload, hardware or cache conditions are given — conditions not stated ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).", "multi_surface": { "rel_timeout": [ "per_request", "config_file" ] }, "throughput_prose": "README claims \"over 10B tokens processed everyday\" — this is a fleet-usage figure, not a measured per-instance throughput; conditions not stated ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).", "benchmark_independence": "Both figures are vendor self-published with no methodology; neither is a third-party benchmark and neither is a claim about a competitor ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway))." }, "guard_pii_default": "sync_observe", "guard_injection_default": "sync_observe", "guard_content_default": "sync_observe", "guard_custom_default": "sync_observe", "rel_notes": "Nested timeout inheritance (strategy level vs target level) is the most granular timeout model in this set; combined with `retry.attempts: 5` and exponential backoff, worst-case wall-clock time should be computed before setting client timeouts ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts), [Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).", "guard_notes": "**The single most important default in this report**: Portkey guardrails ship asynchronous, meaning a freshly configured \"PII guardrail\" annotates the log and lets the request through. Enforcement requires both switching the check to synchronous and setting Deny ([Guardrails](https://docs.portkey.ai/docs/product/guardrails)).", "fit_agent_features_note": "Portkey supports function calling and tool use. Its Responses API documents built-in tool use and reasoning, including web search, file search, Computer Use Assistant, and Remote MCP; its Assistants API documents persistent threads. ([Portkey universal API](https://docs.portkey.ai/docs/product/ai-gateway/universal-api))", "fit_byok_required": "yes", "fit_byok_required_note": "Not explicitly stated, but the documented snippets reference a provider: `provider=\"@YOUR_PROVIDER\"` and examples using `YOUR_OPENAI_API_KEY`; the self-hosted validation step creates \"a virtual key\" before the first call ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))", "fit_client_sdk_langs": [ "Python", "JavaScript" ], "fit_client_sdks_note": "Official examples use the OpenAI SDK and Portkey SDKs: Python packages `openai` and `portkey-ai`/`portkey_ai`, JavaScript packages `openai` and `portkey-ai`, and the Python `Portkey` class. Portkey also documents the Anthropic package `anthropic` for the Messages API. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "vendor_runtime_only", "fit_cloudflare_workers_note": "The Portkey Gateway runs on Cloudflare Workers and uses Cloudflare’s global edge network and data centers, according to a Portkey blog post. ([Why Portkey chose TypeScript for its AI Gateway](https://portkey.ai/blog/why-we-chose-ts-over-python-to-build-potkeys-ai-gateway))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated; \"Portkey's Gateway is open source and free to use. On managed version, Portkey offers a free plan with 10k requests per month.\" ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey))", "fit_ecosystem_notes": "Portkey documents both the hosted gateway URL `https://api.portkey.ai/v1` and local/self-hosted use at `http://localhost:8787/v1`; `PORTKEY_API_KEY` can be skipped when self-hosting. It states that the gateway supports 30+ providers and all models within those providers. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))", "fit_eval_prereqs": "A Portkey account and an API key from the Settings page, with permissions ticked on the API-key modal. No credit card, cloud account, cluster, or provider key stated ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))", "fit_eval_shape": "sdk_install", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "Yes, on the overview page: `Portkey(api_key=\"YOUR_PORTKEY_API_KEY\", provider=\"@YOUR_PROVIDER\")` then `portkey.chat.completions.create(model=\"gpt-3.5-turbo\", messages=[...])` and `print(chat_complete.choices[0].message.content)` ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey)). The make-your-first-request page names the integration options but rendered no code block in the fetched content ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "charts/portkey-gateway and charts/portkey-app in Portkey-AI/helm", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Portkey provides official Kubernetes Helm charts in the `Portkey-AI/helm` repository: `charts/portkey-gateway` for Portkey Hybrid (Gateway) and `charts/portkey-app` for Portkey Full (Full App). ([Portkey official Helm charts](https://github.com/Portkey-AI/helm))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "Portkey documents an MCP Gateway that centralizes authentication and team/user access control for MCP servers and tools, logs tool calls, and supports identity forwarding. It works with Claude Desktop, Cursor, VS Code, and other MCP-compatible clients. ([Portkey Gateway GitHub repository](https://github.com/portkey-ai/gateway))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_gateway_platform", "fit_model_gate" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented). The only permissions wording is on the API-key modal: \"Based on your access level, you might see the relevant permissions on the API key modal\" ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request)); no approval gate in the private-deployment guide either ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))", "fit_onboarding_notes": "Self-hosted validation ends with a health check, creating a virtual key, calling `https://gateway.internal.mycompany.com/v1/chat/completions` with `x-portkey-api-key`, and viewing logs/analytics; port-forward for testing via `kubectl -n portkey port-forward svc/portkey-gateway 8787:8787`. Portkey adds \"20-40ms\" latency per its own docs. Native integrations exist for OpenAI, Azure OpenAI, Anthropic, Langchain, LlamaIndex and Ollama ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2), [Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Portkey documents direct OpenAI SDK compatibility using the exact hosted base URL `https://api.portkey.ai/v1`; local deployments use `http://localhost:8787/v1`. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))", "fit_prod_prereqs": "Kubernetes v1.24+, Helm v3.0+, outbound internet access returning `HTTP/2 200` from `https://control.portkey.ai`, a storage bucket (`s3`, `gcs`, `mongo`, or `wasabi`), an auto-deployed Redis cache, Portkey control-plane access with 1Password credentials, Docker registry username/password plus a gateway JWT, an ingress with class `nginx`, and a Portkey dashboard account at app.portkey.ai ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "sdk_install", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "Portkey documents integrations with LangChain and LlamaIndex. LangChain examples use `langchain`, `langchain-openai`, `ChatOpenAI`, `createHeaders`, and `LangchainCallbackHandler`; LlamaIndex examples use `llama_index.llms.openai.OpenAI`, `ChatMessage`, and `portkey-ai`. ([Portkey LangChain agents integration](https://portkey.ai/docs/integrations/agents/langchain-agents))", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://portkey.ai/docs/introduction/make-your-first-request", "fit_self_host_docs": "command_published", "fit_self_host_install": "`docker pull portkey/gateway-enterprise:latest`; `helm repo add portkey https://portkey-ai.github.io/helm`; `helm repo update`; `helm upgrade --install portkey-gateway portkey/gateway -n portkey -f values.yaml` (plus `kubectl create namespace portkey` and an image-pull secret) ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))", "fit_terraform": "official_provider", "fit_terraform_note": "Portkey provides an official Terraform provider named `Portkey-AI/portkey` on the Terraform Registry. ([Portkey Terraform provider](https://registry.terraform.io/providers/Portkey-AI/portkey/latest/docs))", "fit_terraform_ref": "Portkey-AI/portkey", "fit_third_party_documented": {}, "fit_vendor_time_claim": "Vendor claims, verbatim: \"It takes 2 mins to integrate\" ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey)); page description \"Integrate Portkey and analyze your first LLM call in 2 minutes!\" ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request)); and for private deployment \"Get your AI gateway running in 4 simple steps. Most teams finish in under 2 hours,\" with a breakdown of Total 2 hours / Prerequisites 5 min / Credentials 10 min / Deployment 30 min / Validation 15 min ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Portkey provides a Vercel AI SDK integration. The documentation names the provider package `@portkey-ai/vercel-provider` and uses `createPortkey`. ([Portkey Vercel AI SDK integration](https://docs.portkey.ai/docs/integrations/libraries/vercel))", "fit_vercel_ai_sdk_package": "@portkey-ai/vercel-provider" }, { "slug": "respan", "name": "Respan", "company": "Keywords AI Inc.", "tagline": "Multi-provider routing with integrated tracing, evaluations and prompt management.", "category": "managed_gateway", "deployment": "both", "license": "Proprietary hosted service; separate SDK and MCP repositories have their own licenses.", "founded_year": null, "hq_country": "United States", "url_home": "https://www.respan.ai", "url_docs": "https://www.respan.ai/docs/documentation/overview", "url_pricing": "https://www.respan.ai/pricing", "url_status": "https://status.respan.ai", "url_github": null, "url_changelog": "https://www.respan.ai/docs/changelog/changelog", "url_models_api": "https://www.respan.ai/api/gateway-models/all", "github_stars": null, "model_count": 1525, "model_count_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No single public BYOK fee established; account and plan terms apply.", "seat_fee_usd_month": null, "free_tier": "Free platform plan: 100,000 logs, 1,000 scores, five datasets, two evaluators and five prompts. Model usage is funded separately.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Additional Team seats", "amount": "$15/member; five members included" } ], "self_host_cost_note": "Enterprise quote and customer infrastructure costs; no public self-hosted price.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Compatible API and exportable telemetry reduce migration work; stored prompts and gateway controls remain service-specific.", "api_surfaces": [ "OpenAI-compatible Chat Completions", "Anthropic Messages passthrough", "OpenAI Responses (route-dependent)" ], "sdk_note": "Change the base URL and API key in a compatible SDK. Gateway-specific controls use additional parameters.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": null, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": null, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Quickstart states 50–150 ms added; percentile, workload and independent methodology are not supplied.", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": true, "eu_residency": true, "eu_residency_note": "EU residency available upon request; confirm contracted region.", "zdr": null, "zdr_note": "No gateway-wide zero-retention commitment established; plan retention and caching are separate.", "air_gapped": null, "best_for": "Teams wanting gateway routing, tracing, evaluations and prompt management in one platform.", "strengths": [ "Credits and BYOK funding paths", "Ordered fallback and weighted routes", "Integrated tracing and online evaluation" ], "weaknesses": [ "Vendor reports 50–150 ms added gateway latency", "Self-hosting is enterprise and sales-led", "HIPAA offer conflicts with the public standard terms" ], "avoid_if": "You require a publicly reproducible self-host install or a confirmed zero-retention contract before evaluation.", "reviews": [], "sources": { "zdr": "https://www.respan.ai/legal/terms-of-use", "gdpr": "https://www.respan.ai/docs/documentation/compliance", "name": "https://www.respan.ai/legal/terms-of-use", "slug": "https://www.respan.ai/legal/terms-of-use", "soc2": "https://www.respan.ai/docs/documentation/compliance", "cap_mcp": "https://www.respan.ai/docs/documentation/mcp", "company": "https://www.respan.ai/legal/terms-of-use", "fit_mcp": "https://www.respan.ai/docs/documentation/mcp", "license": "https://www.respan.ai/legal/terms-of-use", "sla_url": "https://www.respan.ai/pricing", "tagline": "https://www.respan.ai/ai-gateway", "avoid_if": "https://www.respan.ai/ai-gateway", "best_for": "https://www.respan.ai/ai-gateway", "category": "https://www.respan.ai/ai-gateway", "sdk_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "url_docs": "https://www.respan.ai/ai-gateway", "url_home": "https://www.respan.ai/ai-gateway", "zdr_note": "https://www.respan.ai/legal/terms-of-use", "byok_mode": "https://www.respan.ai/docs/documentation/admin/credits-billing", "cap_evals": "https://www.respan.ai/ai-gateway", "free_tier": "https://www.respan.ai/pricing", "guard_pii": "https://www.respan.ai/docs/documentation/pii-redaction", "hipaa_baa": "https://www.respan.ai/docs/documentation/compliance", "logs_note": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "rel_notes": "https://www.respan.ai/docs/documentation/features/gateway/retries", "strengths": "https://www.respan.ai/ai-gateway", "air_gapped": "https://www.respan.ai/docs/documentation/architecture_review", "deployment": "https://www.respan.ai/pricing", "eval_hooks": "https://www.respan.ai/docs/documentation/features/evals/online-evals", "extra_fees": "https://www.respan.ai/pricing", "hq_country": "https://www.respan.ai/legal/terms-of-use", "url_status": "https://www.respan.ai/ai-gateway", "weaknesses": "https://www.respan.ai/ai-gateway", "zdr_status": "https://www.respan.ai/legal/terms-of-use", "access_meta": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "cap_budgets": "https://www.respan.ai/docs/documentation/features/gateway/limits", "cost_export": "https://www.respan.ai/docs/documentation/features/observe/export-retention", "guard_notes": "https://www.respan.ai/docs/documentation/pii-redaction", "log_content": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "model_count": "https://www.respan.ai/api/gateway-models/all", "url_pricing": "https://www.respan.ai/ai-gateway", "api_surfaces": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "cap_failover": "https://www.respan.ai/docs/documentation/features/gateway/advanced", "eu_residency": "https://www.respan.ai/docs/documentation/architecture_review", "fit_mcp_note": "https://www.respan.ai/docs/documentation/mcp", "overage_note": "https://www.respan.ai/pricing", "regions_note": "https://www.respan.ai/docs/documentation/architecture_review", "tracing_note": "https://www.respan.ai/docs/documentation/features/tracing/quickstart", "byok_fee_note": "https://www.respan.ai/docs/documentation/admin/credits-billing", "cache_support": "https://www.respan.ai/docs/documentation/features/gateway/caching", "cost_controls": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", "fit_cloud_iam": "https://www.respan.ai/docs/documentation/admin/llm-provider-keys", "pricing_model": "https://www.respan.ai/pricing", "training_note": "https://www.respan.ai/legal/terms-of-use", "url_changelog": "https://www.respan.ai/ai-gateway", "byok_mode_note": "https://www.respan.ai/docs/documentation/admin/credits-billing", "byok_supported": "https://www.respan.ai/docs/documentation/admin/credits-billing", "cap_guardrails": "https://www.respan.ai/docs/documentation/pii-redaction", "certifications": "https://www.respan.ai/docs/documentation/compliance", "config_as_code": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", "fit_eval_shape": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_model_gate": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "fit_prod_shape": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "guard_pii_note": "https://www.respan.ai/docs/documentation/pii-redaction", "latency_claims": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "min_commitment": "https://www.respan.ai/pricing", "provider_count": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "rel_retry_note": "https://www.respan.ai/docs/documentation/features/gateway/retries", "retention_days": "https://www.respan.ai/pricing", "retention_note": "https://www.respan.ai/pricing", "sla_uptime_pct": "https://www.respan.ai/pricing", "streaming_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "url_models_api": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "cap_rate_limits": "https://www.respan.ai/docs/documentation/features/gateway/limits", "deployment_note": "https://www.respan.ai/pricing", "eval_hooks_note": "https://www.respan.ai/docs/documentation/features/evals/online-evals", "fit_credit_card": "https://www.respan.ai/pricing", "guardrails_note": "https://www.respan.ai/docs/documentation/pii-redaction", "logs_by_default": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "model_count_low": "https://www.respan.ai/api/gateway-models/all", "tracing_support": "https://www.respan.ai/docs/documentation/features/tracing/quickstart", "api_surface_note": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "cap_cache_simple": "https://www.respan.ai/docs/documentation/features/gateway/caching", "cap_virtual_keys": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", "cost_attribution": "https://www.respan.ai/docs/documentation/features/gateway/limits", "deployment_modes": "https://www.respan.ai/pricing", "feedback_capture": "https://www.respan.ai/docs/documentation/features/evals/online-evals", "fit_eval_prereqs": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_eval_snippet": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_prod_prereqs": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "log_content_note": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "model_count_high": "https://www.respan.ai/api/gateway-models/all", "model_count_note": "https://www.respan.ai/api/gateway-models/all", "portability_note": "https://www.respan.ai/docs/documentation/features/observe/export-retention", "prompt_path_note": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "rel_retry_config": "https://www.respan.ai/docs/documentation/features/gateway/retries", "architecture_note": "https://www.respan.ai/docs/documentation/architecture_review", "cap_observability": "https://www.respan.ai/ai-gateway", "cap_pii_redaction": "https://www.respan.ai/docs/documentation/pii-redaction", "eu_residency_note": "https://www.respan.ai/docs/documentation/architecture_review", "fit_byok_required": "https://www.respan.ai/docs/documentation/admin/credits-billing", "fit_openai_dropin": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_vercel_ai_sdk": "https://www.respan.ai/docs/integrations/gateway/vercel-ai-sdk", "guard_pii_default": "https://www.respan.ai/docs/documentation/pii-redaction", "openai_compatible": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "product_kind_note": "https://www.respan.ai/ai-gateway", "rel_fallback_note": "https://www.respan.ai/docs/documentation/features/gateway/advanced", "streaming_support": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "throughput_claims": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "architecture_class": "https://www.respan.ai/docs/documentation/architecture_review", "cache_pricing_note": "https://www.respan.ai/docs/documentation/features/gateway/caching", "cap_cache_semantic": "https://www.respan.ai/docs/documentation/features/gateway/caching", "cap_load_balancing": "https://www.respan.ai/docs/documentation/features/gateway/load-balancing", "fit_cloud_iam_note": "https://www.respan.ai/docs/documentation/admin/llm-provider-keys", "fit_quickstart_url": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_self_host_docs": "https://www.respan.ai/pricing", "guard_failure_mode": "https://www.respan.ai/docs/documentation/pii-redaction", "guard_model_policy": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", "log_retention_note": "https://www.respan.ai/pricing", "merchant_of_record": "https://www.respan.ai/docs/documentation/admin/credits-billing", "pricing_model_note": "https://www.respan.ai/pricing", "provider_count_low": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "rel_fallback_shape": "https://www.respan.ai/docs/documentation/features/gateway/advanced", "export_destinations": "https://www.respan.ai/docs/documentation/features/observe/export-retention", "fit_ecosystem_notes": "https://www.respan.ai/docs/integrations/overview", "fit_model_gate_note": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "guardrails_location": "https://www.respan.ai/docs/documentation/pii-redaction", "key_management_note": "https://www.respan.ai/docs/documentation/admin/llm-provider-keys", "latency_overhead_ms": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "latency_source_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "provider_count_high": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "rel_fallback_config": "https://www.respan.ai/docs/documentation/features/gateway/advanced", "requires_vendor_sdk": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "self_host_cost_note": "https://www.respan.ai/pricing", "api_openai_responses": "https://www.respan.ai/docs/documentation/features/gateway/responses-api", "fit_client_sdk_langs": "https://www.respan.ai/docs/integrations/overview", "fit_client_sdks_note": "https://www.respan.ai/docs/integrations/overview", "fit_credit_card_note": "https://www.respan.ai/pricing", "fit_gateway_platform": "https://www.respan.ai/docs/integrations/gateway/model-providers/custom", "fit_onboarding_notes": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_quickstart_steps": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "self_host_data_plane": "https://www.respan.ai/pricing", "cap_prompt_management": "https://www.respan.ai/ai-gateway", "data_export_available": "https://www.respan.ai/docs/documentation/features/observe/export-retention", "deployment_modes_note": "https://www.respan.ai/pricing", "fit_eval_snippet_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_python_frameworks": "https://www.respan.ai/docs/integrations/overview", "api_anthropic_messages": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "first_party_vs_partner": "https://www.respan.ai/ai-gateway", "fit_byok_required_note": "https://www.respan.ai/docs/documentation/admin/credits-billing", "fit_openai_dropin_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_vercel_ai_sdk_note": "https://www.respan.ai/docs/integrations/gateway/vercel-ai-sdk", "prompt_transits_vendor": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "rel_retry_default_note": "https://www.respan.ai/docs/documentation/features/gateway/retries", "cache_read_discount_pct": "https://www.respan.ai/docs/documentation/features/gateway/prompt-caching", "cache_write_premium_pct": "https://www.respan.ai/docs/documentation/features/gateway/prompt-caching", "cap_conditional_routing": "https://www.respan.ai/docs/documentation/features/gateway/load-balancing", "custom_endpoint_support": "https://www.respan.ai/docs/integrations/gateway/model-providers/custom", "guard_model_policy_note": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", "model_count_source_note": "https://www.respan.ai/api/gateway-models/all", "rel_load_balancing_note": "https://www.respan.ai/docs/documentation/features/gateway/load-balancing", "self_host_artifact_note": "https://www.respan.ai/pricing", "trains_on_customer_data": "https://www.respan.ai/legal/terms-of-use", "export_destinations_note": "https://www.respan.ai/docs/documentation/features/observe/export-retention", "guard_execution_location": "https://www.respan.ai/docs/documentation/pii-redaction", "api_openai_responses_note": "https://www.respan.ai/docs/documentation/features/gateway/responses-api", "fit_gateway_platform_note": "https://www.respan.ai/docs/integrations/gateway/model-providers/custom", "fit_vercel_ai_sdk_package": "https://www.respan.ai/docs/integrations/gateway/vercel-ai-sdk", "rel_load_balancing_config": "https://www.respan.ai/docs/documentation/features/gateway/load-balancing", "fit_python_frameworks_note": "https://www.respan.ai/docs/integrations/overview", "provider_count_source_note": "https://www.respan.ai/docs/documentation/features/gateway/models-catalog", "api_anthropic_messages_note": "https://www.respan.ai/docs/documentation/features/gateway/routing-and-passthrough", "api_openai_chat_completions": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_quickstart_steps_caveat": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "observability_requires_saas": "https://www.respan.ai/docs/documentation/features/tracing/quickstart", "api_openai_chat_completions_note": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart" }, "verified_at": { "zdr": "2026-09-15", "gdpr": "2026-09-15", "name": "2026-09-15", "slug": "2026-09-15", "soc2": "2026-09-15", "cap_mcp": "2026-09-15", "company": "2026-09-15", "fit_mcp": "2026-09-15", "license": "2026-09-15", "sla_url": "2026-09-15", "tagline": "2026-09-15", "avoid_if": "2026-09-15", "best_for": "2026-09-15", "category": "2026-09-15", "sdk_note": "2026-09-15", "url_docs": "2026-09-15", "url_home": "2026-09-15", "zdr_note": "2026-09-15", "byok_mode": "2026-09-15", "cap_evals": "2026-09-15", "free_tier": "2026-09-15", "guard_pii": "2026-09-15", "hipaa_baa": "2026-09-15", "logs_note": "2026-09-15", "rel_notes": "2026-09-15", "strengths": "2026-09-15", "air_gapped": "2026-09-15", "deployment": "2026-09-15", "eval_hooks": "2026-09-15", "extra_fees": "2026-09-15", "hq_country": "2026-09-15", "url_status": "2026-09-15", "weaknesses": "2026-09-15", "zdr_status": "2026-09-15", "access_meta": "2026-09-15", "cap_budgets": "2026-09-15", "cost_export": "2026-09-15", "guard_notes": "2026-09-15", "log_content": "2026-09-15", "model_count": "2026-09-23", "url_pricing": "2026-09-15", "api_surfaces": "2026-09-15", "cap_failover": "2026-09-15", "eu_residency": "2026-09-15", "fit_mcp_note": "2026-09-15", "overage_note": "2026-09-15", "regions_note": "2026-09-15", "tracing_note": "2026-09-15", "byok_fee_note": "2026-09-15", "cache_support": "2026-09-15", "cost_controls": "2026-09-15", "fit_cloud_iam": "2026-09-15", "pricing_model": "2026-09-15", "training_note": "2026-09-15", "url_changelog": "2026-09-15", "byok_mode_note": "2026-09-15", "byok_supported": "2026-09-15", "cap_guardrails": "2026-09-15", "certifications": "2026-09-15", "config_as_code": "2026-09-15", "fit_eval_shape": "2026-09-15", "fit_model_gate": "2026-09-15", "fit_prod_shape": "2026-09-15", "guard_pii_note": "2026-09-15", "latency_claims": "2026-09-15", "min_commitment": "2026-09-15", "provider_count": "2026-09-15", "rel_retry_note": "2026-09-15", "retention_days": "2026-09-15", "retention_note": "2026-09-15", "sla_uptime_pct": "2026-09-15", "streaming_note": "2026-09-15", "url_models_api": "2026-09-15", "cap_rate_limits": "2026-09-15", "deployment_note": "2026-09-15", "eval_hooks_note": "2026-09-15", "fit_credit_card": "2026-09-15", "guardrails_note": "2026-09-15", "logs_by_default": "2026-09-15", "model_count_low": "2026-09-23", "tracing_support": "2026-09-15", "api_surface_note": "2026-09-15", "cap_cache_simple": "2026-09-15", "cap_virtual_keys": "2026-09-15", "cost_attribution": "2026-09-15", "deployment_modes": "2026-09-15", "feedback_capture": "2026-09-15", "fit_eval_prereqs": "2026-09-15", "fit_eval_snippet": "2026-09-15", "fit_prod_prereqs": "2026-09-15", "log_content_note": "2026-09-15", "model_count_high": "2026-09-23", "model_count_note": "2026-09-23", "portability_note": "2026-09-15", "prompt_path_note": "2026-09-15", "rel_retry_config": "2026-09-15", "architecture_note": "2026-09-15", "cap_observability": "2026-09-15", "cap_pii_redaction": "2026-09-15", "eu_residency_note": "2026-09-15", "fit_byok_required": "2026-09-15", "fit_openai_dropin": "2026-09-15", "fit_vercel_ai_sdk": "2026-09-15", "guard_pii_default": "2026-09-15", "openai_compatible": "2026-09-15", "product_kind_note": "2026-09-15", "rel_fallback_note": "2026-09-15", "streaming_support": "2026-09-15", "throughput_claims": "2026-09-15", "architecture_class": "2026-09-15", "cache_pricing_note": "2026-09-15", "cap_cache_semantic": "2026-09-15", "cap_load_balancing": "2026-09-15", "fit_cloud_iam_note": "2026-09-15", "fit_quickstart_url": "2026-09-15", "fit_self_host_docs": "2026-09-15", "guard_failure_mode": "2026-09-15", "guard_model_policy": "2026-09-15", "log_retention_note": "2026-09-15", "merchant_of_record": "2026-09-15", "pricing_model_note": "2026-09-15", "provider_count_low": "2026-09-15", "rel_fallback_shape": "2026-09-15", "export_destinations": "2026-09-15", "fit_ecosystem_notes": "2026-09-15", "fit_model_gate_note": "2026-09-15", "guardrails_location": "2026-09-15", "key_management_note": "2026-09-15", "latency_overhead_ms": "2026-09-15", "latency_source_note": "2026-09-15", "provider_count_high": "2026-09-15", "rel_fallback_config": "2026-09-15", "requires_vendor_sdk": "2026-09-15", "self_host_cost_note": "2026-09-15", "api_openai_responses": "2026-09-15", "fit_client_sdk_langs": "2026-09-15", "fit_client_sdks_note": "2026-09-15", "fit_credit_card_note": "2026-09-15", "fit_gateway_platform": "2026-09-15", "fit_onboarding_notes": "2026-09-15", "fit_quickstart_steps": "2026-09-15", "self_host_data_plane": "2026-09-15", "cap_prompt_management": "2026-09-15", "data_export_available": "2026-09-15", "deployment_modes_note": "2026-09-15", "fit_eval_snippet_note": "2026-09-15", "fit_python_frameworks": "2026-09-15", "api_anthropic_messages": "2026-09-15", "first_party_vs_partner": "2026-09-15", "fit_byok_required_note": "2026-09-15", "fit_openai_dropin_note": "2026-09-15", "fit_vercel_ai_sdk_note": "2026-09-15", "prompt_transits_vendor": "2026-09-15", "rel_retry_default_note": "2026-09-15", "cache_read_discount_pct": "2026-09-15", "cache_write_premium_pct": "2026-09-15", "cap_conditional_routing": "2026-09-15", "custom_endpoint_support": "2026-09-15", "guard_model_policy_note": "2026-09-15", "model_count_source_note": "2026-09-23", "rel_load_balancing_note": "2026-09-15", "self_host_artifact_note": "2026-09-15", "trains_on_customer_data": "2026-09-15", "export_destinations_note": "2026-09-15", "guard_execution_location": "2026-09-15", "api_openai_responses_note": "2026-09-15", "fit_gateway_platform_note": "2026-09-15", "fit_vercel_ai_sdk_package": "2026-09-15", "rel_load_balancing_config": "2026-09-15", "fit_python_frameworks_note": "2026-09-15", "provider_count_source_note": "2026-09-15", "api_anthropic_messages_note": "2026-09-15", "api_openai_chat_completions": "2026-09-15", "fit_quickstart_steps_caveat": "2026-09-15", "observability_requires_saas": "2026-09-15", "api_openai_chat_completions_note": "2026-09-15" }, "locked_fields": [], "published": true, "updated_at": "2026-09-23T22:06:27.334485+00:00", "vendor_status": null, "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": null, "last_release_date": null, "sla_uptime_pct": null, "sla_url": "https://www.respan.ai/pricing", "claim_basis": { "avoid_if": "editorial", "best_for": "editorial", "strengths": "editorial", "weaknesses": "editorial", "latency_claims": "vendor" }, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Managed gateway traffic passes through Respan before reaching the upstream provider.", "logs_by_default": "full_content", "logs_note": "Gateway requests are automatically logged; review retention and redaction before sending sensitive content.", "retention_days": null, "retention_note": "Free: 7 days; Team: 30 days; Enterprise: custom. Confirm the policy configured for the project.", "zdr_status": "not_published", "trains_on_customer_data": "not_published", "training_note": "No sufficiently specific gateway-wide no-training commitment established from reviewed public legal and compliance pages.", "subprocessors_url": null, "regions_note": "US East (Virginia), US West (Oregon); EU on request.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Vendor states Type II; report under NDA, not independently examined.", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA by request.", "status": "available" }, "iso27001": { "detail": "Badge on gateway marketing; certificate not reviewed.", "status": "claimed" }, "hipaa_baa": { "detail": "Compliance docs offer a BAA, but public Terms §1 disallows HIPAA-regulated use. Obtain a superseding signed agreement before relying on the offer.", "status": "conflicting" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Hosted Gateway pre-redaction happens after Respan receives the request. Telemetry redaction does not change upstream input.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Platform plans plus model usage funded with credits or BYOK. Team is advertised at $199/month billed annually.", "min_commitment": "The advertised $199/month Team rate requires annual billing; enterprise commitments are quoted.", "overage_note": "Team lists $8 per additional 100,000 logs and $1 per additional 1,000 scores. Enterprise volume pricing is quoted.", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact response caching plus separate Anthropic prompt-cache passthrough. No universal read/write discount applies.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "supported": true }, "virtual_keys": { "note": "Temporary keys with limit policies; enforcement depends on environment configuration.", "supported": true }, "rate_limiting": { "supported": true }, "spend_webhooks": { "supported": true }, "budget_caps_per_key": { "note": "Lifetime and recurring caps; verify environment enforcement.", "supported": true }, "budget_caps_per_team": { "note": "Organization and customer scopes documented; arbitrary team budgets not established.", "supported": null } }, "cost_attribution": "API key, customer_identifier and organization limits; request metadata supports attribution.", "cost_export": "Export selected records and cost fields as CSV or JSONL.", "product_kind_note": "Respan (formerly Keywords AI) combines a managed LLM gateway with observability, evaluations and prompt tooling.", "deployment_modes": [ "saas", "self_host" ], "deployment_note": "Enterprise self-hosting is sales-led.", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "partial", "api_embeddings": "not_documented", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Unified Chat Completions and provider-native passthrough are separate routes. Do not infer image generation, audio or batch inference from multimodal telemetry.", "model_count_low": 1525, "model_count_high": 1525, "model_count_source_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "Integrations directory includes provider setup and tracing integrations; no comparable exhaustive upstream count recorded.", "first_party_vs_partner": "Routes third-party models through a unified endpoint or provider-native passthrough.", "custom_endpoint_support": "Custom provider supports an OpenAI-compatible base URL and credentials; create a matching model entry.", "byok_mode": "both", "merchant_of_record": "Respan credits fund supported upstream traffic; BYOK uses the customer’s provider account.", "key_management_note": "Provider keys may be stored in the dashboard or supplied per request. Multiple weighted credentials and exact-model credential overrides are documented.", "api_openai_chat_completions_note": "OpenAI SDK Chat Completions with base URL https://api.respan.ai/api/.", "api_anthropic_messages_note": "Provider-native Anthropic passthrough under /api/anthropic/.", "api_openai_responses_note": "Documented /api/responses routes: OpenAI, Azure OpenAI and Perplexity Agent API, with route-specific model and credential requirements.", "api_embeddings_note": null, "api_images_note": null, "api_audio_note": null, "api_batch_note": null, "deployment_modes_note": "Cloud on Free and Team; Enterprise lists Cloud and Self-hosted. Public installation artifact and air-gap operation not established.", "byok_mode_note": "Use Respan credits or connect provider credentials; available funding mode depends on the model.", "access_meta": { "model_count_kind": "enumerated" }, "rel_timeout_config": "not_documented", "rel_timeout_note": null, "rel_retry_config": "per_request", "rel_retry_note": "retry_enabled, num_retries and retry_after govern current-route attempts; UI also available.", "rel_retry_default_note": "num_retries includes the initial attempt. Exponential backoff with jitter, capped at 60 seconds; no stable default attempt count recorded.", "rel_fallback_config": "per_request", "rel_fallback_note": "fallback_models is tried in order after the current route exhausts eligible attempts; preflight and fail-fast errors stop the chain.", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": null, "rel_multi_region_config": "not_documented", "rel_multi_region_note": null, "rel_load_balancing_config": "per_request", "rel_load_balancing_note": "Weighted model groups and weighted provider credentials may be configured in the dashboard or request.", "architecture_class": "vendor_saas", "architecture_note": "AWS ECS API servers, Redis queues, Celery consumers, PostgreSQL and ClickHouse; hosted architecture as documented.", "self_host_data_plane": "yes", "self_host_artifact_note": "Pricing lists enterprise self-hosting; no public gateway installation command was found.", "latency_claims": [ { "url": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "unit": "ms", "scope": "gateway_overhead", "value": "50–150", "metric": "not stated", "publisher": "vendor", "conditions": "Quickstart warning; no workload or percentile published. Not independently measured." } ], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "stream=True documented for Chat Completions.", "guard_pii": "sync_block", "guard_pii_note": "Pre-forward redaction masks detected entities; it transforms content rather than rejecting the entire request. Independent telemetry redaction; both disabled by default.", "guard_injection": "not_documented", "guard_injection_note": null, "guard_content": "not_documented", "guard_content_note": null, "guard_custom": "not_documented", "guard_custom_note": null, "guard_model_policy": "not_documented", "guard_model_policy_note": "Provider credential model lists do not establish an API-key model allowlist.", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": null, "log_content": "full_request_and_response", "log_content_note": "Gateway logging includes request/response content and usage. Telemetry-only integrations are a separate path.", "log_opt_out": "not_documented", "log_opt_out_note": null, "tracing_support": "otel", "tracing_note": "Existing OpenTelemetry exporters can send spans to Respan; tracing SDK adds nested application spans.", "export_destinations": [ "CSV", "JSONL" ], "export_destinations_note": "One-time and recurring log exports with selectable fields, filters and sampling.", "feedback_capture": "not_documented", "feedback_capture_note": null, "eval_hooks": "yes", "eval_hooks_note": "Sampled production span or completed-trace scoring using deployed evaluator pipelines.", "observability_requires_saas": "Hosted default uses Respan storage and UI; enterprise self-hosting is advertised separately.", "log_retention_note": "Plan-dependent retention: 7 days Free, 30 days Team, custom Enterprise.", "production_meta": { "qualified": [ "guard_pii" ], "multi_surface": { "rel_retry_config": [ "per_request", "dashboard_only" ], "rel_fallback_config": [ "per_request", "dashboard_only" ], "rel_load_balancing_config": [ "per_request", "dashboard_only" ] } }, "guard_pii_default": "off_by_default", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Retries happen before fallback. Invalid credentials, oversized context and model-side read timeout can be fail-fast.", "guard_notes": "Probabilistic PII detection. No documented general fail-open/fail-closed guarantee found.", "fit_agent_features_note": null, "fit_byok_required": "optional", "fit_byok_required_note": "Some catalog listings require BYOK. Credits-supported routes do not require your own provider key.", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Respan SDKs plus compatible upstream SDKs.", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Provider credentials are configured upstream; calls to Respan authenticate using a Respan API key.", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": null, "fit_credit_card": "not_required", "fit_credit_card_note": "No credit card required to start the free platform plan; model traffic still requires credits or an eligible provider key.", "fit_ecosystem_notes": "Tracing integration does not by itself prove gateway API coverage.", "fit_eval_prereqs": "Respan account and API key; credits or an eligible upstream provider key.", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Python streaming and tool-calling examples, plus curl prompt invocation.", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Respan is the request gateway; custom upstream endpoints are configurable.", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": null, "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "Hosted Platform MCP exposes Respan data and management tools; this is not evidence of a general MCP proxy.", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_kubernetes", "fit_model_gate", "fit_terraform" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "Active status does not guarantee account access. Catalog labels distinguish Credits from BYOK.", "fit_onboarding_notes": "Use the exact model ID and confirm its Credits/BYOK mode before calling.", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Chat Completions base URL and key replacement; provider-specific and Responses routes need their documented setup.", "fit_prod_prereqs": "Configure production credentials, spending limits, retention, fallback and logging policy.", "fit_prod_selfhost_shape": null, "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [ "LangChain", "LangGraph", "LlamaIndex", "Pydantic AI", "CrewAI" ], "fit_python_frameworks_note": "Gateway integrations documented separately from tracing integrations.", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": "Three setup steps before the optional prompt-management step; this is not a timed integration test.", "fit_quickstart_url": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", "fit_self_host_docs": "documented_no_command", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": null, "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "integration_documented", "fit_vercel_ai_sdk_note": "Use createOpenAI with the Respan base URL and explicitly call .chat(...) for Chat Completions.", "fit_vercel_ai_sdk_package": "@ai-sdk/openai" }, { "slug": "vercel-ai-gateway", "name": "Vercel AI Gateway", "company": "Vercel", "tagline": "Vercel-operated gateway that routes AI SDK and OpenAI-format requests to many providers with zero token markup.", "category": "managed_gateway", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2015, "hq_country": "United States", "url_home": "https://vercel.com/ai-gateway", "url_docs": "https://vercel.com/docs/ai-gateway", "url_pricing": "https://vercel.com/docs/ai-gateway/pricing", "url_status": "https://www.vercel-status.com", "url_github": null, "url_changelog": "https://vercel.com/changelog", "url_models_api": "https://ai-gateway.vercel.sh/v1/models", "github_stars": null, "model_count": 386, "model_count_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": null, "token_markup_pct": 0, "credit_fee_pct": 0, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No markup or fee on BYOK, but BYOK requires purchased AI Gateway Credits (paid tier); failed BYOK requests fall back to Vercel system credentials and are billed against credits, and BYOK spend is not covered by budgets.", "seat_fee_usd_month": 0, "free_tier": "Every team gets free AI Gateway credits on a subset of models with lower per-model rate limits; the amount is not published. Pro plans include $20 of credit across resources.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Team-wide zero data retention", "amount": "$0.10 per 1,000 requests (Pro and Enterprise)" }, { "label": "Team-wide provider allowlist", "amount": "$0.10 per 1,000 successful requests (Pro and Enterprise)" }, { "label": "Custom Reporting writes", "amount": "$0.075 per 1,000 tag / user ID / quota entity writes" }, { "label": "Custom Reporting queries", "amount": "$5 per 1,000 queries to the reporting endpoint" }, { "label": "Optional Vercel Pro developer seat", "amount": "$20 per month" }, { "label": "Optional Trace Drains delivery", "amount": "$0.05 per 1,000 traces (Pro and Enterprise)" }, { "label": "Optional Trace Drains egress", "amount": "$0.50 per GB (Pro and Enterprise)" } ], "self_host_cost_note": null, "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": null, "api_surfaces": [ "Vercel AI SDK", "OpenAI Chat Completions", "OpenAI Responses" ], "sdk_note": "Native to the Vercel AI SDK via providerOptions.gateway (order, only, sort, caching, byok); OIDC-based auth for apps deployed on Vercel.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": false, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": false, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No published gateway-overhead figure. An independent hands-on comparison reported 'consistent time to first token' against direct providers without publishing millisecond numbers; the LLM Gateway TTFT benchmark did not test Vercel AI Gateway.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": true, "eu_residency": null, "eu_residency_note": "Vercel states no data is stored permanently inside EU regions on the platform (EU caching is ephemeral); no AI Gateway-specific EU residency guarantee was found.", "zdr": true, "zdr_note": "Available on Pro and Enterprise only. Per-request ZDR costs nothing; team-wide ZDR is billed at $0.10 per 1,000 requests.", "air_gapped": false, "best_for": "Teams already shipping on Vercel with the AI SDK who want zero token markup and no credit-purchase fee.", "strengths": [ "Zero markup on tokens, including with BYOK, and no credit-purchase fee", "Public unauthenticated models endpoint with pricing and per-provider endpoint data", "Per-request ZDR and provider allowlists at no extra cost", "Deep AI SDK integration plus provider ordering, sorting, timeouts and model fallbacks", "Strong platform compliance posture: SOC 2 Type 2, ISO 27001, HIPAA on enterprise, GDPR" ], "weaknesses": [ "Narrower long-tail catalog and fewer provider deployments per open-weight model than OpenRouter", "Team-wide ZDR and team-wide provider allowlists are metered at $0.10 per 1,000 requests", "BYOK requires purchased credits, can silently fall back to Vercel credentials, and is excluded from budgets", "No semantic cache and no native content guardrails" ], "avoid_if": "You need the widest possible provider pool per model, self-hosting, or you are not otherwise on Vercel.", "reviews": [ { "url": "https://zackproser.com/blog/vercel-ai-gateway-review", "why": "Independent engineer benchmarks the gateway's latency and pricing against direct provider calls while using it for real Claude Code work.", "date": null, "type": "written", "title": "Vercel AI Gateway Review: Measured Speed & Pricing", "publisher": "zackproser.com" }, { "url": "https://www.infoq.com/news/2025/09/vercel-ai-gateway/", "why": "InfoQ's Daniel Dominguez covers the GA launch and collects developer pushback on failover behaviour and how pricing compares with OpenRouter.", "date": "2025-09-12", "type": "written", "title": "Vercel Introduces AI Gateway for Multi-Model Integration", "publisher": "InfoQ" }, { "url": "https://www.youtube.com/watch?v=pql8C8elbnk", "why": "Hands-on build of a multi-model chatbot through the gateway with the AI SDK, covering key setup, model switching and what the Vercel dashboard reports.", "date": "2025-08-24", "type": "video", "title": "New Vercel AI Gateway is Live! Here's How to Use It", "publisher": "YouTube - OrcDev" }, { "url": "https://www.reddit.com/r/LangChain/comments/1u37foh/vercel_ai_gateway_is_it_ready_for_production/", "why": "Enterprise-readiness thread where practitioners weigh gateway lock-in, auditability of agent actions and keeping an abstracted model-access layer behind it.", "date": "2026-06-11", "type": "community", "title": "Vercel AI Gateway, is it ready for production enterprise use?", "publisher": "Reddit - r/LangChain" }, { "url": "https://news.ycombinator.com/item?id=47889982", "why": "Practitioner digs into the gateway's cache-read pricing ($0.01/M flash vs $0.14/M pro) and the 64% discount versus direct and OpenRouter pricing.", "date": "2026-04-24", "type": "community", "title": "Why is cache for DeepSeek-v4 cheapest on Vercel AI Gateway?", "publisher": "Hacker News" } ], "sources": { "zdr": "https://vercel.com/docs/ai-gateway/pricing", "gdpr": "https://vercel.com/security", "soc2": "https://vercel.com/security", "license": "https://mcp.directory/blog/vercel-ai-gateway-vs-portkey-vs-openrouter-vs-litellm-2026", "sdk_note": "https://vercel.com/docs/ai-gateway/provider-options", "url_docs": "https://vercel.com/docs/ai-gateway", "url_home": "https://vercel.com/docs/ai-gateway/pricing", "zdr_note": "https://vercel.com/docs/ai-gateway/pricing", "cap_audio": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway", "cap_video": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway", "free_tier": "https://vercel.com/docs/ai-gateway/pricing", "hipaa_baa": "https://vercel.com/security", "air_gapped": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "deployment": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "extra_fees": "https://vercel.com/docs/ai-gateway/pricing", "hq_country": "https://en.wikipedia.org/wiki/Vercel", "url_status": "https://www.vercel-status.com", "zdr_status": "https://vercel.com/docs/ai-gateway/security-and-compliance/zdr", "cap_budgets": "https://vercel.com/docs/ai-gateway/byok", "model_count": "https://ai-gateway.vercel.sh/v1/models", "url_pricing": "https://vercel.com/docs/ai-gateway/pricing", "api_surfaces": "https://vercel.com/docs/ai-gateway/provider-options", "cap_failover": "https://vercel.com/docs/ai-gateway", "founded_year": "https://en.wikipedia.org/wiki/Vercel", "product_fate": "https://vercel.com/changelog", "regions_note": "https://vercel.com/legal/dpa", "tracing_note": "https://vercel.com/docs/ai-gateway/observability-and-spend/trace-drains", "byok_fee_note": "https://vercel.com/docs/ai-gateway/byok", "cap_image_gen": "https://vercel.com/pricing", "url_changelog": "https://vercel.com/changelog", "vendor_status": "https://vercel.com/blog/series-f", "byok_supported": "https://vercel.com/docs/ai-gateway/byok", "cap_embeddings": "https://vercel.com/pricing", "cap_guardrails": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "certifications": "https://security.vercel.com/", "credit_fee_pct": "https://vercel.com/docs/ai-gateway/pricing", "retention_days": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs", "sla_uptime_pct": "https://vercel.com/legal/sla", "url_models_api": "https://vercel.com/docs/ai-gateway/models-and-providers", "cap_rate_limits": "https://vercel.com/docs/ai-gateway/pricing", "logs_by_default": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs", "model_count_low": "https://ai-gateway.vercel.sh/v1/models", "tracing_support": "https://vercel.com/docs/ai-gateway/observability-and-spend/trace-drains", "cap_cache_simple": "https://vercel.com/docs/ai-gateway/provider-options", "cap_virtual_keys": "https://vercel.com/docs/ai-gateway/byok", "model_count_high": "https://ai-gateway.vercel.sh/v1/models", "model_count_note": "https://ai-gateway.vercel.sh/v1/models", "token_markup_pct": "https://vercel.com/docs/ai-gateway/pricing", "cap_observability": "https://vercel.com/pricing", "eu_residency_note": "https://vercel.com/security", "last_release_date": "https://vercel.com/changelog/hy4-preview-now-available-on-ai-gateway", "openai_compatible": "https://vercel.com/docs/ai-gateway/byok", "subprocessors_url": "https://security.vercel.com/", "cap_cache_semantic": "https://api7.ai/openrouter-vs-vercel-ai-gateway", "cap_load_balancing": "https://vercel.com/pricing", "pricing_model_note": "https://vercel.com/docs/ai-gateway/pricing", "seat_fee_usd_month": "https://vercel.com/docs/ai-gateway/pricing", "guardrails_location": "https://vercel.com/docs/ai-gateway/security-and-compliance", "latency_source_note": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway", "requires_vendor_sdk": "https://vercel.com/docs/ai-gateway/sdks-and-apis/openai-chat-completions", "data_export_available": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs", "prompt_transits_vendor": "https://vercel.com/legal/dpa", "vercel_ai_gateway_byok": "https://vercel.com/docs/ai-gateway/byok", "cap_conditional_routing": "https://vercel.com/docs/ai-gateway/provider-options", "model_count_source_note": "https://ai-gateway.vercel.sh/v1/models", "trains_on_customer_data": "https://vercel.com/docs/ai-gateway/security-and-compliance/disallow-prompt-training", "vercel_ai_gateway_budgets": "https://vercel.com/docs/ai-gateway/observability-and-spend/budgets", "vercel_ai_gateway_pricing": "https://vercel.com/docs/ai-gateway/pricing" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "soc2": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "cap_video": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "air_gapped": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-09-16", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-09-23", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "tracing_note": "2026-09-16", "byok_fee_note": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "credit_fee_pct": "2026-08-29", "url_models_api": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_low": "2026-09-23", "tracing_support": "2026-09-16", "cap_cache_simple": "2026-08-29", "cap_virtual_keys": "2026-08-29", "model_count_high": "2026-09-23", "model_count_note": "2026-09-23", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "pricing_model_note": "2026-09-16", "seat_fee_usd_month": "2026-09-16", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29", "model_count_source_note": "2026-09-23" }, "locked_fields": [], "published": true, "updated_at": "2026-09-23T22:06:27.721853+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed, with AI Gateway models and features shipping continuously. Vercel raised a Series F in September 2025 at a $9.3B valuation.", "last_release_date": "2026-08-28", "sla_uptime_pct": 99.99, "sla_url": "https://vercel.com/legal/sla", "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Hosted only, with primary processing facilities in the United States.", "logs_by_default": "full_content", "logs_note": "Vercel's own documentation contradicts itself here. The logs page shows captured inputs and outputs in a transcript view; the security overview states the gateway uses zero data retention by default; the ZDR page scopes ZDR to paid tiers. Get the behaviour for your specific tier in writing before you rely on it.", "retention_days": 30, "retention_note": "Routing attempt details are kept for 30 days. Prompts and outputs are kept for zero days where ZDR actually applies — which is the point the documentation disagrees on.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Vercel does not use your prompts or responses for training. Separately, it does not by default avoid providers that do — you must set disallowPromptTraining to filter those out.", "subprocessors_url": "https://security.vercel.com/", "regions_note": "US primary, and the data-processing agreement permits transfer anywhere in the world. No customer-VPC option and no documented gateway region pinning.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type 2, third-party audited", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA published", "status": "certified" }, "iso27001": { "detail": "listed on the trust center", "status": "certified" }, "hipaa_baa": { "detail": "listed on the trust center", "status": "certified" } }, "guardrails_location": "none", "guardrails_note": "No guardrails or PII redaction is documented. The security surface is ZDR plus routing filters on provider training policy.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "markup_only", "pricing_model_note": "Gateway credits are pay-as-you-go with no token markup or mandatory platform subscription. Optional governance, reporting and trace exports have separate charges; some require Pro or Enterprise. Include only incremental platform costs for the selected features.", "min_commitment": "None. Credits purchasable at any time with no obligation to renew; custom volume discounts on Enterprise.", "overage_note": "Trace Drains meter trace events delivered and trace data transferred; Pro plans include no allowance for either. Team-wide provider allowlist: $0.10 per 1,000 successful requests. Team-wide zero data retention: $0.10 per 1,000 requests. Custom Reporting: $0.075 per 1,000 tag/user/quota-entity writes and $5 per 1,000 reporting-endpoint queries. All billed outside credits.", "cache_support": "none", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Pricing page does not state whether the gateway caches or passes through provider caching, and no cached-token pricing is published. Because Vercel charges provider list price with 0% markup, any provider cache discount would reach the customer unchanged.", "cost_controls": { "enforcement": "mixed", "spend_alerts": { "note": "Not stated.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated as virtual keys; budgets attach to keys.", "tier": null, "supported": null }, "rate_limiting": { "note": "Platform-set per model, higher on paid tier; not user-configurable.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Per-request `only` filter free on all plans. Team-wide provider allowlist is Pro/Enterprise metered at $0.10 per 1,000 successful requests.", "tier": "paid", "supported": true }, "budget_caps_per_key": { "note": "Budgets at four scopes — team, project, API key, team member. Minimum $1, default monthly refresh. Enforcement is soft: the request that crosses the limit still completes, then HTTP 402. BYOK spend excluded from budgets.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Team-scope budgets supported (see per-key).", "tier": null, "supported": true } }, "cost_attribution": "By tags, user IDs and quota entity IDs via Custom Reporting (queried through the reporting endpoint at $5/1k queries). Per team/key/customer not explicitly stated.", "cost_export": "Reporting API plus Trace Drains (Pro/Enterprise, metered). CSV/S3/warehouse not stated.", "product_kind_note": "Vercel describes AI Gateway as \"a unified API to access hundreds of AI models through a single endpoint\", handling key management, budgets, retries and provider failover ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), updated 2026-08-27). It is a hosted routing proxy in front of third-party providers, not an inference platform.", "deployment_modes": [ "saas" ], "deployment_note": "Fully managed service reached at the `https://ai-gateway.vercel.sh/v1` base URL; no customer-installed component is documented ([Vercel AI Gateway getting started](https://vercel.com/docs/ai-gateway/getting-started), updated 2026-08-22). Tier gating applies to features rather than deployment: BYOK is \"available on the paid tier\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)) and zero-data-retention options are priced for Pro/Enterprise ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), updated 2026-08-23).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap: set the OpenAI (or Anthropic) SDK base URL to `https://ai-gateway.vercel.sh/v1` and use a Vercel AI Gateway API key; the AI SDK integration is optional, not required ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), 2026-05-11; [getting started](https://vercel.com/docs/ai-gateway/getting-started), 2026-08-22). Model discovery uses `GET /v1/models`, `GET /v1/models/{creator}/{model}/endpoints` ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), updated 2026-08-24).", "model_count_low": 386, "model_count_high": 386, "model_count_source_note": "Count of entries returned by the models API on 2026-09-23. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "n.a. Vercel's models-and-providers page states no provider total ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24); the BYOK credential table lists 5 providers whose keys can be supplied (Anthropic, OpenAI, Azure, Google Vertex AI, Amazon Bedrock) ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)).", "first_party_vs_partner": "All third-party routed; Vercel operates no model hardware and brokers access to provider endpoints, exposing per-model provider endpoints for routing ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24; [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).", "custom_endpoint_support": "n.a. Registering a customer-hosted endpoint (vLLM, Ollama, SageMaker, private URL) is not documented; BYOK is limited to \"any provider listed in our catalog\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok); [Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24).", "byok_mode": "both", "merchant_of_record": "Vercel invoices credit-mode usage, with \"no markup and no platform fee on tokens\", and Enterprise can pay by invoice ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23). In BYOK mode the upstream provider bills the customer's own account, but failed BYOK requests fall back to Vercel's system credentials and are billed to credits ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)).", "key_management_note": "Customers add provider credentials in the AI Gateway dashboard for the five supported providers, and gateway keys authenticate the caller; storage mechanism (vault/KMS) is n.a. ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)). Because it is hosted-only, requests transit Vercel; zero-data-retention options are offered at $0.10 per 1,000 requests on Pro/Enterprise ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23).", "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` on `https://ai-gateway.vercel.sh/v1`, alongside `GET /v1/models` ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), updated 2026-05-11).", "api_anthropic_messages_note": "Yes: `POST /v1/messages` plus `POST /v1/messages/count_tokens` ([Vercel AI Gateway Anthropic Messages API](https://vercel.com/docs/ai-gateway/sdks-and-apis/anthropic-messages-api), updated 2026-08-11).", "api_openai_responses_note": "Yes: a dedicated Responses API surface is documented ([Vercel AI Gateway Responses API](https://vercel.com/docs/ai-gateway/sdks-and-apis/responses)).", "api_embeddings_note": "Yes: `POST /v1/embeddings` ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), 2026-05-11).", "api_images_note": "Yes: image generation (and video generation) are documented gateway capabilities ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).", "api_audio_note": "Yes, both directions: text-to-speech (\"Generate spoken audio from text\"), speech-to-text transcription, and realtime audio over WebSockets ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).", "api_batch_note": "n.a. as an OpenAI-style Batch API; video generation is documented as running through background jobs ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).", "deployment_modes_note": "hosted SaaS only ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. (not documented).", "byok_mode_note": "both: usage can run on Vercel AI Gateway Credits, or on the customer's own provider keys, and BYOK still \"requires purchased credits\" and is \"available on the paid tier\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok); [Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23).", "access_meta": { "qualified": [ "api_audio", "api_batch" ], "model_count_kind": "enumerated", "provider_count_kind": null }, "rel_timeout_config": "per_request", "rel_timeout_note": "`providerOptions.gateway.providerTimeouts` in milliseconds, minimum `1000`, maximum `789000`. It measures time until the provider starts streaming and is cleared on the first token. **BYOK only.** Default value not stated ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).", "rel_retry_config": "not_documented", "rel_retry_note": "No retry count or backoff strategy on the fetched pages; the documented recovery path is the fallback model list. Default: `n.a.` ([Vercel model fallbacks](https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED: a `models` array tried in order (with a top-level `models` shorthand on the Chat Completions surface) ([Vercel model fallbacks](https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_configurable", "rel_health_check_note": "It is automatic provider selection rather than a configurable health check or breaker: default provider choice is based on \"recent uptime and latency\" ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "`not_documented` ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).", "rel_load_balancing_config": "per_request", "rel_load_balancing_note": "`per_request`, without weights. Controls are `order`, `only` and `sort` (`'cost'`, `'ttft'`, `'tps'`); no weight key is documented ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).", "architecture_class": "vendor_saas", "architecture_note": "No runtime is disclosed anywhere on the fetched Vercel AI Gateway docs, so it is classified `vendor_saas` on that basis ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).", "self_host_data_plane": "no", "self_host_artifact_note": "No self-host artifact is offered on the fetched docs ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported. Documented caveats: \"Some providers don't support stream cancellation\" and \"You may still be charged for timed-out requests\"; the provider timeout only covers time-to-first-token and is cleared once streaming starts ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).", "guard_pii": "not_documented", "guard_pii_note": "The security and compliance section documents allowlists and ZDR, and states nothing about PII detection ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))", "guard_injection": "not_documented", "guard_injection_note": "`not_documented` ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))", "guard_content": "not_documented", "guard_content_note": "`not_documented` ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))", "guard_custom": "not_documented", "guard_custom_note": "No regex/word-list/JSON-schema policy engine ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))", "guard_model_policy": "yes", "guard_model_policy_note": "Provider Allowlist and Model Allowlist restrict what a team can call; \"A request only returns 403 when no allowed provider can serve it\", opt-in on Pro and Enterprise ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "There is no guardrail service; the analogous documented failure is ZDR routing, which fails closed with a `no_providers_available` 400 error when no ZDR provider can serve the model ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr))", "log_content": "configurable", "log_content_note": "The Logs view shows a transcript of request inputs and outputs, i.e. prompt/completion text is visible, yet the same docs say \"AI Gateway uses zero data retention by default\" and \"permanently deletes your prompts and responses after requests complete\" ([Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs), [Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance)) — see Contradictions", "log_opt_out": "yes", "log_opt_out_note": "Team-wide ZDR from the AI Gateway Settings tab, or per request `providerOptions.gateway.zeroDataRetention: true`; team-wide ZDR costs $0.10 per 1,000 requests while per-request ZDR is free on Pro/Enterprise ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr))", "tracing_support": "otel", "tracing_note": "Native OTLP/HTTP Trace Drains on Pro and Enterprise; separate delivery and egress charges. Request, routing, model-attempt and provider spans contain metadata, not prompt or completion content. Request traces do not establish complete application or agent tracing.", "export_destinations": [ "CSV export" ], "export_destinations_note": "CSV and JSON export from the logs view; no streaming destinations documented ([Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability))", "observability_requires_saas": null, "log_retention_note": "Routing/observability details are kept 30 days by default, with Observability Plus extending the window ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability), [Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` No numeric latency benchmark is published. Per-request observability is provided instead: response metadata `responseTimeMs`, plus `providerTimeout` and `configuredTimeoutMs`, and error code `PROVIDER_TIMEOUT` ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).", "multi_surface": {}, "throughput_prose": "`n.a.` `sort: 'tps'` exists as a routing preference but no measured figure is published ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).", "benchmark_independence": "No figures published ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "The timeout control being BYOK-only is the sharpest gotcha here: on Vercel-managed credentials there is no documented way to bound a slow provider ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).", "guard_notes": "Governance here is routing-level (which provider/model, and whether the provider retains data), not content-level. BYOK keys are skipped under ZDR unless explicitly marked ZDR-compliant ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr)).", "fit_agent_features_note": "Tool calling is documented through OpenAI Chat Completions and the AI SDK, including `tools`, `tool(...)`, `inputSchema`, `execute`, and `stopWhen: stepCountIs(5)` for continued processing after tool execution. The AI SDK Gateway provider also documents provider-executed tools and built-in tools including `perplexity_search`, `exa_search`, `tako_search`, and `parallel_search`; sessions, threads, and A2A are not stated. ([AI SDK integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/ai-sdk))", "fit_byok_required": "no", "fit_byok_required_note": "No. \"AI Gateway also supports Bring Your Own Key (BYOK) for any provider listed in our catalog,\" i.e. optional ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing)); the quickstart uses an AI Gateway key or OIDC ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))", "fit_client_sdk_langs": [ "TypeScript", "Python" ], "fit_client_sdks_note": "Documented clients include OpenAI client libraries with TypeScript and Python examples, the TypeScript `ai` package, and official Python SDKs for OpenAI (`pip install openai`) and Anthropic (`pip install anthropic`); both Python SDKs provide async clients. The ecosystem documentation also lists LangChain, LlamaIndex, Mastra, Pydantic AI, LiteLLM, and Langfuse integrations. ([AI Gateway Python SDKs](https://vercel.com/docs/ai-gateway/sdks-and-apis/python))", "fit_cloud_iam": "oidc_only", "fit_cloud_iam_note": "No AWS/Azure/GCP IAM reuse is documented; instead Vercel documents OIDC-based auth with `VERCEL_OIDC_TOKEN` so deployments can call the gateway without a separate API key. ([Vercel AI Gateway authentication docs](https://vercel.com/docs/ai-gateway/authentication))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "No for the first call: \"A payment method is not required for the first API call; a free tier is available… Your free credits start when you make your first AI Gateway request.\" ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))", "fit_ecosystem_notes": "AI Gateway provides unified access to hundreds of models through a single endpoint, with integrations for the AI SDK, OpenAI Chat Completions, OpenAI Responses, Anthropic Messages, and other frameworks. The Vercel documentation describes API-key and Vercel OIDC authentication, dashboard-based usage management and observability, provider BYOK, and availability through AWS Marketplace private offers; self-hosting and single-cloud requirements are not stated. ([Vercel AI Gateway overview](https://vercel.com/ai-gateway))", "fit_eval_prereqs": "None stated as prerequisites; within the steps you create `.env.local` with `AI_GATEWAY_API_KEY` (an AI Gateway API key), or use OIDC tokens instead. No credit card, cloud account, or cluster stated ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))", "fit_eval_shape": "sdk_install", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `streamText({ model: 'openai/gpt-5.6-sol', prompt: 'Invent a new holiday and describe its traditions.' })`, streaming the response and printing token usage and finish reason ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. (not documented)", "fit_mcp": "client_tooling", "fit_mcp_note": "Per Vercel's AI Gateway versus MCP comparison page, the Vercel AI SDK provides `createMCPClient` for converting MCP tools into AI SDK tools, while `mcp-handler` supports building and hosting MCP servers. The page also identifies `mcp.vercel.com` as an official remote MCP server with OAuth. ([AI Gateway vs MCP](https://vercel.com/i/ai-gateway-vs-mcp))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_gateway_platform", "fit_kubernetes", "fit_terraform" ] }, "fit_model_gate": "tier_limited", "fit_model_gate_note": "No approval gate, but a catalogue limit on the free tier: it includes \"a subset of models, not the full catalog,\" and other models require purchasing AI Gateway Credits ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))", "fit_onboarding_notes": "The hub page splits quickstarts by modality (text, image, video, realtime voice agents, speech, audio transcription) and advertises \"350+ more models\" for text plus \"unified access to 200+ models through the AI SDK for Python,\" so the effort is a Node/TS project scaffold rather than a bare curl ([Getting Started with AI Gateway](https://vercel.com/docs/ai-gateway/getting-started), [Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Existing OpenAI client libraries can be used by setting the base URL to `https://ai-gateway.vercel.sh/v1` and authenticating with an AI Gateway API key or Vercel OIDC token. ([OpenAI Chat Completions integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/openai-chat-completions))", "fit_prod_prereqs": "A Vercel team account and AI Gateway credits beyond the free tier: \"Every Vercel team account gets access to both a free tier and a paid tier for AI Gateway Credits,\" and buying credits requires \"Choose your payment method and click on Confirm and Pay\" ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "sdk_install", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "Official documentation covers LangChain through the Chat Completions endpoint and LlamaIndex through `llama-index-llms-vercel-ai-gateway`; the LlamaIndex class is `VercelAIGateway` from `llama_index.llms.vercel_ai_gateway`. ([AI Gateway Python SDKs](https://vercel.com/docs/ai-gateway/sdks-and-apis/python))", "fit_quickstart_steps": 5, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://vercel.com/docs/ai-gateway/getting-started", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented)", "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Vercel AI Gateway works with the Vercel AI SDK. The documented npm package is `ai` (`npm install ai@latest`), and the AI Gateway provider is available through `@ai-sdk/gateway`; OpenAI-compatible access can also use `@ai-sdk/openai-compatible`. ([AI SDK integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/ai-sdk))", "fit_vercel_ai_sdk_package": "@ai-sdk/gateway" }, { "slug": "agentgateway", "name": "agentgateway", "company": "agentgateway project, an Agentic AI Foundation initiative under the Linux Foundation (created by Solo.io)", "tagline": "Rust single-binary gateway for LLM, MCP, A2A and ordinary HTTP traffic, run entirely on your own infrastructure.", "category": "open_source", "deployment": "self_host_only", "license": "Apache-2.0", "founded_year": null, "hq_country": null, "url_home": "https://agentgateway.dev", "url_docs": "https://agentgateway.dev/docs/standalone/latest/", "url_pricing": null, "url_status": null, "url_github": "https://github.com/agentgateway/agentgateway", "url_changelog": "https://github.com/agentgateway/agentgateway/releases", "url_models_api": null, "github_stars": 4985, "model_count": 1002, "model_count_note": "The project's own Model and Provider Cookbook headlines \"1002+ Models\" across its provider catalogue, and any additional model is reachable through the custom/OpenAI-compatible provider, so the number is a floor rather than a closed list ([Model and Provider Cookbook](https://agentgateway.dev/models)).", "provider_count": 44, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Every request runs on your own upstream credentials, supplied inline, from an `$ENV_VAR`, from a file-backed env var, from a Kubernetes secret, or passed straight through from the caller's token; the project bills nothing and issues no keys of its own ([API keys](https://agentgateway.dev/docs/standalone/latest/llm/api-keys/)).", "seat_fee_usd_month": 0, "free_tier": "Everything is free: the whole gateway is Apache-2.0 and there is no hosted tier, no account, no seat and no request meter. You pay your infrastructure and your model providers ([agentgateway/agentgateway](https://api.github.com/repos/agentgateway/agentgateway)).", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "Self-hosting is the only mode and it is free of vendor charges. The binary is a single Rust process (13-22 MB resident in the project's own benchmark runs), so the practical floor is one small container plus, optionally, SQLite or PostgreSQL if you want USD budgets, the cost dashboard or database-backed request logs ([Database](https://agentgateway.dev/docs/standalone/latest/setup/database/), [benchmark](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm-part-2/)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "Low lock-in by construction: Apache-2.0 code, a config file you own, your own database and your own upstream provider keys, so leaving means deleting a binary. The migration door swings inward too - `agentgateway import --from litellm` converts a LiteLLM proxy config with per-field compatibility findings - but there is no documented exporter back out to another gateway, and config is agentgateway-specific YAML or Gateway API CRs. Operational portability caveats: USD budgets, the cost dashboard and persisted logs depend on a database you must run, and the model cost catalog is populated by importing price files (`agctl catalog import --source models.dev`) rather than by a vendor-maintained feed ([import](https://agentgateway.dev/docs/standalone/latest/configuration/import/), [model costs](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/costs/)).", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Responses", "OpenAI Embeddings", "OpenAI Models", "OpenAI Realtime (WebSocket)", "Anthropic Messages", "Anthropic token count", "Cohere-style Rerank", "Google Gemini generateContent", "Provider passthrough", "Config resource API (admin)" ], "sdk_note": "No client SDK exists or is needed: you point an OpenAI-compatible client at the gateway, e.g. `openai.OpenAI(api_key=\"anything\", base_url=\"http://localhost:4000/v1\")`. Operations use the `agctl` CLI and a config resource API rather than a client library ([OpenAI SDK integration](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/), [agctl](https://agentgateway.dev/docs/standalone/latest/operations/agctl/)).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": false, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 0.863, "latency_source_note": "All latency numbers are project-published, authored by Solo.io's Lin Sun against a mock LLM backend with Fortio, so they isolate proxy overhead rather than model time; scripts are open at [linsun/litellm-agw-perf](https://github.com/linsun/litellm-agw-perf). The recorded 0.863 ms is the p50 from the most recent maximum-throughput run (32 connections, 1 KB payload) in the 13 Aug 2026 post ([Rust-mode benchmark](https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/)); earlier runs report p50 0.831 ms / p99 1.970 ms at max load and p50 0.227 ms / p99 0.436 ms when held at a fixed 3,000 QPS ([part 1](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/), [part 2](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm-part-2/)). No independent benchmark was found.", "throughput_rps": 35502, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Not applicable in the vendor sense: there is no vendor-operated plane, so residency is wherever you deploy the binary. No residency, region or data-boundary statement appears on the project pages checked ([introduction](https://agentgateway.dev/docs/standalone/latest/about/introduction/), [enterprise](https://agentgateway.dev/enterprise)).", "zdr": null, "zdr_note": "Not applicable rather than offered: prompts never reach a project-operated service, so there is no vendor retention to disable. Whether prompt bodies are stored at all is your own logging choice ([Access logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/)).", "air_gapped": null, "best_for": "Platform teams that already run Kubernetes or Envoy-style infrastructure and want one Rust data plane in front of LLM, MCP, A2A and ordinary HTTP traffic, with per-key USD budgets and guardrails defined in version-controlled YAML.", "strengths": [ "Single Apache-2.0 Rust binary covering LLM, MCP, A2A, HTTP, gRPC and TCP in one data plane, with no hosted dependency", "Guardrails default to enforcing: regex guards default to masking and every external moderation guard defaults to reject, unlike gateways that ship log-only", "Per-virtual-key USD and token budgets with pre-request Block enforcement, plus a built-in cost dashboard that needs no Prometheus or Grafana", "Vendor-published Fortio benchmarks put it at ~35,000 QPS with sub-2 ms p99 and 13-22 MB resident memory against a mock backend", "Neutral governance: Linux Foundation donation in 2025, AAIF hosted project in 2026, 300+ contributors across 60+ organisations", "Documented one-command migration from LiteLLM proxy configs with per-field compatibility findings" ], "weaknesses": [ "The homepage markets \"semantic caching\" but no semantic or response cache appears anywhere in the documentation - only provider prompt-cache breakpoint control", "Response guardrails do not run on streamed responses by default (`streaming: Disabled`), and `mask` silently never applies to streams", "`routing.failover` alone does not fail over: the docs warn you must also configure `health.eviction`, and the triggering request still fails unless retries are set", "The admin address that serves the config resource API has no authentication at all - network isolation is the only control", "No published SLA, status page, SOC 2 / ISO / HIPAA posture, subprocessor list or log-retention setting; compliance is entirely the operator's job", "No documented image, audio, video or batch endpoint, no Terraform provider, no Vercel AI SDK package and no LangChain/LlamaIndex integration guide" ], "avoid_if": "You want a hosted control plane with dashboards, prompt management, evals and a compliance posture you can point an auditor at, or you have no appetite to run and upgrade a proxy yourself.", "reviews": [ { "url": "https://dev.to/spacewander/agentgateway-review-a-feature-rich-new-ai-gateway-53lm", "why": "Line-by-line source review of MCP session statefulness, multi-backend tool multiplexing, OpenAPI-to-MCP conversion limits, OAuth/JWKS handling and A2A support; the most technically specific outside assessment found. Author affiliation is not stated on the page.", "date": "2025-12-02", "type": "written", "title": "Agentgateway Review: A Feature-Rich New AI Gateway", "publisher": "dev.to - spacewander" }, { "url": "https://blog.mygraphql.com/en/posts/ai/ai-devops/agent-gateway/agentgateway-impl/", "why": "Independent code-level walkthrough of the Cargo workspace, crate responsibilities, Tokio threading model and devcontainer setup, and openly questions whether another proxy is needed alongside Nginx, HAProxy and Envoy Gateway. Reports no benchmarks or functional tests.", "date": "2025-09-05", "type": "written", "title": "Analysis of the AI Agent Bus Gateway 'agentgateway' - Implementation", "publisher": "blog.mygraphql.com - Mark Zhu" }, { "url": "https://www.fmind.dev/articles/agentgateway-vs-litellm/", "why": "Customer-facing evaluation of source trees, licences, published advisories and docs at agentgateway v1.4.1 vs LiteLLM 1.9x. Explicitly not a load test, and it pushes back on the project's own benchmark - noting the 12 GB LiteLLM memory figure conflicts with LiteLLM's own 359 MB result on a different harness. The author discloses being an AAIF Ambassador, so treat it as semi-affiliated.", "date": "2026-08-16", "type": "written", "title": "LiteLLM is the known option. agentgateway is the open one.", "publisher": "fmind.dev - Médéric Hurier" }, { "url": "https://www.reddit.com/r/rust/comments/1mq4gvm/agentgateway_a_fast_feature_rich_kubernetes/", "why": "Project-introduction thread in r/rust where maintainers field questions from Rust practitioners. Useful for community reaction, but the opening post is by a project contributor rather than an independent reviewer.", "date": "2025-08-14", "type": "community", "title": "Agentgateway: a fast, feature rich, Kubernetes native proxy", "publisher": "Reddit - r/rust" } ], "sources": { "cap_mcp": "https://agentgateway.dev/docs/standalone/latest/mcp/guardrails/setup/", "company": "https://agentgateway.dev/blog/2026-06-04-agentgateway-joins-aaif/", "fit_mcp": "https://agentgateway.dev/docs/standalone/latest/mcp/guardrails/setup/", "license": "https://api.github.com/repos/agentgateway/agentgateway", "category": "https://github.com/agentgateway/agentgateway", "url_docs": "https://agentgateway.dev/docs/standalone/latest/", "api_audio": "https://agentgateway.dev/docs/standalone/latest/llm/api-types/realtime/", "api_batch": "https://agentgateway.dev/docs/standalone/latest/llm/about/", "byok_mode": "https://agentgateway.dev/docs/standalone/latest/llm/api-keys/", "free_tier": "https://api.github.com/repos/agentgateway/agentgateway", "guard_pii": "https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/regex/", "api_images": "https://agentgateway.dev/docs/standalone/latest/llm/about/", "deployment": "https://agentgateway.dev/docs/standalone/latest/about/introduction/", "eval_hooks": "https://agentgateway.dev/docs/standalone/latest/llm/observability/", "url_github": "https://github.com/agentgateway/agentgateway", "zdr_status": "https://agentgateway.dev/docs/standalone/latest/about/introduction/", "access_meta": "https://agentgateway.dev/models", "cap_budgets": "https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/budget-limits/per-key/", "log_content": "https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/", "model_count": "https://agentgateway.dev/models", "api_surfaces": "https://agentgateway.dev/docs/standalone/latest/llm/about/", "github_stars": "https://github.com/agentgateway/agentgateway", "product_fate": "https://agentgateway.dev/blog/2026-06-04-agentgateway-joins-aaif/", "cache_support": "https://agentgateway.dev/docs/standalone/latest/configuration/traffic-management/llm/", "cost_controls": "https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/budget-limits/per-key/", "fit_cloud_iam": "https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/attribution/", "pricing_model": "https://api.github.com/repos/agentgateway/agentgateway", "url_changelog": "https://github.com/agentgateway/agentgateway/releases", "vendor_status": "https://agentgateway.dev/blog/2026-06-04-agentgateway-joins-aaif/", "byok_supported": "https://agentgateway.dev/docs/standalone/latest/llm/api-keys/", "cap_guardrails": "https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/", "certifications": "https://agentgateway.dev/enterprise", "config_as_code": "https://agentgateway.dev/docs/standalone/latest/reference/configuration/schema/", "fit_kubernetes": "https://agentgateway.dev/docs/standalone/latest/setup/install/helm/", "latency_claims": "https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/", "provider_count": "https://agentgateway.dev/models", "throughput_rps": "https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/", "logs_by_default": "https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/", "tracing_support": "https://agentgateway.dev/docs/standalone/latest/observability/traces/setup/", "cost_attribution": "https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/attribution/", "deployment_modes": "https://agentgateway.dev/docs/standalone/latest/about/introduction/", "rel_retry_config": "https://agentgateway.dev/docs/standalone/latest/configuration/resiliency/retries/", "fit_openai_dropin": "https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/", "guard_pii_default": "https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/regex/", "last_release_date": "https://api.github.com/repos/agentgateway/agentgateway/releases/latest", "openai_compatible": "https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/", "streaming_support": "https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/", "throughput_claims": "https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/", "architecture_class": "https://api.github.com/repos/agentgateway/agentgateway", "cap_cache_semantic": "https://agentgateway.dev/docs/standalone/latest/configuration/traffic-management/llm/", "guard_failure_mode": "https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/", "guard_model_policy": "https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/virtual-keys/", "rel_fallback_shape": "https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/", "security_incidents": "https://api.github.com/repos/agentgateway/agentgateway/security-advisories", "export_destinations": "https://agentgateway.dev/docs/standalone/latest/observability/traces/setup/", "guardrails_location": "https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/", "latency_overhead_ms": "https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/", "fit_quickstart_steps": "https://agentgateway.dev/docs/standalone/latest/quickstart/llm/", "self_host_data_plane": "https://agentgateway.dev/docs/standalone/latest/setup/install/binary/", "fit_self_host_install": "https://agentgateway.dev/docs/standalone/latest/setup/install/binary/", "first_party_vs_partner": "https://agentgateway.dev/docs/standalone/latest/llm/about/", "prompt_transits_vendor": "https://agentgateway.dev/docs/standalone/latest/about/introduction/", "rel_health_check_config": "https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/", "trains_on_customer_data": "https://agentgateway.dev/docs/standalone/latest/about/introduction/" }, "verified_at": { "cap_mcp": "2026-09-02", "fit_mcp": "2026-09-02", "license": "2026-09-02", "api_audio": "2026-09-02", "api_batch": "2026-09-02", "byok_mode": "2026-09-02", "guard_pii": "2026-09-02", "api_images": "2026-09-02", "deployment": "2026-09-02", "eval_hooks": "2026-09-02", "url_github": "2026-09-02", "zdr_status": "2026-09-02", "cap_budgets": "2026-09-02", "log_content": "2026-09-02", "model_count": "2026-09-02", "github_stars": "2026-09-23", "cache_support": "2026-09-02", "fit_cloud_iam": "2026-09-02", "pricing_model": "2026-09-02", "vendor_status": "2026-09-02", "byok_supported": "2026-09-02", "cap_guardrails": "2026-09-02", "fit_kubernetes": "2026-09-02", "provider_count": "2026-09-02", "throughput_rps": "2026-09-02", "logs_by_default": "2026-09-02", "tracing_support": "2026-09-02", "rel_retry_config": "2026-09-02", "fit_openai_dropin": "2026-09-02", "guard_pii_default": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "architecture_class": "2026-09-02", "cap_cache_semantic": "2026-09-02", "guard_failure_mode": "2026-09-02", "guard_model_policy": "2026-09-02", "rel_fallback_shape": "2026-09-02", "guardrails_location": "2026-09-02", "latency_overhead_ms": "2026-09-02", "fit_quickstart_steps": "2026-09-02", "self_host_data_plane": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "rel_health_check_config": "2026-09-02", "trains_on_customer_data": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-23T22:06:27.501324+00:00", "vendor_status": "foundation", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Alive and shipping fast. Created by Solo.io, donated to the Linux Foundation in 2025 and accepted as an Agentic AI Foundation hosted project in 2026 - the fourth AAIF initiative - with 300+ active contributors across 60+ organisations including CoreWeave, Red Hat, Solo.io, Adobe, Salesforce, Amdocs and Microsoft. v1.5.0 shipped 27 Aug 2026 and the default branch was pushed the day this entry was verified. Commercial support exists as a separate distribution, Solo Enterprise for agentgateway, which is a different product with its own docs and licensing ([AAIF announcement](https://agentgateway.dev/blog/2026-06-04-agentgateway-joins-aaif/), [Enterprise distributions](https://agentgateway.dev/enterprise)).", "last_release_date": "2026-08-27", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "v1.5.0, published 27 Aug 2026. Release notes live on GitHub Releases; the docs also carry a release-notes page. Nightly builds are published as GitHub Actions artifacts and as the `0.0.0-latest-dev` Helm chart version ([releases](https://github.com/agentgateway/agentgateway/releases), [Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/)).", "prompt_transits_vendor": "no", "prompt_path_note": "Prompts go from your client to your own agentgateway process and straight on to the provider you configured; there is no project-operated plane in the path and no telemetry call-home documented. In `passthrough` mode with `opaque` bodies the gateway does not even interpret the payload ([passthrough](https://agentgateway.dev/docs/standalone/latest/llm/api-types/passthrough/)).", "logs_by_default": "metadata_only", "logs_note": "A structured access log is written to stdout for every request by default (key=value, switchable to JSON) with `gen_ai.*` attributes and duration, but prompt and completion bodies are excluded until you opt in. CEL `filter`, `add` and `remove` let you shape or suppress entries ([Access logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/)).", "retention_days": null, "retention_note": "No retention or pruning setting is documented. Logs go to stdout, to an OTLP endpoint, or into a `request_logs` table in your own SQLite or PostgreSQL database, and lifecycle is entirely yours ([database logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/database/), [Database](https://agentgateway.dev/docs/standalone/latest/setup/database/), [Storage modes](https://agentgateway.dev/docs/standalone/latest/setup/storage/)).", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "not_applicable by construction, not by policy: no prompt ever reaches a project-operated service, and no training, telemetry or data-use statement appears on the pages checked ([introduction](https://agentgateway.dev/docs/standalone/latest/about/introduction/), [FAQs](https://agentgateway.dev/docs/standalone/latest/faqs/), [Enterprise distributions](https://agentgateway.dev/enterprise)).", "subprocessors_url": null, "regions_note": "n.a. - no vendor regions exist. You run the process wherever you choose; multi-region behaviour is whatever your own deployment and virtual-model routing do ([virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/)).", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "No SOC 2 statement, trust portal or report reference on the project pages checked", "status": "not_published" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "No DPA offer; there is no vendor to sign one for the upstream project", "status": "not_published" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "detail": "No BAA offer found", "status": "not_published" } }, "guardrails_location": "self_hosted", "guardrails_note": "Guards execute inside your own gateway process; the external ones (OpenAI Moderation, Bedrock Guardrails, Google Model Armor, Azure Content Safety, your own webhook) are outbound calls your gateway makes to services you configure, not to a project-run service ([Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/)).", "acquired_date_basis": null, "security_incidents": [ { "id": "GHSA-mvgg-jvj2-4frq", "url": "https://github.com/agentgateway/agentgateway/security/advisories/GHSA-mvgg-jvj2-4frq", "date": "2026-07-27", "summary": "Stateful MCP sessions can cross routes and overwrite the authorization policy: reusing a session ID on a route targeting a different backend could combine the original backend connection with the new route's policy. Affects v1.3.1 and older.", "severity": "high" }, { "id": "GHSA-jwm2-83f3-52xc", "url": "https://github.com/agentgateway/agentgateway/security/advisories/GHSA-jwm2-83f3-52xc", "date": "2026-06-29", "summary": "AgentgatewayPolicy allows cross-namespace backends without ReferenceGrant.", "severity": "medium" }, { "id": "GHSA-v2x6-wwfw-r2rq", "url": "https://github.com/agentgateway/agentgateway/security/advisories/GHSA-v2x6-wwfw-r2rq", "date": "2026-03-04", "summary": "Missing parameter sanitization in MCP to OpenAPI conversion.", "severity": "medium" } ], "pricing_model": "oss_no_vendor_price", "pricing_model_note": "There is no price of any kind. No pricing page exists on the site (navigation is Docs, Standalone, Kubernetes, Models, Blog, Enterprise, Community), no seats, no request meter and no token markup, because nothing is vendor-operated. The only commercial path is a third-party enterprise distribution, Solo Enterprise for agentgateway, whose pricing is also not published ([Enterprise distributions](https://agentgateway.dev/enterprise), [FAQs](https://agentgateway.dev/docs/standalone/latest/faqs/)).", "min_commitment": "None. Apache-2.0 download, no account, no contract.", "overage_note": "n.a. - no metered plan exists, so there is nothing to overrun. Your own `remoteRateLimit` token buckets and per-key budgets are the only limits, and you set them ([per-key budgets](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/budget-limits/per-key/)).", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "The only caching feature is `promptCaching`, which controls provider-side prompt-cache breakpoints (`cacheSystem`, `cacheMessages`, `cacheTools`, `minTokens`, `cacheMessageOffset`) so the provider's own cache discounts apply; the gateway stores no responses itself and publishes no cached-token pricing. Worth flagging: the homepage markets \"token budgets, semantic caching, and prompt redaction\", but no semantic or response cache appears on any documentation page fetched - the semantic work in the project is semantic *routing*, not caching ([LLM policies](https://agentgateway.dev/docs/standalone/latest/configuration/traffic-management/llm/), [homepage](https://agentgateway.dev)).", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "No alerting feature. You would alert off your own Prometheus metrics or the access log; `onBudgetExceeded: Audit` records a violation without blocking.", "tier": null, "supported": null }, "virtual_keys": { "note": "`llm.policies.apiKey.keys[]`, created in the UI at `/ui/llm/keys`, in the config file, or through the config resource API. Callers present `Authorization: Bearer $VIRTUAL_KEY`.", "tier": null, "supported": true }, "rate_limiting": { "note": "`remoteRateLimit` token buckets with `type: tokens` and per-user descriptors, evaluated in two phases (request estimate, then response actuals), returning 429 when empty. No database needed and it works in both standalone and Kubernetes modes.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated on any page fetched. Guardrail webhooks exist, but no budget or spend webhook.", "tier": null, "supported": null }, "model_allowlists": { "note": "Per virtual key via `allowedModels`, which accepts wildcards such as `[\"gpt-5*\"]`.", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "`budgets[{name, limit:{unit: USD|Tokens, amount}, window:{rolling: 24h}, onBudgetExceeded: Block|Audit}]`. USD budgets require `config.database` (SQLite or PostgreSQL) plus a model cost catalog; token budgets do not. Checked before the request is forwarded.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Not stated. Budgets attach to virtual keys, and rate-limit descriptors can key on a user or JWT claim, but no team or workspace object appears in the docs fetched.", "tier": null, "supported": null } }, "cost_attribution": "Per-request cost lands in the access log as `agw.ai.usage.cost.total` and in CEL as `llm.cost` / `llm.costRates`, so you can split by virtual key, user, model or any custom CEL label. The built-in LLM > Analytics dashboard aggregates it. For AWS, `auth.aws.assumeRole` session names and tags plus Bedrock `requestMetadata` push attribution into CloudTrail, the Cost & Usage Report and Cost Explorer - invoice-grade rather than gateway-estimated ([costs](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/costs/), [attribution](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/attribution/)).", "cost_export": "No CSV or reporting export endpoint is documented. You get the data by querying your own `request_logs` / `request_log_payloads` tables, scraping Prometheus (`agentgateway_gen_ai_client_token_usage`, `agentgateway_cost_catalog_lookups_total`), or shipping access logs over OTLP ([database logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/database/), [metrics](https://agentgateway.dev/docs/standalone/latest/observability/metrics/overview/)).", "product_kind_note": "Positioned as \"an AI-first, open-source, cloud-native gateway control plane and proxy data plane\" and simultaneously as a general-purpose HTTP/gRPC data plane with load balancing, timeouts, retries, TLS, rate limits and authorization, so that you do not run separate \"regular\" and \"AI\" gateways. Three traffic classes are first-class: LLM inference, MCP tool servers and A2A agent traffic ([introduction](https://agentgateway.dev/docs/standalone/latest/about/introduction/), [FAQs](https://agentgateway.dev/docs/standalone/latest/faqs/)).", "deployment_modes": [ "self_host" ], "deployment_note": "Install paths documented: `curl -sL https://agentgateway.dev/install | bash` (pin with `-s -- --version v1.5.0`), `docker run cr.agentgateway.dev/agentgateway:v1.5.0`, Docker Compose, and `helm upgrade -i agentgateway-standalone oci://cr.agentgateway.dev/charts/agentgateway-standalone --version v1.5.0`. The proxy listens on 4000, the UI and config API on 15000, metrics on 15020 ([binary](https://agentgateway.dev/docs/standalone/latest/setup/install/binary/), [Docker](https://agentgateway.dev/docs/standalone/latest/setup/install/docker/), [Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "partial", "api_batch": "not_documented", "api_surface_note": "Route types are declared per route, so one gateway can expose several client dialects at once: `completions`, `responses`, `messages`, `anthropicTokenCount`, `embeddings`, `models`, `rerank`, `realtime`, native Gemini (`models/{model}:generateContent`, `:streamGenerateContent`, `:countTokens`) and `passthrough`. `passthrough` has two sub-modes worth understanding: `opaque` forwards the body without interpretation and **without guardrails**, while `detect` parses enough for telemetry. A separate config resource API on the admin port manages keys, models, routes and policies at runtime ([LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/), [passthrough](https://agentgateway.dev/docs/standalone/latest/llm/api-types/passthrough/), [config resources](https://agentgateway.dev/docs/standalone/latest/operations/config-resources/)).", "model_count_low": 1002, "model_count_high": 1002, "model_count_source_note": "One project-published figure: \"1002+ Models\" on the Model and Provider Cookbook, alongside \"44+ LLM Gateway Providers\" and \"20 API Endpoints\". No page publishes a total elsewhere, and the docs never claim a model count, so this is a single vendor headline rather than two corroborating sources ([Model and Provider Cookbook](https://agentgateway.dev/models)).", "provider_count_low": 20, "provider_count_high": 44, "provider_count_source_note": "Two project-published figures, at different scopes: the LLM overview documents **20** natively supported providers with a per-provider capability matrix (OpenAI, Anthropic, Bedrock, Azure, Gemini, Vertex AI, Copilot, Cohere, Ollama, Baseten, Cerebras, Deepinfra, Deepseek, Groq, Hugging Face, Mistral, OpenRouter, Together AI, xAI, Fireworks), while the Model and Provider Cookbook headlines **44+** once OpenAI-compatible, enterprise/regional and local providers are counted ([LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/), [Model and Provider Cookbook](https://agentgateway.dev/models)).", "first_party_vs_partner": "All third-party routed: the project hosts no models and owns no inference capacity, proxying instead to 20 natively supported providers plus self-hosted runtimes (Ollama, vLLM, LM Studio) and anything OpenAI-compatible via the custom provider ([LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/), [custom providers](https://agentgateway.dev/docs/standalone/latest/llm/providers/custom/)).", "custom_endpoint_support": "Yes: a documented `custom` provider takes an arbitrary host, port, path and auth for any OpenAI-compatible endpoint, and self-hosted runtimes Ollama, vLLM and LM Studio have their own pages ([custom providers](https://agentgateway.dev/docs/standalone/latest/llm/providers/custom/), [Ollama](https://agentgateway.dev/docs/standalone/latest/llm/providers/ollama/)).", "byok_mode": "byok_only", "merchant_of_record": "n.a. - your model providers invoice you directly and the project invoices nothing. The gateway's own cost figures are explicitly \"best-effort and may not exactly match your provider bill\" ([model costs](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/costs/)).", "key_management_note": "Upstream keys are never stored by a vendor. Options are inline literals, `$ENV_VAR` indirection, a file whose contents become an env var, a Kubernetes secret reference, or full passthrough of the caller's token. Virtual keys sit in the config file or, in `hybrid` storage mode, in your own SQLite/PostgreSQL database, and can be created or revoked through the config resource API. Sharp edge: that API is served on the admin address `127.0.0.1:15000` with **no authentication** - anyone who can reach it can list, create or delete keys ([API keys](https://agentgateway.dev/docs/standalone/latest/llm/api-keys/), [config resources](https://agentgateway.dev/docs/standalone/latest/operations/config-resources/)).", "api_openai_chat_completions_note": "Yes: the `completions` route type serves `POST /v1/chat/completions`, and the documented drop-in is `openai.OpenAI(api_key=\"anything\", base_url=\"http://localhost:4000/v1\")` ([Completions](https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/)).", "api_anthropic_messages_note": "Yes: the `messages` route type serves `/v1/messages`, with a companion `anthropicTokenCount` type for `/v1/messages/count_tokens` ([Messages](https://agentgateway.dev/docs/standalone/latest/llm/api-types/messages/), [token count](https://agentgateway.dev/docs/standalone/latest/llm/api-types/token-count/)).", "api_openai_responses_note": "Yes: the `responses` route type serves `/v1/responses`, with documented conversion caveats worth knowing - `stop_sequences` and `top_k` are silently dropped when converting, and unsupported block types return `400 unsupported conversion` ([Responses](https://agentgateway.dev/docs/standalone/latest/llm/api-types/responses/)).", "api_embeddings_note": "Yes: an `embeddings` route type serves `/v1/embeddings`, and a separate `rerank` type serves `/v2/rerank` and `/v1/rerank` ([Embeddings](https://agentgateway.dev/docs/standalone/latest/llm/api-types/embeddings/), [Rerank](https://agentgateway.dev/docs/standalone/latest/llm/api-types/rerank/)).", "api_images_note": "n.a. as a gateway route type: no `/v1/images` or image-generation route appears among the documented API types ([LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/), [Completions](https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/), [passthrough](https://agentgateway.dev/docs/standalone/latest/llm/api-types/passthrough/)). Image models such as `gpt-image-1` and `dall-e-3` do appear in the project's provider cookbook model lists, and `passthrough` would forward such a call opaquely, but no image endpoint is translated or guarded ([Model and Provider Cookbook](https://agentgateway.dev/models)).", "api_audio_note": "partial: a `realtime` route type proxies OpenAI's `/v1/realtime` WebSocket, but every documented example uses `modalities: [\"text\"]` and no audio modality, transcription or TTS path is shown. There is no `/v1/audio/*` route type. The Realtime page also lists real limits: prompt guards, prompt enrichment and body-based rate limiting do not apply to WebSocket traffic ([Realtime](https://agentgateway.dev/docs/standalone/latest/llm/api-types/realtime/)).", "api_batch_note": "n.a. (no batch or async bulk endpoint on any page fetched: [LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/), [Completions](https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/), [passthrough](https://agentgateway.dev/docs/standalone/latest/llm/api-types/passthrough/)).", "deployment_modes_note": "self_host only, in two shapes. **Standalone**: one binary, Docker image or the single `agentgateway-standalone` Helm chart, driven by a config file, with an editable UI and ClickOps. **Kubernetes**: adds a control plane, two Helm charts, Gateway API custom resources, xDS dynamic config, a read-only UI and a GitOps workflow. No SaaS and no hybrid vendor plane exist ([introduction](https://agentgateway.dev/docs/standalone/latest/about/introduction/), [Kubernetes control plane](https://agentgateway.dev/docs/standalone/latest/setup/install/kubernetes/)).", "byok_mode_note": "byok_only: there are no platform credits and no project-issued upstream keys. Credentials are supplied inline, as `$ENV_VAR`, from a file-backed env var, as a Kubernetes secret, or passed straight through from the caller's own token; virtual keys are an internal authorisation layer in front of those upstream credentials, not a billing relationship ([API keys](https://agentgateway.dev/docs/standalone/latest/llm/api-keys/), [virtual keys](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/virtual-keys/)).", "access_meta": { "qualified": [ "api_audio", "api_images" ], "model_count_kind": "vendor_total", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "config_file", "rel_timeout_note": "`config_file`, at four levels: route `requestTimeout` (whole client request including retries) and `backendRequestTimeout` (each individual try), backend `http.requestTimeout`, and `tcp.connectTimeout` for connection establishment. No per-request header or query override is documented ([Timeouts](https://agentgateway.dev/docs/standalone/latest/configuration/resiliency/timeouts/)).", "rel_retry_config": "config_file", "rel_retry_note": "`retry.attempts`, `retry.backoff` (example `500ms`) and `retry.codes` (example `[429, 500, 503]`). A retry prefers a different backend from the one that just failed. Caveat: retries are disabled once a request body exceeds the buffering threshold, because the body can no longer be replayed ([Retries](https://agentgateway.dev/docs/standalone/latest/configuration/resiliency/retries/)).", "rel_retry_default_note": "No retries by default - `retry` is opt-in, and the codes shown (`429, 500, 503`) are an example rather than a documented default set ([Retries](https://agentgateway.dev/docs/standalone/latest/configuration/resiliency/retries/)).", "rel_fallback_config": "config_file", "rel_fallback_note": "ORDERED and WEIGHTED both available, through `llm.virtualModels[]` (v1.3+): `routing.weighted` distributes by weight, `routing.failover` uses priority groups, `routing.conditional` picks a target from a CEL `when:` expression. **Read the warning**: `routing.failover` on its own does not fail over - you must also configure `health.eviction`, and even then the request that trips the failure still fails unless retries are configured ([Virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "config_file", "rel_health_check_note": "Passive health checking on virtual models: `unhealthyExpression` decides what counts as a failure, and `eviction` takes `duration`, `consecutiveFailures`, `healthThreshold` and `restoreHealth`. This is outlier ejection rather than active probing ([Virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "n.a. as a feature: you can point virtual-model targets at regional provider endpoints and fail over between them, but no cross-region or geo-aware routing construct is documented ([Virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/), [multiple LLMs](https://agentgateway.dev/docs/standalone/latest/llm/providers/multiple-llms/)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights supported: `routing.weighted` with per-target weights inside a virtual model, and ordinary backend load balancing from the general-purpose data plane underneath ([Virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/)).", "architecture_class": "compiled_binary", "architecture_note": "`compiled_binary`. Rust data plane (7.35 MB of Rust in the language breakdown) with a Go Kubernetes controller (2.65 MB) and a TypeScript UI (1.07 MB); ships as a single static binary and a `cr.agentgateway.dev/agentgateway` container image ([agentgateway/agentgateway](https://api.github.com/repos/agentgateway/agentgateway), [Docker](https://agentgateway.dev/docs/standalone/latest/setup/install/docker/)).", "self_host_data_plane": "yes", "self_host_artifact_note": "`curl -sL https://agentgateway.dev/install | bash` for the binary; `cr.agentgateway.dev/agentgateway:v1.5.0` for Docker and Compose; OCI Helm charts `oci://cr.agentgateway.dev/charts/agentgateway-standalone` for standalone and `agentgateway-crds` + `agentgateway` for Kubernetes mode; nightly builds via GitHub Actions artifacts or chart version `0.0.0-latest-dev` ([binary](https://agentgateway.dev/docs/standalone/latest/setup/install/binary/), [Docker](https://agentgateway.dev/docs/standalone/latest/setup/install/docker/), [Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/)).", "latency_claims": [ { "url": "https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/", "unit": "ms", "scope": "gateway_overhead", "value": "0.831", "metric": "p50", "publisher": "vendor", "conditions": "Fortio, 32 connections, 1 KB payload, 3 s max-throughput run against a mock LLM backend; p90 1.533 ms, p99 1.970 ms. Project-published, authored by Solo.io's Lin Sun." }, { "url": "https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm-part-2/", "unit": "ms", "scope": "gateway_overhead", "value": "0.227", "metric": "p50", "publisher": "vendor", "conditions": "Fortio held at a fixed 3,000 QPS for 30 s against a mock LLM backend; p99 0.436 ms, 13 MiB resident, 13.4% CPU. Project-published." }, { "url": "https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/", "unit": "ms", "scope": "gateway_overhead", "value": "0.863", "metric": "p50", "publisher": "vendor", "conditions": "Maximum-throughput run against a mock Anthropic backend at 35,502 QPS; p99 1.972 ms. Same harness re-run against LiteLLM's Rust mode. Project-published." }, { "url": "https://agentgateway.dev/blog/2026-08-20-benchmarking-agentgateway-epp-proxy-overhead/", "unit": "s", "scope": "end_to_end", "value": "0.2", "metric": "TTFT p90", "publisher": "vendor", "conditions": "Google Summer of Code study of inference-gateway EPP overhead at 60 QPS, comparing against a plain Kubernetes Service where TTFT p90 was 135.6 s; inter-token latency ~50 ms vs 30.3 ms direct. Real model backend, not a mock." } ], "throughput_claims": [ { "url": "https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/", "unit": "QPS", "scope": "gateway_overhead", "value": "36,933", "metric": "max throughput", "publisher": "vendor", "conditions": "Fortio, 32 connections, 1 KB payload, 3 s, mock LLM backend, 22 MB average memory. LiteLLM measured at 3,198 QPS / 11.8 GB on the same harness. Project-published." }, { "url": "https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/", "unit": "QPS", "scope": "gateway_overhead", "value": "35,502", "metric": "max throughput", "publisher": "vendor", "conditions": "Same harness re-run against LiteLLM's Rust mode, which reached 984 QPS at p99 71.451 ms. Mock backend. Project-published." }, { "url": "https://www.solo.io/products/agentgateway", "unit": "multiple", "scope": "marketing", "value": "35x throughput, 300x memory, 122x latency", "metric": "marketing summary", "publisher": "vendor", "conditions": "Commercial distribution landing page invites you to reproduce it with `agentgateway benchmark --compare`; no conditions stated on the page itself." } ], "streaming_support": "yes", "streaming_note": "SSE streaming is supported on the LLM route types, but the guardrail interaction is the thing to know: response guards default to `streaming: Disabled`, meaning they do not run at all on a streamed response, and the `mask` action never applies to streams - content passes through with no error and no event. Use `reject` if you need enforcement on streaming traffic ([Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/), [Completions](https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/)).", "guard_pii": "sync_block", "guard_pii_note": "Built-in regex guard with named PII patterns - `creditCard`, `ssn`, `email`, `phoneNumber` - plus arbitrary custom patterns, running inline on request and response paths. Actions are `mask`, `reject` or `audit`, and the default is `mask` - so a freshly configured PII guard modifies traffic synchronously rather than merely logging it, the opposite default from gateways that ship guardrails asynchronous. Note the streaming exception: on streamed responses `mask` silently does nothing ([Regex filters](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/regex/)).", "guard_injection": "sync_block", "guard_injection_note": "Prompt-injection and jailbreak detection comes from external services rather than a built-in classifier: Azure Content Safety exposes `detectJailbreak`, and Google Model Armor and Bedrock Guardrails cover similar ground. Actions are `reject` or `audit` ([Azure Content Safety](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/azure-content-safety/), [Google Model Armor](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/google-model-armor/)).", "guard_content": "sync_block", "guard_content_note": "`openAIModeration` calls the OpenAI moderation endpoint inline; Bedrock Guardrails, Google Model Armor and Azure Content Safety provide the same function through their own services. All default to `reject` ([OpenAI moderation](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/moderation/), [Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/)).", "guard_custom": "sync_block", "guard_custom_note": "A `webhook` guard posts to your own classifier and honours `reject` or `audit`; arbitrary regex patterns cover deterministic policies. Guards run in sequence and can be scoped with `scope: [systemPrompt, messages, toolInput, toolOutput]`, though setting `scope` **replaces** the default (`systemPrompt` + `messages`) rather than adding to it, and only the regex and Bedrock Guardrails types accept a non-default scope ([Webhooks](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/webhooks/), [Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/)).", "guard_model_policy": "yes", "guard_model_policy_note": "Per-virtual-key `allowedModels` with wildcard support (`[\"gpt-5*\"]`) gates which models a key may reach, and CEL-based authorization plus `modelAliases` and route-level `llm.models[]` matching give further control ([virtual keys](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/virtual-keys/), [LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/)).", "guard_execution_location": "both", "guard_third_party": [ "OpenAI Moderation", "AWS Bedrock Guardrails", "Google Model Armor", "Azure Content Safety" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "The docs describe verdict semantics and give latency guidance (regex under 1 ms, external moderation 50-200 ms) but never state what happens when the external guard service itself errors or times out - fail-open versus fail-closed is unspecified ([Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/), [Webhooks](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/webhooks/), [multi-layer](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/multi-layer/)).", "log_content": "configurable", "log_content_note": "Metadata by default, full bodies on request. The UI has an explicit \"Include prompts and completions in logs\" toggle, and the database equivalent is `frontendPolicies.accessLog.database.llm: full`, which writes bodies into a `request_log_payloads` table ([Access logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/), [database logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/database/)).", "log_opt_out": "yes", "log_opt_out_note": "Body logging is off unless you turn it on, and a CEL `filter` plus `remove` can suppress entries or drop individual fields entirely ([Access logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/)).", "tracing_support": "otel", "tracing_note": "OpenTelemetry tracing with documented collector configs for Jaeger, Datadog, Honeycomb, Grafana Cloud and a plain OTel collector, plus a `gen_ai.*` attribute reference. Access logs can also be exported over OTLP independently of traces ([Traces](https://agentgateway.dev/docs/standalone/latest/observability/traces/setup/), [attribute reference](https://agentgateway.dev/docs/standalone/latest/observability/traces/attribute-reference/)).", "export_destinations": [ "OpenTelemetry (OTLP)", "Prometheus", "Grafana", "Jaeger", "Datadog", "Honeycomb", "Grafana Cloud", "Langfuse", "LangSmith", "Arize Phoenix" ], "export_destinations_note": "Any OTLP-compatible backend, with per-destination guides for Jaeger, Datadog, Honeycomb and Grafana Cloud on the tracing side, Prometheus and Grafana for metrics, and dedicated LLM-observability pages for Langfuse, LangSmith and Arize Phoenix ([Traces](https://agentgateway.dev/docs/standalone/latest/observability/traces/setup/), [Langfuse](https://agentgateway.dev/docs/standalone/latest/integrations/llm-observability/langfuse/)).", "feedback_capture": "no", "feedback_capture_note": "n.a. (no feedback, rating or annotation endpoint on any page fetched: [LLM observability](https://agentgateway.dev/docs/standalone/latest/llm/observability/), [Access logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/view/), [LLM overview](https://agentgateway.dev/docs/standalone/latest/llm/about/)). Downstream tools like Langfuse or Phoenix would own that.", "eval_hooks": "no", "eval_hooks_note": "n.a. as a gateway feature: no eval runner, dataset, scoring or experiment concept appears in the docs. The built-in LLM playground lets you fire ad-hoc requests, and the Langfuse/LangSmith/Phoenix integrations exist precisely because evaluation lives outside the gateway ([playground](https://agentgateway.dev/docs/standalone/latest/llm/playground/), [Langfuse](https://agentgateway.dev/docs/standalone/latest/integrations/llm-observability/langfuse/)).", "observability_requires_saas": "No: everything is local. Prometheus metrics on `:15020`, access logs to stdout or your own SQLite/PostgreSQL, OTLP export to whatever you run, and a built-in cost/analytics dashboard at `/ui/llm/analytics` that needs only `config.database` and a model catalog - no external Prometheus or Grafana required ([metrics](https://agentgateway.dev/docs/standalone/latest/observability/metrics/overview/), [cost dashboard](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/dashboard/)).", "log_retention_note": "Unlimited and entirely yours - and that is a burden, not a perk. Nothing prunes the `request_logs` or `request_log_payloads` tables and no retention setting is documented, so you own the lifecycle ([database logs](https://agentgateway.dev/docs/standalone/latest/observability/access-logs/database/), [Database](https://agentgateway.dev/docs/standalone/latest/setup/database/)).", "production_meta": { "sources": {}, "qualified": [ "api_audio", "api_images", "guard_pii", "guard_injection", "guard_content", "guard_custom", "latency_overhead_ms", "throughput_rps" ], "latency_prose": "Four project-published measurements exist, all authored by Solo.io's Lin Sun with open scripts at [linsun/litellm-agw-perf](https://github.com/linsun/litellm-agw-perf). Three use Fortio against a **mock** LLM backend, so they measure proxy overhead only: p50 0.831 ms / p99 1.970 ms at max throughput ([part 1](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/)), p50 0.227 ms / p99 0.436 ms when held at 3,000 QPS ([part 2](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm-part-2/)), and p50 0.863 ms / p99 1.972 ms in the Rust-mode rematch ([part 3](https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/)). A fourth, separate study of inference-gateway EPP overhead at 60 QPS with a real model reports TTFT p90 falling from 135.6 s to 0.2 s against a plain Kubernetes Service, with inter-token latency ~50 ms versus 30.3 ms direct ([EPP study](https://agentgateway.dev/blog/2026-08-20-benchmarking-agentgateway-epp-proxy-overhead/)).", "multi_surface": { "rel_timeout": [ "config_file" ], "rel_fallback": [ "config_file" ] }, "throughput_prose": "36,933 QPS at 32 connections with 22 MB average memory in the June run, and 35,502 QPS in the August rematch, both against mock backends ([part 1](https://agentgateway.dev/blog/2026-06-26-benchmarking-agentgateway-vs-litellm/), [part 3](https://agentgateway.dev/blog/2026-08-13-benchmarking-agentgateway-vs-litellm-rust-mode/)). The commercial distribution page compresses this to \"300x memory, 35x throughput, 122x latency\" with no conditions attached ([Solo.io](https://www.solo.io/products/agentgateway)).", "benchmark_independence": "All figures are project-published and comparative against a named competitor, which is the least independent shape a benchmark can take. Methodology is unusually transparent - tool, connection count, payload size, duration and open scripts are all stated - but the harness is the project's own. One outside evaluation examined the numbers without re-running them and flagged a specific discrepancy: the ~12 GB LiteLLM memory figure conflicts with LiteLLM's own reported 359 MB peak on a different harness, and it declined to treat the result as a settled speed winner ([fmind.dev](https://www.fmind.dev/articles/agentgateway-vs-litellm/)). No genuinely independent benchmark was found." }, "guard_pii_default": "sync_block", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "not_documented", "rel_notes": "The reliability model is unusually explicit about its own failure modes, which is a point in its favour: the docs state that failover without eviction does nothing, that the triggering request still fails without retries, and that retries stop working above the body-buffering threshold. Everything is config-file driven with no per-request escape hatch, so reliability behaviour is reviewable in Git but cannot be tuned by a caller ([Retries](https://agentgateway.dev/docs/standalone/latest/configuration/resiliency/retries/), [Virtual models](https://agentgateway.dev/docs/standalone/latest/llm/virtual-models/)).", "guard_notes": "**The defaults here enforce rather than observe**, which is the opposite of several hosted gateways: regex guards default to `mask` and every external guard defaults to `reject`. Shared guards go in `llm.policies.guardrails` and merge with per-model `llm.models[].guardrails`, and MCP traffic has its own separate guardrail surface. Three real gaps to plan around: response guards do not run on streamed responses unless you enable streaming mode, `passthrough` routes with `opaque` bodies get no guardrails at all, and the WebSocket realtime path is exempt from prompt guards entirely ([Prompt guards](https://agentgateway.dev/docs/standalone/latest/llm/prompt-guards/overview/), [passthrough](https://agentgateway.dev/docs/standalone/latest/llm/api-types/passthrough/), [Realtime](https://agentgateway.dev/docs/standalone/latest/llm/api-types/realtime/), [MCP guardrails](https://agentgateway.dev/docs/standalone/latest/mcp/guardrails/setup/)).", "fit_agent_features_note": "Function calling and tool use are documented for LLM routes, and the agent surface goes further than most gateways: A2A agent-to-agent traffic is a first-class protocol alongside MCP, with MCP guardrails, MCP authorization, tool-level access control and MCP observability as separate documented features. Context compression and semantic routing exist as project blog subjects rather than core docs pages ([Agent connectivity](https://agentgateway.dev/docs/standalone/latest/agent/about/), [MCP guardrails](https://agentgateway.dev/docs/standalone/latest/mcp/guardrails/setup/), [MCP authorization](https://agentgateway.dev/docs/standalone/latest/configuration/security/mcp-authz/)).", "fit_byok_required": "yes", "fit_byok_required_note": "Yes, unavoidably: there is no credit pool or vendor account, so an upstream provider key (or a passthrough caller token, or a local Ollama/vLLM endpoint) is required before the gateway can serve a single request ([API keys](https://agentgateway.dev/docs/standalone/latest/llm/api-keys/), [quickstart](https://agentgateway.dev/docs/standalone/latest/quickstart/llm/)).", "fit_client_sdk_langs": [ "Python", "JavaScript" ], "fit_client_sdks_note": "No first-party SDK exists. Documented clients are the standard OpenAI SDKs (`openai` for Python, `openai` for JavaScript) pointed at the gateway base URL, plus `curl`; the operator-side tooling is the `agctl` CLI ([OpenAI SDK](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/), [curl](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/curl/)).", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "native_iam for the three big clouds: AWS backend auth supports `auth.aws.assumeRole` with CEL/JWT-derived session names and tags, and Azure and GCP have their own backend-auth provider pages, so upstream calls can use cloud identity rather than static keys ([AWS integration](https://agentgateway.dev/docs/standalone/latest/integrations/cloud-providers/aws/), [invoice-grade attribution](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/attribution/)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. - it is a native Rust binary, not a Workers-compatible runtime, and no Cloudflare Workers deployment appears in the docs ([binary](https://agentgateway.dev/docs/standalone/latest/setup/install/binary/), [Docker](https://agentgateway.dev/docs/standalone/latest/setup/install/docker/), [Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/)).", "fit_credit_card": "not_required", "fit_credit_card_note": "No card, no account, no email. `curl -sL https://agentgateway.dev/install | bash` and you are running ([binary](https://agentgateway.dev/docs/standalone/latest/setup/install/binary/)).", "fit_ecosystem_notes": "Deep infrastructure ecosystem, thin application ecosystem. On the infra side: Kubernetes Gateway API conformance, xDS dynamic config, cert-manager and external-dns integrations, eight IdP guides (Auth0, Authentik, Descope, Entra ID, Keycloak, oauth2-proxy, Okta, Tailscale) and cloud backend-auth for AWS, Azure and GCP. On the client side, documented consumers are coding agents and chat UIs - Claude Code, Claude Desktop, Codex, Continue, Cursor, Devin, GitHub Copilot, Antigravity, VS Code, opencode, LibreChat, Open WebUI, Chatbot UI, Goose. What is missing is the Python-framework layer: no LangChain or LlamaIndex integration page, no Vercel AI SDK provider, no Terraform provider and no SCIM. Migration is covered from one direction only, `agentgateway import --from litellm` ([integrations index](https://agentgateway.dev/llms.txt), [import](https://agentgateway.dev/docs/standalone/latest/configuration/import/)).", "fit_eval_prereqs": "A machine and a provider API key. No account, no card, no cluster, no database: the binary reads a YAML file and serves on port 4000, with the UI and playground on 15000 ([quickstart](https://agentgateway.dev/docs/standalone/latest/quickstart/llm/)). A database is needed only later, for USD budgets, the cost dashboard or database-backed logs.", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, a complete runnable pair: a YAML `llm.models` block with `provider: openAI` and `apiKey: \"$OPENAI_API_KEY\"`, then `openai.OpenAI(api_key=\"anything\", base_url=\"http://localhost:4000/v1\")` followed by `chat.completions.create(...)` ([quickstart](https://agentgateway.dev/docs/standalone/latest/quickstart/llm/), [OpenAI SDK](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It is the gateway, and explicitly a general-purpose one: the FAQ positions it as an HTTP/gRPC data plane with load balancing, timeouts, retries, TLS, rate limits and authorization that can front ordinary APIs and microservices, so you do not run separate \"regular\" and \"AI\" gateways ([FAQs](https://agentgateway.dev/docs/standalone/latest/faqs/)).", "fit_helm_chart": "agentgateway-standalone (standalone mode); agentgateway-crds + agentgateway (Kubernetes mode), all OCI charts under cr.agentgateway.dev/charts", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Official OCI Helm charts: `helm upgrade -i agentgateway-standalone oci://cr.agentgateway.dev/charts/agentgateway-standalone --version v1.5.0` for standalone, and a separate CRDs + control-plane pair for Kubernetes mode with Gateway API custom resources, xDS and a read-only GitOps UI ([Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/), [Kubernetes control plane](https://agentgateway.dev/docs/standalone/latest/setup/install/kubernetes/)).", "fit_mcp": "mcp_gateway", "fit_mcp_note": "A full MCP gateway, arguably the project's centre of gravity: static and dynamic MCP servers, stdio/SSE/streamable-HTTP transports, MCP authentication and authorization, tool-level access control, MCP-specific guardrails, MCP rate limiting, MCP observability, and OpenAPI-to-MCP bridging. MCP spec 2026-07-28 support landed in v1.4 ([MCP guardrails](https://agentgateway.dev/docs/standalone/latest/mcp/guardrails/setup/), [MCP observability](https://agentgateway.dev/docs/standalone/latest/mcp/mcp-observability/), [v1.4 announcement](https://agentgateway.dev/blog/2026-08-03-mcp-spec-2026-07-28/)).", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_terraform", "fit_vercel_ai_sdk", "fit_model_gate", "air_gapped", "retention_days", "sla_uptime_pct", "founded_year", "hq_country" ] }, "fit_model_gate": "none", "fit_model_gate_note": "No gate on the gateway's side: any model your upstream credential can reach is reachable through a route the moment you add it. Access restrictions are ones you impose yourself, through virtual-key `allowedModels` or CEL authorization ([quickstart](https://agentgateway.dev/docs/standalone/latest/quickstart/llm/), [virtual keys](https://agentgateway.dev/docs/standalone/latest/llm/cost-controls/virtual-keys/)).", "fit_onboarding_notes": "The five quickstart steps are: set provider credentials as environment variables, start the gateway, enable the LLM feature, add a model, send a request. The built-in UI adds a ClickOps path with an LLM playground and a CEL playground for testing expressions, and `agctl` covers config inspection, request tracing, log level changes and CPU/heap profiling. `agentgateway import --from litellm --file litellm.yaml` converts an existing LiteLLM proxy config and prints per-field compatibility findings, which is the cheapest possible migration test ([quickstart](https://agentgateway.dev/docs/standalone/latest/quickstart/llm/), [playground](https://agentgateway.dev/docs/standalone/latest/llm/playground/), [import](https://agentgateway.dev/docs/standalone/latest/configuration/import/)).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes, a literal base-URL swap with a throwaway key: `openai.OpenAI(api_key=\"anything\", base_url=\"http://localhost:4000/v1\")`. The gateway authenticates callers with its own virtual keys if you configure them, otherwise the upstream key never leaves the config ([OpenAI SDK](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/), [Completions](https://agentgateway.dev/docs/standalone/latest/llm/api-types/completions/)).", "fit_prod_prereqs": "For Kubernetes mode: a cluster, `kubectl`, `helm`, the CRDs chart and the control-plane chart, plus Gateway API custom resources. For anything with USD budgets, the cost dashboard or persisted request logs: `config.database` pointing at SQLite or PostgreSQL, and `config.storage.mode: hybrid` if you want the config resource API to list what it writes. One hard operational requirement: keep the unauthenticated admin address on localhost or otherwise network-isolated ([Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/), [Database](https://agentgateway.dev/docs/standalone/latest/setup/database/), [Storage modes](https://agentgateway.dev/docs/standalone/latest/setup/storage/), [config resources](https://agentgateway.dev/docs/standalone/latest/operations/config-resources/)).", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "infra_rollout", "fit_python_frameworks": null, "fit_python_frameworks_note": "n.a. - no LangChain, LlamaIndex or other Python-framework integration page exists in the documentation index. The FAQ asserts compatibility with \"any agentic framework supporting MCP and A2A protocols, including LangGraph, AutoGen, kagent, Claude Desktop, and OpenAI SDK\", but that is a protocol-level claim with no framework guide behind it ([FAQs](https://agentgateway.dev/docs/standalone/latest/faqs/), [docs index](https://agentgateway.dev/llms.txt)).", "fit_quickstart_steps": 5, "fit_quickstart_steps_caveat": "Counts the LLM quickstart only; separate MCP and non-agentic-HTTP quickstarts exist, and the five steps assume you already hold a provider key.", "fit_quickstart_url": "https://agentgateway.dev/docs/standalone/latest/quickstart/llm/", "fit_self_host_docs": "command_published", "fit_self_host_install": "`curl -sL https://agentgateway.dev/install | bash`; `docker run -d --name agentgateway -p 4000:4000 -p 15000:15000 cr.agentgateway.dev/agentgateway:v1.5.0`; `helm upgrade -i agentgateway-standalone oci://cr.agentgateway.dev/charts/agentgateway-standalone --namespace agentgateway-system --create-namespace --version v1.5.0`", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (no Terraform provider, module or `terraform` reference on any page fetched, including the docs index: [docs index](https://agentgateway.dev/llms.txt), [Helm](https://agentgateway.dev/docs/standalone/latest/setup/install/helm/), [Kubernetes control plane](https://agentgateway.dev/docs/standalone/latest/setup/install/kubernetes/)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": "Project claims, verbatim: \"Get started with agentgateway in less than 5 minutes - local-first\" ([homepage](https://agentgateway.dev)). The commercial distribution page claims \"One binary, zero dependencies. Running in under 2 minutes.\" and \"In 15 minutes, you'll see exactly why agentgateway exists\" ([Solo.io](https://www.solo.io/products/agentgateway)).", "fit_vercel_ai_sdk": "openai_compat_only", "fit_vercel_ai_sdk_note": "No Vercel AI SDK provider package or integration page exists; the SDK's OpenAI-compatible provider would work against the gateway's `/v1` base URL, but that path is not documented by the project ([docs index](https://agentgateway.dev/llms.txt), [OpenAI SDK](https://agentgateway.dev/docs/standalone/latest/integrations/llm-clients/openai-sdk/)).", "fit_vercel_ai_sdk_package": null }, { "slug": "apache-apisix-ai-gateway", "name": "Apache APISIX AI Gateway", "company": "Apache Software Foundation", "tagline": "AI proxy plugins for the Apache APISIX API gateway, adding LLM routing and token limits to an OpenResty data plane.", "category": "open_source", "deployment": "self_host_only", "license": "Apache-2.0", "founded_year": null, "hq_country": "United States", "url_home": "https://apisix.apache.org/", "url_docs": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "url_pricing": null, "url_status": null, "url_github": "https://github.com/apache/apisix", "url_changelog": null, "url_models_api": null, "github_stars": 17155, "model_count": null, "model_count_note": "No model count is published; models are whatever the configured upstream provider exposes.", "provider_count": 10, "token_markup_pct": 0, "credit_fee_pct": 0, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Provider credentials are configured in the plugin; the project charges nothing.", "seat_fee_usd_month": 0, "free_tier": "Entire project is free under Apache-2.0, including the ai-proxy and ai-proxy-multi AI gateway plugins.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "No license cost at all; you run APISIX and its etcd control store on your own infrastructure. The docs are co-branded with API7, a commercial vendor offering a supported distribution, whose pricing is not published on these pages.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Embeddings", "Anthropic Messages" ], "sdk_note": "No SDK: you declare an APISIX route with the ai-proxy or ai-proxy-multi plugin, which normalizes requests and can convert to the Anthropic /v1/messages format.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 0.2, "latency_source_note": "Project-published (Apache APISIX, not AI-specific): under 0.2 ms average added latency, ~18,000 QPS on a single core and 140,000 QPS on an 8-core AWS instance at 0.2 ms. These are general gateway figures rather than AI-plugin measurements; a third-party 2026 comparison estimates roughly 1-2 ms overhead on the AI path.", "throughput_rps": 18000, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "Traffic never leaves your infrastructure, so there is no vendor retention to negotiate away.", "air_gapped": null, "best_for": "Teams already running APISIX that want basic multi-provider LLM proxying, retries and token-based rate limiting without adding another gateway.", "strengths": [ "Fastest published data-plane numbers here: ~18k QPS per core at under 0.2 ms added latency", "Fully Apache-2.0 with no paid tier, gated features or vendor account required", "Mature routing primitives: round-robin/consistent-hash/semantic load balancing, retries with time windows, active health checks and fallback strategies", "Token-level logging variables and token-based rate limiting for LLM traffic" ], "weaknesses": [ "Not AI-first: no response or semantic caching, budgets, virtual keys or prompt management in the AI plugins", "Only about ten provider integrations documented, versus 20-140 for LLM-native gateways", "Plugin configuration is Lua/OpenResty-shaped and the published performance numbers are for generic proxying, not LLM paths", "No compliance certifications, support SLA or hosted control plane from the project itself" ], "avoid_if": "You want LLM-native features like semantic caching, spend budgets, virtual keys or a managed control plane out of the box.", "reviews": [ { "url": "https://medium.com/syntonize/enterprise-ready-llm-gateway-with-apache-apisix-part-2-bd796db06340", "why": "Consultancy engineer builds an enterprise LLM gateway with APISIX's ai-proxy, key-auth, consumer-restriction and OIDC plugins wired to Entra ID, with config shown.", "date": "2025-05-21", "type": "written", "title": "Enterprise-ready LLM Gateway with Apache APISIX (Part 2)", "publisher": "Medium - Syntonize" }, { "url": "https://www.youtube.com/watch?v=hKB9JFrtttA", "why": "Third-party case study of an enterprise agent platform fronted by the APISIX AI gateway, routing between private-datacenter and public models plus MCP/A2A traffic.", "date": "2026-02-01", "type": "video", "title": "Building an Enterprise AI Agent Platform: APISIX AI Gateway in Practice", "publisher": "YouTube - Let's Talk Tech" } ], "sources": { "cap_mcp": "https://github.com/apache/apisix", "company": "https://github.com/apache/apisix", "license": "https://github.com/apache/apisix", "sdk_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "free_tier": "https://github.com/apache/apisix", "hq_country": "https://en.wikipedia.org/wiki/The_Apache_Software_Foundation", "weaknesses": "https://dev.to/pranay_batta/best-open-source-ai-gateway-in-2026-2flb", "api7_pricing": "https://api7.ai/pricing", "api_surfaces": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "cap_failover": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/", "github_stars": "https://github.com/apache/apisix", "product_fate": "https://github.com/apache/apisix/releases", "regions_note": "https://apisix.apache.org/ai-gateway/", "apisix_github": "https://github.com/apache/apisix", "vendor_status": "https://apisix.apache.org/", "byok_supported": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "cap_embeddings": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "cap_guardrails": "https://github.com/apache/apisix", "certifications": "https://apisix.apache.org/ai-gateway/", "config_as_code": "https://apisix.apache.org/docs/apisix/deployment-modes/", "credit_fee_pct": "https://github.com/apache/apisix", "provider_count": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "throughput_rps": "https://github.com/apache/apisix", "apisix_ai_proxy": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "cap_rate_limits": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "logs_by_default": "https://apisix.apache.org/ai-gateway/", "model_count_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "token_markup_pct": "https://github.com/apache/apisix", "apisix_ai_gateway": "https://apisix.apache.org/ai-gateway/", "cap_observability": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "last_release_date": "https://github.com/apache/apisix/releases/tag/3.18.0", "openai_compatible": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "cap_load_balancing": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/", "seat_fee_usd_month": "https://github.com/apache/apisix", "guardrails_location": "https://apisix.apache.org/docs/apisix/plugins/ai-prompt-guard/", "latency_overhead_ms": "https://github.com/apache/apisix", "latency_source_note": "https://apisix.apache.org/", "requires_vendor_sdk": "https://apisix.apache.org/docs/apisix/3.17/plugins/ai-proxy/", "self_host_cost_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", "data_export_available": "https://apisix.apache.org/docs/apisix/plugins/http-logger/", "prompt_transits_vendor": "https://apisix.apache.org/ai-gateway/", "apisix_ai_rate_limiting": "https://apisix.apache.org/docs/apisix/plugins/ai-rate-limiting/", "cap_conditional_routing": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/" }, "verified_at": { "cap_mcp": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "free_tier": "2026-08-29", "weaknesses": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "github_stars": "2026-09-23", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "throughput_rps": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_note": "2026-08-29", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-23T22:06:27.652468+00:00", "vendor_status": "foundation", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "An Apache Software Foundation top-level project on a regular release train, 3.16 through 3.18 between April and August 2026. There is no vendor behind it, which cuts both ways: nobody can be acquired or change the licence, and nobody owes you a support contract or an SLA.", "last_release_date": "2026-08-20", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "no", "prompt_path_note": "Apache-licensed software you deploy yourself. There is no foundation-operated hosted gateway, so no third party receives your prompts.", "logs_by_default": "not_applicable", "logs_note": "Token usage goes to your own access logs and observability stack. Whatever it keeps is your decision.", "retention_days": null, "retention_note": "Determined entirely by the log store you point it at.", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "The project ships software rather than a data-processing service; no vendor receives prompts.", "subprocessors_url": null, "regions_note": "Anywhere you run it. There are no vendor-managed regions.", "certifications": { "itar": { "status": "not_applicable" }, "soc2": { "status": "not_applicable" }, "fedramp": { "status": "not_applicable" }, "gdpr_dpa": { "status": "not_applicable" }, "iso27001": { "status": "not_applicable" }, "hipaa_baa": { "status": "not_applicable" } }, "guardrails_location": "self_hosted", "guardrails_note": "Prompt guards, content moderation and auditing run in your own gateway before traffic reaches a model, checking content against allow and deny patterns you define.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "oss_no_vendor_price", "pricing_model_note": "Apache 2.0 OSS project — no ASF pricing page, tier list or paid SKU exists. Commercial support is sold by a separate vendor, API7.ai: API7 Cloud Standard is $2 per 1M API calls (floored to $0.01) + $250/gateway group/month + $10/service/month; API7 Enterprise is annual licensing by API gateway CPU core with a custom quote.", "min_commitment": "None for OSS. API7 Enterprise is annual per-CPU-core licensing.", "overage_note": "None for OSS. API7 Cloud is pay-as-you-go 'with no usage limitation', so cost grows with volume rather than being an overage penalty.", "cache_support": "none", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "The AI Gateway overview and ai-rate-limiting plugin docs do not document a response cache or provider-cache passthrough — usage fields like cached_tokens are surfaced but no gateway-owned cache is described. APISIX never prices tokens, so any caching saving is 100% the customer's.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not supported / not stated.", "tier": null, "supported": false }, "virtual_keys": { "note": "Supported in effect via Consumer and Consumer Group objects, though not branded as virtual keys.", "tier": null, "supported": true }, "rate_limiting": { "note": "Free, pre-request.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not supported / not stated.", "tier": null, "supported": false }, "model_allowlists": { "note": "Not stated (routing/fallback across providers is; model allowlisting is not).", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Token-quota caps rather than dollar budgets. Pre-request. ai-rate-limiting plugin controls tokens consumed within a time frame with limit_strategy of total_tokens/prompt_tokens/completion_tokens or an expression. Default local policy keeps counters per node — effective quota scales with node count.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Scoped by Route, Service, Consumer, Consumer Group or custom attributes.", "tier": null, "supported": true } }, "cost_attribution": "Token consumption trackable by Route, Service, Consumer, Consumer Group or custom attributes. Dollar cost attribution not documented.", "cost_export": "None stated. Observability comes from the plugin ecosystem.", "product_kind_note": "Apache calls it \"APISIX AI Gateway\", \"a complete AI gateway product\" built on \"a modern API gateway\", with `ai-proxy` / `ai-proxy-multi` plugins that \"simplify access to large language models (LLMs) and embedding models\" ([Introducing APISIX AI Gateway](https://apisix.apache.org/blog/2025/04/08/introducing-apisix-ai-gateway/), 2025-04-08). Its own AI page calls it \"an open-source LLM gateway and proxy\" ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)). It is a self-hosted routing proxy, not a model platform.", "deployment_modes": [ "saas", "self_host", "on_prem" ], "deployment_note": "Configuration is applied to a locally run gateway through the APISIX Admin API (`http://127.0.0.1:9180/apisix/admin/routes`) and a `config.yaml`, reloaded after changes ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). Commercial on-prem licensing is per gateway CPU core, annually: \"Our on-premises API gateway pricing is based on API Gateway CPU cores\" ([API7 pricing](https://api7.ai/pricing)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "partial", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Not a base-URL-swap SaaS: you configure APISIX routes whose `override.endpoint` points at a provider, so clients call your gateway's route paths ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). No proprietary SDK. Streaming caveats are explicit: `max_stream_duration_ms` can truncate an SSE stream without a protocol terminator such as `[DONE]`, `message_stop` or `response.completed`, and Bedrock streaming returns AWS EventStream binary framing (`Content-Type: application/vnd.amazon.eventstream`) that clients must parse themselves; Bedrock routes must end in `/converse` (`/model//converse-stream` for streaming) ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a.; no model total is stated, only examples such as `gpt-4`, `deepseek-chat`, `text-embedding-3-small` ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/), undated).", "provider_count_low": 10, "provider_count_high": 20, "provider_count_source_note": "\"20+ model providers\" on the AI gateway page ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/), undated). The plugin reference enumerates 10 provider values — `openai`, `deepseek`, `azure-openai`, `aimlapi`, `anthropic`, `openrouter`, `gemini`, `vertex-ai`, `bedrock`, `openai-compatible` — plus \"other OpenAI-compatible APIs\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), undated).", "first_party_vs_partner": "All third-party: the gateway forwards to upstream provider APIs (OpenAI, DeepSeek, Azure OpenAI, Anthropic, OpenRouter, Gemini, Vertex AI, Bedrock, AIMLAPI) and ships no models of its own ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "custom_endpoint_support": "Yes: any endpoint can be set with `override.endpoint`, and it is required for the `openai-compatible` provider — \"Custom LLM provider endpoint, required when `provider` is `openai-compatible`\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)). vLLM/Ollama/SageMaker by name: n.a.", "byok_mode": "byok_only", "merchant_of_record": "The upstream provider, implicitly: the gateway is software the customer runs with its own provider keys, and no APISIX-side billing for model usage exists ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)). API7 invoices only the gateway license, per CPU core ([API7 pricing](https://api7.ai/pricing)). An explicit statement of who invoices model usage: n.a.", "key_management_note": "Keys live in the customer's own APISIX configuration, per provider instance via `instances.auth.header` / `auth.query`, with GCP service-account JSON in `auth.gcp.service_account_json` or the `GCP_SERVICE_ACCOUNT` environment variable; AWS `secret_access_key` and `session_token` are \"stored encrypted\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). There is no vendor plane: request/response payload logging is opt-in via `logging.payloads` (with `logging.summaries` for model, duration and token counts) and goes to the operator's own `http-logger`/`kafka-logger` sinks; incoming client headers including `Authorization` and `Cookie` are forwarded upstream by default unless stripped ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "api_openai_chat_completions_note": "Yes: routes proxy to the upstream `/v1/chat/completions` path and requests use the OpenAI `messages` array format ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).", "api_anthropic_messages_note": "Yes: setting the route URI to `/v1/messages` triggers automatic conversion between Anthropic Messages format and OpenAI-compatible format, including SSE streaming ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "api_openai_responses_note": "Partially documented: `openai-responses` appears as a target protocol and in the `max_tokens` mapping table as \"OpenAI Responses API\", but no `/v1/responses` path is given ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "api_embeddings_note": "Yes: documented with an override to `https://api.openai.com/v1/embeddings` using `text-embedding-3-small` ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).", "api_images_note": "n.a. (no image endpoint documented on the pages fetched).", "api_audio_note": "n.a. (no STT/TTS endpoint documented on the pages fetched).", "api_batch_note": "n.a.", "deployment_modes_note": "self-host (open-source data plane) ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)); on-prem commercial edition via API7 ([API7 pricing](https://api7.ai/pricing)). API7 also markets a managed option — \"the open-source & managed AI gateway\" ([API7 pricing](https://api7.ai/pricing)). Hybrid/VPC and air-gapped: n.a. — The managed/SaaS shape is offered by API7, the commercial steward, not by the Apache APISIX project itself.", "byok_mode_note": "byok_only: every example configures the customer's own provider credentials (`OPENAI_API_KEY`, `DEEPSEEK_API_KEY`, GCP service-account JSON, AWS keys) in the plugin's `auth` block ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).", "access_meta": { "qualified": [], "model_count_kind": null, "provider_count_kind": "vendor_total" }, "rel_timeout_config": "config_file", "rel_timeout_note": "`ai-proxy` / `ai-proxy-multi` key `timeout`, integer milliseconds, range `1`–`60000`, **default `30000`**. Connection reuse is tuned with `keepalive`, `keepalive_timeout` (`60000`) and `keepalive_pool` (`30`) ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/), [APISIX ai-proxy](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy/)).", "rel_retry_config": "config_file", "rel_retry_note": "Retry/fallback ordering is driven by `provider.priority`; no retry-count key, default count or backoff strategy is documented on the fetched plugin pages: `n.a.` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)).", "rel_retry_default_note": null, "rel_fallback_config": "config_file", "rel_fallback_note": "ORDERED via `provider.priority` (higher-priority providers are tried first, lower priorities act as fallbacks); WEIGHTED distribution is separately available via `provider.weight` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)).", "rel_fallback_shape": "weighted", "rel_health_check_config": "config_file", "rel_health_check_note": "`config_file`, and it is both an upstream health check and unhealthy-node ejection: APISIX will \"Enable health check on the upstream node and will automatically filter unhealthy nodes during load balancing\", plus \"Intelligent tracking of unhealthy upstream services\" (circuit breaking) ([apache/apisix](https://github.com/apache/apisix)). The AI Gateway landing page also advertises \"health checks and weighted balancing\" without naming keys ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "`not_documented` for provider failover. The only HA statement is about the control store: \"Support to configure multiple etcd addresses in the same cluster\" ([apache/apisix](https://github.com/apache/apisix)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights supported: `provider.weight`, `balancer.algorithm` (`roundrobin` default, or `chash`) with `balancer.hash_on` / `balancer.key` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)). The repo describes round-robin \"with weight\", consistent-hash balancing, and custom algorithms in the `balancer` phase ([apache/apisix](https://github.com/apache/apisix)).", "architecture_class": "interpreted_proxy", "architecture_note": "Repo language breakdown Lua 81.6% (OpenResty-style plugin runtime; Java/Go/Python/Node.js plugins run out-of-process via RPC, and Wasm is experimental) ([apache/apisix](https://github.com/apache/apisix)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Runs \"from bare-metal to Kubernetes\", official Helm charts installed with `helm repo add apisix https://apache.github.io/apisix-helm-chart` then `helm install apisix apisix/apisix` ([apache/apisix](https://github.com/apache/apisix), [APISIX Helm chart](https://apisix.apache.org/docs/helm-chart/apisix/)).", "latency_claims": [ { "url": "https://github.com/apache/apisix", "unit": "ms", "scope": "gateway_overhead", "value": "0.2", "metric": "mean", "publisher": "vendor", "conditions": "18k QPS on a single core. General API-gateway benchmark, NOT measured on LLM proxying; payload and cache state not stated." }, { "url": "https://github.com/apache/apisix", "unit": "ms", "scope": "gateway_overhead", "value": "0.2", "metric": "mean", "publisher": "vendor", "conditions": "140,000 QPS on an eight-core AWS server. Again general gateway traffic, not LLM proxying." } ], "throughput_claims": [ { "url": "https://github.com/apache/apisix", "unit": "QPS", "scope": "capacity", "value": "18,000", "metric": "sustained", "publisher": "vendor", "conditions": "Single core, general gateway traffic rather than LLM proxying." }, { "url": "https://github.com/apache/apisix", "unit": "QPS", "scope": "capacity", "value": "140,000", "metric": "sustained", "publisher": "vendor", "conditions": "Eight-core AWS server, general gateway traffic." } ], "streaming_support": "partial", "streaming_note": "SSE handling is documented for the `mcp-bridge` plugin, which converts stdio MCP servers into \"scalable HTTP SSE services\"; no LLM-streaming caveat is stated on the fetched AI plugin pages ([apache/apisix](https://github.com/apache/apisix)).", "guard_pii": "not_documented", "guard_pii_note": "No PII/DLP plugin on the fetched AI plugin pages ([AI Gateway](https://apisix.apache.org/ai-gateway/), [ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "guard_injection": "sync_block", "guard_injection_note": "`ai-prompt-guard` matches allow/deny regex patterns against prompts and rejects with HTTP 400; detection is regex, not a classifier ([ai-prompt-guard](https://apisix.apache.org/docs/apisix/next/plugins/ai-prompt-guard/))", "guard_content": "sync_block", "guard_content_note": "`ai-aws-content-moderation` sends the request body to AWS Comprehend and rejects when a category or the overall toxicity threshold is exceeded, returning `400 Bad Request` with e.g. \"request body exceeds PROFANITY threshold\" ([ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))", "guard_custom": "sync_block", "guard_custom_note": "Policy form is PCRE-style allow/deny pattern lists in `ai-prompt-guard`, plus per-category numeric moderation thresholds ([ai-prompt-guard](https://apisix.apache.org/docs/apisix/next/plugins/ai-prompt-guard/), [ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages — model selection is per route via `ai-proxy` configuration rather than an administrative allow/deny list ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "guard_execution_location": "both", "guard_third_party": [ "AWS Comprehend" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))", "log_content": "metadata_only", "log_content_note": "`metadata_only` — and in fact less: in `ai-proxy`, `logging.summaries` and `logging.payloads` both default to `false`, so neither token summaries nor request/response bodies are logged unless enabled ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "log_opt_out": "yes", "log_opt_out_note": "`logging.payloads: false` (the default) disables body logging while `logging.summaries: true` can keep token statistics ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "tracing_support": "otel", "tracing_note": "OpenTelemetry natively via the `opentelemetry` plugin (binary OTLP over HTTP, configurable sampler, optional full request-lifecycle spans); the plugin is disabled by default. Multi-step agent traces are not modelled — spans are HTTP-request-scoped ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/))", "export_destinations": [ "OpenTelemetry", "Prometheus", "Kafka", "HTTP logger", "Loki", "Datadog", "Splunk", "Google Cloud Logging" ], "export_destinations_note": "OTLP/HTTP collectors for traces ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/)); `http-logger` and `kafka-logger` for AI request logs, with the `llm_content_risk_level` variable available to loggers ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([AI Gateway](https://apisix.apache.org/ai-gateway/))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` ([AI Gateway](https://apisix.apache.org/ai-gateway/))", "observability_requires_saas": "No — everything runs and exports from the customer's own deployment ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/))", "log_retention_note": "`n.a.` — determined by the customer's log sink ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "Two figures, both general-gateway (not LLM-specific): \"The single-core QPS reaches 18k with an average delay of fewer than **0.2 milliseconds**\" (metric: average delay; single core; payload/cache/hardware model unspecified) and \"APISIX's QPS reaches 140,000 with a latency of only **0.2 ms**\" using \"AWS's eight-core server\" (payload and cache conditions not stated). No p50/p99 or gateway-overhead breakdown is given ([apache/apisix](https://github.com/apache/apisix)).", "multi_surface": {}, "throughput_prose": "**18k QPS** single-core and **140,000 QPS** on an eight-core AWS server; payload and cache conditions not stated, and neither figure is attributed to LLM proxying ([apache/apisix](https://github.com/apache/apisix)).", "benchmark_independence": "Vendor/project self-published (Apache APISIX repo). No independent third-party benchmark and no claims about competitors on the fetched pages ([apache/apisix](https://github.com/apache/apisix))." }, "guard_pii_default": "not_documented", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "The published QPS/latency numbers describe APISIX as a general API gateway, not as an AI proxy carrying token streams; do not read them as LLM gateway overhead ([apache/apisix](https://github.com/apache/apisix)).", "guard_notes": "APISIX is the most privacy-default of the group: with payload and summary logging both off by default, an operator must deliberately turn on observability, which is the inverse of most hosted gateways ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).", "fit_agent_features_note": "APISIX-MCP documentation describes Cursor Agent mode and YOLO Mode automatically invoking MCP tools, including plugin analysis and route-management tools such as `get_plugins_list`, `get_plugin_schema`, `create_route`, `update_route`, and `get_route`. It also states that the workflow supports task decomposition, closed-loop validation, and iterative optimization; gateway-native sessions, threads, and A2A are not documented. ([APISIX and AI/MCP intelligent API management](https://apisix.apache.org/blog/2025/04/01/embrace-intelligent-api-management-with-ai-and-mcp/))", "fit_byok_required": "yes", "fit_byok_required_note": "Effectively yes: the ai-proxy examples configure provider API keys or AWS credentials (Bedrock \"requires AWS SigV4 credentials and a non-empty AWS region\"), though the page does not state a general first-call rule ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))", "fit_client_sdk_langs": [], "fit_client_sdks_note": "The API7 integration catalog lists OpenAI SDK and Anthropic SDK integrations. The documented TypeScript integration uses `ai`, `@ai-sdk/openai`, and `@ai-sdk/openai-compatible`; official Apache APISIX client libraries and supported client languages are not otherwise documented. ([API7 AI Gateway integrations](https://docs.api7.ai/ai-gateway/integrations))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Yes: the `ai-proxy` plugin documents Amazon Bedrock authentication with AWS IAM credentials and SigV4 signing and Vertex AI authentication with a GCP service-account JSON (`auth.gcp.service_account_json`); Azure OpenAI uses an `api-key` header (no Entra/managed identity documented). ([API7/APISIX ai-proxy plugin reference](https://docs.api7.ai/hub/ai-proxy))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented); open-source install pages state no payment requirement ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))", "fit_ecosystem_notes": "Apache APISIX is open source, Apache 2.0 licensed, platform agnostic, and described as having no vendor lock-in, portable configuration, and standalone, cluster, bare-metal, and Kubernetes deployment options. Its ecosystem includes an Ingress Controller, Helm Charts, Docker, Java/Go/Python plugin runners, MCP components, and integrations with OpenAI, DeepSeek, OpenAI-compatible services, AWS Comprehend, Azure OpenAI, and Azure AI Search; API7 documentation also describes open-source gateway, cloud, and hybrid-cloud access. ([Apache APISIX AI Gateway](https://apisix.apache.org/ai-gateway/))", "fit_eval_prereqs": "Docker version 20.10 or later and `curl`; no account, credit card, cloud account, cluster, or licence. For an AI completion you additionally need a provider key — the plugin examples use `OPENAI_API_KEY`, `DEEPSEEK_API_KEY`, `AZ_OPENAI_API_KEY`, `ANTHROPIC_API_KEY` ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "Yes, but in two parts: the getting-started page verifies with `curl \"http://127.0.0.1:9080/get\"` (not a completion), while the ai-proxy page shows the LLM call `curl \"http://127.0.0.1:9080/anything\" -X POST -H \"Content-Type: application/json\" -H \"Host: api.openai.com\" -d '{\"messages\":[{\"role\":\"system\",\"content\":\"You are a mathematician\"},{\"role\":\"user\",\"content\":\"What is 1+1?\"}]}'` after you configure a Route with the `ai-proxy` plugin ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "Apache APISIX is itself an open-source API gateway and AI gateway, built on NGINX and etcd, with an AI-focused plugin ecosystem. Documented AI plugins include `ai-proxy`, `ai-proxy-multi`, `ai-request-rewrite`, `ai-rate-limiting`, `ai-prompt-decorator`, `ai-prompt-template`, `ai-prompt-guard`, `ai-aws-content-moderation`, and `ai-rag`. ([Introducing Apache APISIX AI Gateway](https://apisix.apache.org/blog/2025/04/08/introducing-apisix-ai-gateway/))", "fit_helm_chart": "apache/apisix-helm-chart; API7 ships api7/gateway from https://charts.api7.ai", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Apache APISIX documents a Kubernetes Ingress Controller and Helm Charts. The official Apache repository is `apache/apisix-helm-chart`, containing the Apache APISIX Helm Chart and Apache APISIX Ingress Controller Helm Chart; a separate API7 chart is named `api7/gateway` in the `https://charts.api7.ai` repository. ([Apache APISIX Helm Charts repository](https://github.com/apache/apisix-helm-chart))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "Apache APISIX documents MCP support through gateway proxying and plugins: it can proxy Streamable HTTP traffic between MCP clients and backend MCP servers, and the `mcp-bridge` plugin converts stdio-based MCP servers to scalable HTTP SSE services. APISIX-MCP is also documented as an open-source npm/GitHub MCP server, available with `npx -y apisix-mcp`, for managing APISIX through its Admin API. ([Apache APISIX MCP Protocol AI Gateway guide](https://apisix.apache.org/learning-center/mcp-protocol-ai-gateway/))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_credit_card", "fit_model_gate", "fit_terraform" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented) ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/), [Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/))", "fit_onboarding_notes": "After install you must configure a Route plus `ai-proxy` with provider auth and model; Kubernetes examples require `kubectl apply -f ai-proxy-ic.yaml` and declarative config uses `adc sync -f adc.yaml`; logger examples require setting up Kafka. Non-container installs need `apisix init` then `apisix start`, with config in `conf/config.yaml` and optional systemd management ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/), [Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))", "fit_openai_dropin": "partial", "fit_openai_dropin_note": "An OpenAI SDK integration is listed, but generic drop-in compatibility is not explicitly documented. The Vercel AI SDK example configures a custom `baseURL` of `http://127.0.0.1:3000/v1` with `@ai-sdk/openai`. ([API7 Vercel AI SDK integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/vercel-ai-sdk))", "fit_prod_prereqs": "etcd as the configuration store (installed automatically by the Docker and Helm methods); Docker 20.10+ for the container path; \"Production environments should enable Admin API authentication\"; the quickstart containers use Docker host network mode. No database, Redis, ClickHouse, load balancer, control-plane account, licence, or S3 requirement is stated ([Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/), [Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain", "LangGraph", "LlamaIndex" ], "fit_python_frameworks_note": "Python integrations are documented for LangChain through `langchain-openai` and `ChatOpenAI`, and for LangGraph through `langgraph`. LlamaIndex is documented through `llama-index-llms-openai` and the `OpenAIResponses` class; `OpenAILike` is also mentioned for Chat Completions compatibility. ([API7 LangChain integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/langchain))", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "The getting-started page is prose plus commands, and its verification call is not an LLM completion. The actual LLM call lives on the ai-proxy plugin reference page.", "fit_quickstart_url": "https://apisix.apache.org/docs/apisix/getting-started/README/", "fit_self_host_docs": "command_published", "fit_self_host_install": "`git clone https://github.com/apache/apisix-docker.git`, `cd apisix-docker/example`, `docker-compose -p docker-apisix up -d`; ARM: `docker-compose -p docker-apisix -f docker-compose-arm64.yml up -d`; Helm: `helm repo add apisix https://charts.apiseven.com`, `helm repo update`, `helm install apisix apisix/apisix --create-namespace --namespace apisix`; RPM: `sudo yum install apisix`; DEB: `sudo apt install -y apisix` ([Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented)", "fit_terraform_ref": null, "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "openai_compat_only", "fit_vercel_ai_sdk_note": "Vercel AI SDK integration is documented using the npm packages `ai` and `@ai-sdk/openai`; it creates a provider with `createOpenAI`. The example uses the gateway base URL `http://127.0.0.1:3000/v1`. ([API7 Vercel AI SDK integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/vercel-ai-sdk))", "fit_vercel_ai_sdk_package": "@ai-sdk/openai with a custom baseURL" }, { "slug": "bifrost", "name": "Bifrost", "company": "Maxim AI", "tagline": "Go-based open-source AI gateway focused on low proxy overhead, with an enterprise tier for clustering and SSO.", "category": "open_source", "deployment": "self_host_only", "license": "Apache-2.0", "founded_year": 2023, "hq_country": "United States", "url_home": "https://www.getmaxim.ai/bifrost", "url_docs": "https://docs.getbifrost.ai/", "url_pricing": "https://www.getmaxim.ai/pricing", "url_status": null, "url_github": "https://github.com/maximhq/bifrost", "url_changelog": "https://github.com/maximhq/bifrost/releases", "url_models_api": null, "github_stars": 8359, "model_count": null, "model_count_note": "No aggregate model total is published. The documentation states \"20+ AI providers\" and lists example models rather than a catalogue, so provider breadth is the only figure the vendor stands behind.", "provider_count": 23, "token_markup_pct": 0, "credit_fee_pct": 0, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Self-hosted with your own provider keys and vault integration; no gateway token fee.", "seat_fee_usd_month": 0, "free_tier": "OSS tier is \"Free Forever\": drop-in gateway, OpenTelemetry observability, budgets, rate limits, virtual keys, custom routing, fallbacks, simple and semantic caching, MCP gateway, prompt repository.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "OSS is free (Apache-2.0); infra cost only. Enterprise adds guardrails, cluster mode, adaptive load balancing, SAML/OIDC SSO, vault integration, log exports, audit logs, RBAC and SLAs at custom pricing after a 14-day trial; VPC, on-prem and air-gapped installs are enterprise options.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions", "Anthropic Messages", "OpenAI Embeddings", "MCP" ], "sdk_note": "Drop-in replacement: change the base URL of an existing OpenAI/Anthropic client; also ships a Go SDK and CLI.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 0.99, "latency_source_note": "Vendor-run (Maxim AI): 0.99 ms mean gateway overhead vs LiteLLM's 40 ms against a 60 ms mock OpenAI upstream at 500 RPS on a t3.medium; a separate Maxim stress test reports ~11-20 microseconds internal overhead at 5,000 RPS on a t3.xlarge with 100% success. Not independently verified, and LiteLLM's own competing benchmark measured Bifrost at 4.54 ms added p99.", "throughput_rps": 5000, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "No branded ZDR commitment. Functionally reachable through disable_content_logging.", "air_gapped": true, "best_for": "Teams that want a fast Go gateway they can self-host for free and later buy clustering, SSO and guardrails from a single vendor.", "strengths": [ "Go implementation with the lowest vendor-claimed proxy overhead in this set", "Generous OSS tier: semantic caching, MCP gateway, budgets, virtual keys and prompt repository at no cost", "Apache-2.0 license with VPC, on-prem and air-gapped deployment paths", "Ships with OpenTelemetry-native observability tied into Maxim's evaluation platform" ], "weaknesses": [ "Headline latency and throughput figures are all vendor-run against mock upstreams; no independent benchmark found", "Much smaller community than LiteLLM, Kong or APISIX (7.6k vs 16.8k-57.5k stars)", "Guardrails, cluster mode, adaptive load balancing, SSO, RBAC and audit logs are enterprise-only with unpublished pricing", "Provider and model counts are inconsistent across the vendor's own pages" ], "avoid_if": "You need certified compliance artifacts, published enterprise pricing, or a large third-party operator community to lean on.", "reviews": [ { "url": "https://kleinloog.ch/articles/the-3-llm-gateways-on-my-shortlist/", "why": "Arnoud Kleinloog explains why Bifrost (Maxim AI) made his personal shortlist over LiteLLM, Portkey, Helicone, BricksLLM and Martian, with deployment and maturity caveats.", "date": "2026-04-19", "type": "written", "title": "The 3 LLM Gateways on My Shortlist", "publisher": "kleinloog.ch" }, { "url": "https://www.youtube.com/watch?v=xPdAOvvxtLs", "why": "Established Python-teaching channel runs Bifrost locally in Docker, configures OpenAI/Anthropic/Mistral/Ollama providers and wires it into Claude Code and an MCP server.", "date": "2026-02-09", "type": "video", "title": "Bifrost: High-Speed Open Source AI Gateway", "publisher": "YouTube - NeuralNine" }, { "url": "https://www.youtube.com/watch?v=5hk5d_GKY_s", "why": "Independent tutorial that deploys Bifrost via Docker, adds providers including a custom OpenAI-compatible endpoint, then inspects its usage, cost and latency monitoring.", "date": "2026-01-29", "type": "video", "title": "Bifrost LLM Proxy Tutorial: Route and Monitor Requests Across Multiple Providers", "publisher": "YouTube - Ready Tensor" } ], "sources": { "cap_mcp": "https://www.getmaxim.ai/pricing", "company": "https://github.com/maximhq/bifrost", "license": "https://github.com/maximhq/bifrost", "sdk_note": "https://github.com/maximhq/bifrost", "cap_audio": "https://docs.getbifrost.ai/", "free_tier": "https://www.getmaxim.ai/pricing", "air_gapped": "https://www.getmaxim.ai/pricing", "hq_country": "https://www.cbinsights.com/company/maxim-ai", "zdr_status": "https://docs.getbifrost.ai/enterprise/log-exports", "cap_budgets": "https://www.getmaxim.ai/pricing", "model_count": "https://docs.getbifrost.ai/", "api_surfaces": "https://docs.getbifrost.ai/", "cap_failover": "https://www.getmaxim.ai/pricing", "founded_year": "https://www.cbinsights.com/company/maxim-ai", "github_stars": "https://github.com/maximhq/bifrost", "product_fate": "https://github.com/maximhq/bifrost/releases", "regions_note": "https://docs.getbifrost.ai/enterprise/log-exports", "cap_image_gen": "https://docs.getbifrost.ai/", "url_changelog": "https://github.com/maximhq/bifrost/releases", "vendor_status": "https://www.getmaxim.ai/blog/announcing-maxim-ais-general-availability-and-the-3m-funding-round-led-by-elevation-capital/", "byok_supported": "https://docs.getbifrost.ai/", "cap_embeddings": "https://docs.getbifrost.ai/", "cap_guardrails": "https://www.getmaxim.ai/pricing", "certifications": "https://www.getmaxim.ai/bifrost/docs/enterprise/governance", "config_as_code": "https://docs.getbifrost.ai/deployment-guides/config-json", "credit_fee_pct": "https://www.getmaxim.ai/pricing", "provider_count": "https://github.com/maximhq/bifrost", "retention_days": "https://docs.getbifrost.ai/enterprise/log-exports", "sla_uptime_pct": "https://www.getmaxim.ai/bifrost/resources/enterprise-deployment", "throughput_rps": "https://www.getmaxim.ai/bifrost", "bifrost_pricing": "https://www.getmaxim.ai/pricing", "cap_rate_limits": "https://www.getmaxim.ai/pricing", "logs_by_default": "https://docs.getbifrost.ai/enterprise/log-exports", "cap_cache_simple": "https://www.getmaxim.ai/pricing", "cap_virtual_keys": "https://www.getmaxim.ai/pricing", "model_count_note": "https://www.getmaxim.ai/bifrost", "token_markup_pct": "https://www.getmaxim.ai/pricing", "cap_observability": "https://www.getmaxim.ai/pricing", "last_release_date": "https://github.com/maximhq/bifrost/releases/tag/core%2Fv1.8.4", "openai_compatible": "https://github.com/maximhq/bifrost", "cap_cache_semantic": "https://www.getmaxim.ai/pricing", "cap_load_balancing": "https://www.getmaxim.ai/pricing", "seat_fee_usd_month": "https://www.getmaxim.ai/pricing", "guardrails_location": "https://docs.getbifrost.ai/enterprise/log-exports", "latency_overhead_ms": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "latency_source_note": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "requires_vendor_sdk": "https://www.getmaxim.ai/docs/bifrost/usage/http-transport/integrations/openai-compatible", "self_host_cost_note": "https://www.getmaxim.ai/pricing", "cap_prompt_management": "https://www.getmaxim.ai/pricing", "data_export_available": "https://docs.getbifrost.ai/api-reference/logging/get-logs", "prompt_transits_vendor": "https://www.getmaxim.ai/docs/self-hosting/overview", "cap_conditional_routing": "https://www.getmaxim.ai/pricing" }, "verified_at": { "cap_mcp": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "cap_audio": "2026-08-29", "free_tier": "2026-08-29", "air_gapped": "2026-08-29", "hq_country": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-31", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "github_stars": "2026-09-25", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "throughput_rps": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "cap_virtual_keys": "2026-08-29", "model_count_note": "2026-08-31", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-25T16:55:57.720036+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed with several releases a week. Built by Maxim AI, an independent venture-backed company that raised a $3M seed led by Elevation Capital.", "last_release_date": "2026-08-27", "sla_uptime_pct": 99.999, "sla_url": "https://www.getmaxim.ai/bifrost/resources/enterprise-deployment", "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "depends", "prompt_path_note": "A binary, Docker image or embedded Go library you run. Maxim also offers a private deployment where both data and control planes sit in your VPC, or a VPC-peered hybrid.", "logs_by_default": "full_content", "logs_note": "Every offloadable payload field is uploaded to object storage by default — your storage, but stored. Setting disable_content_logging skips content and does not persist reveal data.", "retention_days": 365, "retention_note": "Default 365 days, minimum one day, and zero disables cleanup entirely rather than disabling storage.", "zdr_status": "not_published", "trains_on_customer_data": "not_published", "training_note": "Nothing on the README, enterprise docs or governance page addresses training.", "subprocessors_url": null, "regions_note": "Your own infrastructure. Log offload targets S3 or GCS in a region you name. Maxim offers full VPC isolation or a VPC-peered hybrid.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type 2 for Maxim AI", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "reported compliant on the governance page", "status": "certified" }, "iso27001": { "detail": "trust center says compliant, governance page says in progress", "status": "conflicting" }, "hipaa_baa": { "detail": "trust center says compliant, governance page says not applicable", "status": "conflicting" } }, "guardrails_location": "self_hosted", "guardrails_note": "Guardrail redaction happens in the Bifrost instance you run, and the reveal mapping stays with your own log row.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "oss_with_managed_tier", "pricing_model_note": "OSS 'Free Forever' (Docker/K8s/Go binary) for developers, small teams and self-managed deployments. Enterprise is 'Custom Pricing' for VPC/on-prem/air-gapped, quote-only.", "min_commitment": "None for OSS. Enterprise not stated.", "overage_note": "None stated. Self-hosting infrastructure cost is yours.", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Simple (exact) and semantic caching both in OSS. Semantic caching 'reduces costs and latency for similar queries with semantic similarity caching'. Bifrost prices no tokens, so caching saving is fully the customer's. MCP Code Mode claims to cut tokens by 50%+ separately.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "'Real-time notifications for budget limits, failures and performance issues' via email/Slack/PagerDuty/Teams/Webhook is Enterprise.", "tier": "enterprise", "supported": true }, "virtual_keys": { "note": "Supported, free — 'control access permissions, budgets, rate limits, and routing per consumer'.", "tier": "oss", "supported": true }, "rate_limiting": { "note": "Supported, free.", "tier": "oss", "supported": true }, "spend_webhooks": { "note": "Webhook notification channel is Enterprise.", "tier": "enterprise", "supported": true }, "model_allowlists": { "note": "Not stated for models. MCP tool allow-lists are: 'strict allow-lists for which MCP tools are available per virtual key'.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Enforced through virtual keys.", "tier": "oss", "supported": true }, "budget_caps_per_team": { "note": "Hierarchical cost control across virtual keys, teams and customers.", "tier": "oss", "supported": true } }, "cost_attribution": "Per virtual key, consumer, team and customer. Per-user and per-tag not stated.", "cost_export": "OTel-compatible metrics and traces (any OTel backend can receive data). CSV, cost API, webhook-of-spend, S3 and warehouse export not stated.", "product_kind_note": "Maxim calls Bifrost \"a high-performance AI gateway unifying 20+ providers through a single OpenAI-compatible API\", with \"automatic failover, load balancing, semantic caching, and enterprise-grade governance\" ([Bifrost overview](https://docs.getbifrost.ai/overview)), and on GitHub \"the fastest enterprise AI gateway (50x faster than LiteLLM)\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). It is a self-hosted routing proxy (also embeddable as a Go SDK).", "deployment_modes": [ "self_host", "hybrid_vpc", "on_prem", "air_gapped" ], "deployment_note": "`npx -y @maximhq/bifrost` or `docker run -p 8080:8080 maximhq/bifrost` (versioned tags such as `maximhq/bifrost:v1.3.9`, amd64/arm64), with a mounted data volume holding `config.json`, `config.db` and `logs.db`; config/log stores can be SQLite or PostgreSQL 16+ (UTF8) ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)). Embedded use is `go get github.com/maximhq/bifrost/core` ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). Tier gating: OSS self-managed is \"Free Forever\"; VPC/on-prem/air-gapped, clustering, guardrails and private networking are Enterprise \"Custom Pricing\" ([Maxim pricing](https://www.getmaxim.ai/pricing); [maximhq/bifrost](https://github.com/maximhq/bifrost)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap is the primary pattern: \"Replace existing AI SDK connections by changing just the base URL\" with per-SDK prefixes `.../openai`, `.../anthropic`, `.../genai`, plus native AWS Bedrock SDK integration and LiteLLM/LangChain/PydanticAI compatibility ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27; [Bifrost overview](https://docs.getbifrost.ai/overview)). No proprietary SDK is required, though a Go SDK exists for embedded use ([maximhq/bifrost](https://github.com/maximhq/bifrost)). Models can be addressed as `openai/gpt-4o-mini` or bare `gpt-4o-mini` resolved through the Model Catalog ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a.: no model total is stated; the overview gives \"20+ AI providers\" and names examples only ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "provider_count_low": 20, "provider_count_high": 23, "provider_count_source_note": "Vendor numbers differ: \"20+ providers\" ([Bifrost overview](https://docs.getbifrost.ai/overview), undated) vs \"23+ providers\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). The overview page enumerates 22 named providers (OpenAI, Anthropic, AWS Bedrock, Google Vertex AI, Azure OpenAI, Google Gemini, Groq, Mistral, Cohere, Cerebras, DeepSeek, Ollama, Hugging Face, OpenRouter, Perplexity, ElevenLabs, Nebius, xAI, Parasail, Replicate, SGL, vLLM) ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "first_party_vs_partner": "All third-party or customer-run: Bifrost owns no models and routes to provider APIs plus self-hosted engines such as vLLM (\"self-hosted OpenAI-compatible inference\") and Ollama (\"local inference with OpenAI-compatible format\") ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "custom_endpoint_support": "Yes, indirectly: vLLM and Ollama are first-class providers for self-hosted/local inference ([Bifrost overview](https://docs.getbifrost.ai/overview)). Explicit registration of an arbitrary private URL: n.a. ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "byok_mode": "byok_only", "merchant_of_record": "n.a. as an explicit statement; the pages mention \"cost control\" but do not specify billing, invoicing, credits or payment responsibility ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)), and Maxim invoices only Enterprise \"Custom Pricing\" ([Maxim pricing](https://www.getmaxim.ai/pricing)).", "key_management_note": "Keys are held in the customer's own Config Store (SQLite or PostgreSQL) alongside provider configs and MCP settings, typically referenced from environment variables; Bifrost adds \"intelligent API key distribution with weighted load balancing, model-specific filtering, and automatic failover\" and virtual keys as \"the primary governance entity\" controlling access, budgets, rate limits and routing per consumer, hierarchically at virtual-key, team and customer levels ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Bifrost overview](https://docs.getbifrost.ai/overview)). Because deployment is customer-run, request logs stay in the customer's own logs store ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)); explicit vendor-visibility statements: n.a.", "api_openai_chat_completions_note": "Yes: `POST http://localhost:8080/v1/chat/completions`, working \"with any provider (OpenAI, Anthropic, Bedrock, etc.)\", and all Bifrost APIs \"follow OpenAI request/response format\" ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).", "api_anthropic_messages_note": "Yes, via SDK-compatible base-URL replacement: point the Anthropic SDK at `http://localhost:8080/anthropic` as a \"drop-in replacement for the Anthropic Python and TypeScript SDKs\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27; [Bifrost overview](https://docs.getbifrost.ai/overview)).", "api_openai_responses_note": "n.a. (not documented on the pages fetched).", "api_embeddings_note": "Yes: embeddings are documented for providers including Google Gemini, Cohere, Nebius, SGL and vLLM (\"chat, embeddings, and STT\") ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "api_images_note": "Yes at the interface level: \"support for text, images, audio, and streaming, all behind a common interface\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). Dedicated image endpoint path: n.a.", "api_audio_note": "Yes, both directions: Hugging Face \"TTS, and STT\", ElevenLabs \"text-to-speech and speech-to-text models\", vLLM \"chat, embeddings, and STT\" ([Bifrost overview](https://docs.getbifrost.ai/overview)). Endpoint paths: n.a.", "api_batch_note": "n.a.: no general batch or asynchronous API is described; only Replicate's \"prediction-based architecture with async modes\" ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "deployment_modes_note": "self-host via NPX binary, Docker, Kubernetes or Go binary ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up); [Maxim pricing](https://www.getmaxim.ai/pricing): \"SELF-HOSTED: DOCKER | K8S | GO BINARY\"); hybrid/VPC, on-prem and air-gapped on the Enterprise tier — \"ENTERPRISE READY: VPC | ON-PREM | AIR-GAPPED\" ([Maxim pricing](https://www.getmaxim.ai/pricing)), with \"In-VPC Deployments… Deploy within your private cloud infrastructure with VPC isolation and enhanced security controls\" ([Bifrost overview](https://docs.getbifrost.ai/overview)). Hosted SaaS: n.a. — the docs mention only \"a hosted or production deployment\" as customer-run guidance ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).", "byok_mode_note": "byok_only: providers are configured with the customer's own keys, e.g. `\"value\": \"env.OPENAI_API_KEY\"`, \"Add API keys with clicks, not code\" ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)); the OSS gateway is \"Free Forever\" and no model credits are sold ([Maxim pricing](https://www.getmaxim.ai/pricing)).", "access_meta": { "qualified": [ "api_anthropic_messages", "api_images", "api_audio" ], "model_count_kind": null, "provider_count_kind": "vendor_total" }, "rel_timeout_config": "config_file", "rel_timeout_note": "A `timeout` config parameter is documented, but no default value is published: default `n.a.` ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "rel_retry_config": "config_file", "rel_retry_note": "A `retry` config parameter is documented; default retry count and backoff strategy are not published: `n.a.` ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "rel_retry_default_note": null, "rel_fallback_config": "config_file", "rel_fallback_note": "Fallback across providers/models is documented; WEIGHTED selection applies to API keys (weighted key selection, cited at ~10 ns selection cost) rather than to the fallback chain, and the docs do not state a weighted fallback split ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "rel_fallback_shape": "weighted", "rel_health_check_config": "not_documented", "rel_health_check_note": "`not_documented` as an upstream provider health check or circuit breaker. What is documented is Cluster Mode: peer-to-peer HA failover between Bifrost nodes — i.e. gateway-level HA, not provider ejection ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Cluster Mode covers node failover; no cross-region provider failover configuration is documented ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "`config_file`, with weights on API keys (weighted key selection) ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "architecture_class": "compiled_binary", "architecture_note": "Go, using goroutines and `fasthttp`, distributed as an ~80MB binary ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Docker image `maximhq/bifrost`, `npx -y @maximhq/bifrost`, a `helm-charts` folder, and a Go SDK via `go get github.com/maximhq/bifrost/core` ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "latency_claims": [ { "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "unit": "ms", "scope": "gateway_overhead", "value": "0.99", "metric": "median added", "publisher": "vendor", "conditions": "Derived: median end-to-end 60.99ms against a mock OpenAI holding a fixed 60ms latency." }, { "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "unit": "ms", "scope": "end_to_end", "value": "804", "metric": "p50", "publisher": "vendor", "conditions": "500 RPS, AWS EC2 t3.medium (2 vCPU / 4 GB), us-east-1, 60s, 500 virtual users, real OpenAI Tier 5." }, { "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "unit": "ms", "scope": "end_to_end", "value": "1680", "metric": "p99", "publisher": "vendor", "conditions": "Same 500 RPS t3.medium run; max observed 6.13s." } ], "throughput_claims": [ { "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks", "unit": "req/s", "scope": "capacity", "value": "424", "metric": "sustained", "publisher": "vendor", "conditions": "500 RPS comparison test on t3.medium against real OpenAI. A separate 5,000 RPS stress run is Bifrost-only against a mocked provider with ~10KB payloads." } ], "streaming_support": "yes", "streaming_note": "Supported: the docs state support for \"text, images, audio, and streaming\"; no streaming caveat is stated on the fetched pages ([Bifrost overview](https://docs.getbifrost.ai/overview)).", "guard_pii": "sync_block", "guard_pii_note": "Providers include in-process regex (RE2 patterns with `entity_type`), Gitleaks-based secrets detection, Microsoft Presidio, and Azure AI Language PII. Each supports `action: detect_only | block | redact`, `redaction_strategy: replace | mask | hash`, and `redaction_mode: runtime | logs_only | runtime_reversible`. Azure PII defaults to `detect_only` ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails), [Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))", "guard_injection": "sync_block", "guard_injection_note": "Azure Content Safety jailbreak shield, Model Armor, CrowdStrike AIDR, Gray Swan, Patronus, Lakera, Repello Argus are configurable as guardrail providers ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))", "guard_content": "sync_block", "guard_content_note": "Bedrock Guardrails and Azure Content Safety are supported providers for harmful-content categories ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))", "guard_custom": "sync_block", "guard_custom_note": "Custom policy form is RE2 regex patterns with per-pattern flags, entity types, and actions, plus `ignored_secret_keywords` suppression lists on the secrets provider ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails))", "guard_model_policy": "yes", "guard_model_policy_note": "Virtual keys carry `allowed_models` per provider ([Budgets and limits](https://docs.getbifrost.ai/features/governance/budget-and-limits))", "guard_execution_location": "both", "guard_third_party": [ "Microsoft Presidio", "Azure AI Language", "AWS Bedrock Guardrails", "Google Model Armor", "Gitleaks" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` — the fetched guardrail pages document action outcomes but no explicit timeout/fail-open or fail-closed rule ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails))", "log_content": "configurable", "log_content_note": "Bifrost records metadata plus content: \"actual prompts, completions, and tool traffic\", including chat history, tool definitions and arguments, reasoning traces, embedding inputs, and verbatim provider bodies when `x-bf-store-raw-request-response` is used ([Content logging](https://docs.getbifrost.ai/features/observability/content-logging))", "log_opt_out": "yes", "log_opt_out_note": "`client.disable_content_logging: true` reduces log rows to metadata only; per-request override header `x-bf-disable-content-logging` works only when `client.allow_per_request_content_storage_override` is enabled; each observability connector has its own independent `disable_content_logging` defaulting to `false`; metrics never carry content ([Content logging](https://docs.getbifrost.ai/features/observability/content-logging))", "tracing_support": "otel", "tracing_note": "OpenTelemetry natively, with `disable_root_span_content` and `disable_content_logging` controls over span content ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))", "export_destinations": [ "OpenTelemetry", "Prometheus", "Datadog", "Langfuse", "Maxim", "S3", "Webhook" ], "export_destinations_note": "OTel to Grafana Cloud, Datadog, New Relic, Honeycomb, Langfuse, and self-hosted collectors ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel)); log payload offload to Amazon S3 and Google Cloud Storage only — \"Azure Blob, local filesystem, and data warehouse destinations are not implemented\" — with the logs DB on SQLite, Postgres, or ClickHouse and a BigQuery connector gated by `disable_content_logging` ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports), [Content logging](https://docs.getbifrost.ai/features/observability/content-logging))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — no feedback endpoint documented on the fetched pages ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` from the fetched Bifrost pages ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))", "observability_requires_saas": "No — logs live in the customer's own SQLite/Postgres/ClickHouse and object storage, and telemetry exports to the customer's collectors ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports), [Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))", "log_retention_note": "`client_config.log_retention_days`, default **365 days**, minimum 1 day, `0` disables cleanup ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "The most detailed published set in this report, all vendor-run: - 500 RPS test on AWS EC2 `t3.medium` (2 vCPU / 4 GB, `us-east-1`, 60s, 500 VUs, OpenAI Tier 5): Bifrost **P50 804 ms**, **P99 1.68 s**, **max 6.13 s** (LiteLLM comparison figures noted in field 11) ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - Median end-to-end **60.99 ms** against a mock OpenAI with 60 ms fixed latency, implying **gateway overhead 0.99 ms** ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - 5,000 RPS Bifrost-only stress test (~10KB payloads, mocked OpenAI): **gateway overhead 59 µs** on `t3.medium` and **11 µs** on `t3.xlarge` (4 vCPU / 16 GB), queue wait **47.13 µs** / **1.67 µs**, average latency including provider **2.12 s** / **1.61 s**, 100% success, peak memory 1312.79 MB / 3340.44 MB ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - Headline claim \"**<15µs added latency per request**\", conditions not stated in the claim itself ([maximhq/bifrost](https://github.com/maximhq/bifrost)).", "multi_surface": {}, "throughput_prose": "**424 req/s** sustained in the 500 RPS comparison test on `t3.medium` ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)); the 5,000 RPS stress run is Bifrost-only against a mocked provider with ~10KB payloads ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). Reproduction instructions are published ([Bifrost benchmarking guide](https://docs.getbifrost.ai/benchmarking/getting-started)).", "benchmark_independence": "Vendor-published, and **explicitly flagged: the same benchmark post publishes figures about a COMPETITOR, LiteLLM** — LiteLLM P50 38.65 s, P99 90.72 s, max 92.67 s, median end-to-end 100 ms against the 60 ms mock (≈40 ms overhead), 44.84 req/s ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). Those rival numbers are not verifiable from LiteLLM's own docs and should not be used; LiteLLM's own published figure is `8ms P95 latency at 1k RPS` ([BerriAI/litellm](https://github.com/BerriAI/litellm)). No independent third-party benchmark was found." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "choose_at_setup", "guard_custom_default": "choose_at_setup", "rel_notes": "Bifrost is the only product in this set that publishes hardware, VU count, duration, payload size and mock-provider baseline, which makes its own numbers auditable even where its competitor numbers are not ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)).", "guard_notes": "Bifrost has the most nuanced redaction model here: `logs_only` redacts what is stored while the model still sees the original, `runtime` redacts what the model sees, and `runtime_reversible` stores a reveal mapping that stays on the Bifrost log row and is never exported ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails), [Log exports](https://docs.getbifrost.ai/enterprise/log-exports)).", "fit_agent_features_note": "Bifrost returns LLM tool calls as suggestions requiring a separate execution API call, while Agent Mode supports autonomous multi-step tool execution with configurable auto-approval. Code Mode lets the model use Python to orchestrate multiple tools in a sandbox. ([Bifrost MCP gateway](https://www.getmaxim.ai/bifrost/resources/mcp-gateway))", "fit_byok_required": "optional", "fit_byok_required_note": "Not explicitly stated; the displayed first-call snippets contain no provider key, but provider setup \"supports adding API keys through the UI\" and the last Helm step is \"Configure Providers & Plugins\" ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_client_sdk_langs": [ "Go" ], "fit_client_sdks_note": "An official Go SDK is documented with `go get github.com/maximhq/bifrost/core`. Documented integrations also include the OpenAI, Anthropic, AWS Bedrock, Google GenAI, LiteLLM, and LangChain SDKs; the evidence does not provide a complete list of official Bifrost client-library languages. ([Bifrost GitHub repository](https://github.com/maximhq/bifrost))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Bifrost documents cloud-native authentication through AWS IRSA, GCP Workload Identity, and Azure Azure WIF, and describes this as native IAM integration. Specific AWS SigV4, Azure managed-identity, and GCP service-account authentication are not documented. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented) on either page ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_ecosystem_notes": "Bifrost is open source and supports self-hosted deployment in private VPCs, on-premises or bare-metal environments, air-gapped environments, and single-node or edge deployments. Documented deployment options include AWS, GCP, Azure, generic Kubernetes, Docker Compose, and a single Go binary; air-gapped operation is described as fully offline with no phone-home or telemetry. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))", "fit_eval_prereqs": "None stated: no account, credit card, cloud account, cluster, or provider API key is listed as a prerequisite; provider keys are added later through the Web UI or API ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `curl -X POST http://localhost:8080/v1/chat/completions -H \"Content-Type: application/json\" -d '{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, Bifrost!\"}]}'` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "bifrost/bifrost from https://maximhq.github.io/bifrost/helm-charts", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Bifrost documents the official Helm chart `bifrost/bifrost` from the repository `https://maximhq.github.io/bifrost/helm-charts`. ([Bifrost Helm deployment guide](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "Bifrost acts as both an MCP client and MCP server, connecting to external MCP servers over STDIO, HTTP, and SSE, auto-discovering tools, and exposing them through a single gateway URL. It supports tool filtering, virtual MCP servers, MCP Tool Groups, explicit tool execution, and Tool Hosting. ([Bifrost MCP gateway](https://www.getmaxim.ai/bifrost/resources/mcp-gateway))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_credit_card", "fit_gateway_platform", "fit_model_gate" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or waitlist wording ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_onboarding_notes": "Unversioned `npx`/`docker` commands are explicitly \"for local evaluation,\" so the evaluate and production paths diverge deliberately; configuration is via Web UI or API; verification commands include `kubectl port-forward svc/bifrost 8080:8080`, `curl http://localhost:8080/health` and `/metrics` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Existing OpenAI SDK applications require zero code changes beyond setting `base_url` to `\"http://localhost:8080/openai\"`; the Python and JavaScript examples retain an API key such as `\"your-openai-key\"` or `process.env.OPENAI_API_KEY`. ([Bifrost OpenAI-compatible integration](https://www.getmaxim.ai/docs/bifrost/usage/http-transport/integrations/openai-compatible))", "fit_prod_prereqs": "Kubernetes v1.19+, configured `kubectl`, Helm 3.2.0+, a Persistent Volume provisioner (required for SQLite, optional for Postgres-only), UTF8-encoded PostgreSQL storage, a required `image.tag`, and private registry credentials for Enterprise. The quickstart also says production should \"Pin an image version, enable authentication and TLS, and choose durable storage\" ([Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm), [Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain" ], "fit_python_frameworks_note": "LangChain integration is documented, including `langchain_openai.ChatOpenAI`, `langchain_anthropic.ChatAnthropic`, and `langchain_google_genai.ChatGoogleGenerativeAI`. LlamaIndex is not documented. ([Bifrost LangChain integration](https://docs.getbifrost.ai/integrations/langchain-sdk))", "fit_quickstart_steps": 4, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.getbifrost.ai/quickstart/gateway/setting-up", "fit_self_host_docs": "command_published", "fit_self_host_install": "`npx -y @maximhq/bifrost`; `docker pull maximhq/bifrost` + `docker run -p 8080:8080 maximhq/bifrost`; Helm: `helm repo add bifrost https://maximhq.github.io/bifrost/helm-charts`, `helm repo update`, `kubectl create secret generic bifrost-encryption-key --from-literal=encryption-key=\"$(openssl rand -base64 32)\"`, `helm install bifrost bifrost/bifrost --set image.tag=v1.4.11 ...`, production `helm install bifrost bifrost/bifrost -f production.yaml` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))", "fit_terraform": "official_modules", "fit_terraform_note": "A Terraform module source is documented as `github.com/maximhq/bifrost//terraform/modules/bifrost?ref=terraform/v0.1.0`. A Terraform Registry provider or module name is not documented. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))", "fit_terraform_ref": "github.com/maximhq/bifrost//terraform/modules/bifrost?ref=terraform/v0.1.0; not on the Registry", "fit_third_party_documented": {}, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get Bifrost running as an HTTP API gateway in 30 seconds with zero configuration.\" ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))", "fit_vercel_ai_sdk": "compatible_no_package", "fit_vercel_ai_sdk_note": "Bifrost is stated to be compatible with the Vercel AI SDK. No official provider package name is documented. ([Maxim AI homepage](https://www.getmaxim.ai/))", "fit_vercel_ai_sdk_package": null }, { "slug": "envoy-ai-gateway", "name": "Envoy AI Gateway", "company": "Envoy AI Gateway open-source project in the `envoyproxy` GitHub organization; maintainers are drawn from Tetrate, Bloomberg, Tencent, Netflix and Nutanix plus the KServe, Kubeflow, Envoy Proxy and Envoy Gateway projects", "tagline": "Kubernetes-native open-source AI gateway that adds a unified LLM and MCP data plane on top of CNCF Envoy Gateway, with a stable v1beta1 CRD API and no vendor-hosted tier.", "category": "open_source", "deployment": "self_host_only", "license": "Apache-2.0", "founded_year": null, "hq_country": null, "url_home": "https://aigateway.envoyproxy.io/", "url_docs": "https://aigateway.envoyproxy.io/docs/", "url_pricing": null, "url_status": null, "url_github": "https://github.com/envoyproxy/ai-gateway", "url_changelog": "https://aigateway.envoyproxy.io/release-notes/", "url_models_api": null, "github_stars": 2112, "model_count": null, "model_count_note": "No model catalogue exists: the project ships no hosted model list and publishes no model total. `GET /v1/models` returns only the models an operator declared in their own `AIGatewayRoute` resources ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints), 2026-09-02), and model names are supplied per route via `modelNameOverride` ([model name virtualization](https://aigateway.envoyproxy.io/docs/capabilities/traffic/model-name-virtualization)).", "provider_count": 19, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No fee of any kind: the project takes no payment and no tokens pass through vendor infrastructure, so BYOK carries no surcharge ([LICENSE](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE); [upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth), 2026-09-02).", "seat_fee_usd_month": null, "free_tier": "The whole project is free: Apache-2.0 licence, Helm charts on Docker Hub (`oci://docker.io/envoyproxy/ai-gateway-helm`), a CLI image (`envoyproxy/ai-gateway-cli`) and per-release CLI binaries, with no account, plan or quota. Your costs are the Kubernetes cluster and the upstream provider tokens.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "Self-hosting is the only mode and carries no licence fee: `helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm` followed by `helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm` ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation)). Real cost drivers are the Envoy Gateway control plane and Envoy data plane it requires, the AI Gateway controller (documented starting requests 100m CPU / 128Mi, limits 500m / 256Mi, HPA example min 2 max 10 replicas at 70% CPU) and a Redis instance if you enable token rate limiting or quotas ([scaling](https://aigateway.envoyproxy.io/docs/capabilities/scaling); [rate limiting](https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting), 2026-09-02).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "Configuration is portable YAML: `AIGatewayRoute`, `AIServiceBackend`, `BackendSecurityPolicy`, `GatewayConfig`, `MCPRoute` and `QuotaPolicy` CRDs, with the same schema accepted by `aigw run` as a local config file ([resources](https://aigateway.envoyproxy.io/docs/concepts/resources); [aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun)). Telemetry leaves over Prometheus and OTLP ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing)).", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Completions", "OpenAI Embeddings", "OpenAI Images", "OpenAI Audio (transcriptions/translations)", "OpenAI Responses", "OpenAI Models", "Anthropic Messages", "Cohere v2 Rerank", "MCP (streamable HTTP)" ], "sdk_note": "There is no vendor SDK to install: you point an existing OpenAI-compatible client at the Gateway URL. The standalone `aigw run` mode deliberately reads the OpenAI SDK's own environment variables (`OPENAI_API_KEY`, `OPENAI_BASE_URL`, `AZURE_OPENAI_ENDPOINT`, `OPENAI_API_VERSION`, `OPENAI_ORG_ID`, `OPENAI_PROJECT_ID`) so existing tooling works unchanged ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": 2, "latency_source_note": "The project itself publishes no data-plane overhead number. The only figure attached to Envoy AI Gateway is a third-party one: Tetrate's benchmark index reports that \"an independent benchmark by Broadcom's VMware Cloud Foundation team measured roughly 2 milliseconds of gateway overhead - about 0.01% of end-to-end latency - under sustained enterprise LLM load\", flat at peak saturation, alongside average TTFT of 0.103 s at 190 concurrent users and saturation at 224 concurrent users on a four-H100 cluster ([Tetrate benchmark index](https://tetrate.io/learn/ai/ai-gateway-benchmarks), last updated July 2026). Broadcom's own write-up of that validation describes the methodology - 124 GB high-cardinality dataset, 20,000+ sessions, Markov-modulated Poisson traffic on VMware vSphere Kubernetes Service - but states no millisecond figure ([VMware Cloud Foundation blog](https://blogs.vmware.com/cloud-foundation/2026/07/02/beyond-benchmarks-engineering-a-science-grounded-validation-for-the-envoy-ai-gateway/), 2026-07-02), so the 2 ms number is Tetrate's summary of it and Tetrate co-maintains the project. The project's own published benchmark is control-plane only: 2,000 `AIGatewayRoute` objects served with a consistent ~5 s route-readiness delay caused by the 5-second config-watcher poll, linear CPU and memory growth and zero routing failures, after raising the extension-server gRPC message size from 4 MB to 25 MB ([control-plane scaling](https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling), 2026-09-02). Treat all of these as vendor or vendor-adjacent claims, not as independent gateway-overhead measurements.", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "n.a. as a vendor feature: with self-hosting the operator picks the cluster region and the upstream endpoints, and no EU-residency programme is published ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [installation](https://aigateway.envoyproxy.io/docs/getting-started/installation)).", "zdr": null, "zdr_note": "Zero data retention is not a policy here but a property of the deployment model: self-hosted only, with no vendor endpoint or vendor storage in the request path ([system architecture](https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture), 2026-09-02).", "air_gapped": null, "best_for": "Platform teams already running Envoy Gateway or Gateway API on Kubernetes who want provider-agnostic LLM and MCP routing, cloud-native upstream credential handling and OpenTelemetry GenAI telemetry as ordinary infrastructure, with a stable CRD API and no vendor in the request path.", "strengths": [ "Apache-2.0, self-host-only, with no vendor in the request path and no billing relationship at all", "Stable v1beta1 control-plane CRDs with an explicit never-break promise and documented migration paths", "Broad endpoint coverage across 19 provider configurations: chat, completions, embeddings, images, audio, Responses, Cohere rerank and native Anthropic messages, with cross-provider translation", "Real MCP gateway: server multiplexing, tool filtering, OAuth 2.0 + PKCE and CEL authorization, plus MCP spans and metrics", "Cloud-native upstream auth - EKS Pod Identity/IRSA, Entra ID, GCP workload identity federation - minting short-lived credentials per request", "OpenTelemetry GenAI metrics and OpenInference tracing with a documented Arize Phoenix evaluation path, all into infrastructure you own", "Multi-vendor maintainer base (Tetrate, Bloomberg, Tencent, Netflix, Nutanix) on the CNCF Envoy foundation" ], "weaknesses": [ "No content guardrails whatsoever - no PII, moderation, injection or custom evaluator surface anywhere in the docs", "No gateway-side caching: prompt caching is passthrough of provider cache_control breakpoints only", "Envoy Gateway plus Kubernetes v1.32+ is a hard prerequisite, and token limits additionally require Redis", "No health checks, circuit breaking or multi-region failover documented for AI backends", "QuotaPolicy is v1alpha1-only, outside the stability guarantee, and its serviceQuota field is accepted but not enforced end-to-end", "No prompt management, cost dashboard, virtual keys for downstream callers or vendor SLA", "No Terraform, Vercel AI SDK or Python-framework integration documented; ecosystem glue is left to you", "The only data-plane overhead figure (~2 ms) comes from a maintainer-published summary of a third-party benchmark, not from a neutral head-to-head" ], "avoid_if": "You want content guardrails, PII redaction, a semantic cache, prompt management, cost dashboards or a hosted option out of the box - none of those exist here - or you do not run Kubernetes and do not want Envoy Gateway as a hard prerequisite.", "reviews": [ { "url": "https://blogs.vmware.com/cloud-foundation/2026/07/02/beyond-benchmarks-engineering-a-science-grounded-validation-for-the-envoy-ai-gateway/", "why": "Broadcom's VMware Cloud Foundation team documents a production-like validation of Envoy AI Gateway on vSphere Kubernetes Service - 124 GB high-cardinality dataset, 20,000+ agentic sessions, Markov-modulated Poisson traffic, real GPU inference - and reports the 224-concurrent-user saturation point and the streaming-tax problem, though it states no millisecond overhead figure.", "date": "2026-07-02", "type": "written", "title": "Beyond Benchmarks: Engineering a Science-Grounded Validation for the Envoy AI Gateway", "publisher": "VMware Cloud Foundation Blog (Broadcom)" }, { "url": "https://www.developersdigest.tech/blog/envoy-ai-gateway-llm-production-routing", "why": "Third-party analysis of the 1.0 release that weighs the gateway against direct SDK calls, LiteLLM, Portkey and OpenRouter and discusses fallback limits and rollout risk; useful as outside framing, though it contains no hands-on testing or measurements.", "date": "2026-06-23", "type": "written", "title": "Envoy AI Gateway 1.0 Makes LLM Routing an Infrastructure Concern", "publisher": "Developers Digest" }, { "url": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "why": "Maintainer-run benchmark index - read as vendor material, not review. It is the only page that puts a number (~2 ms) on data-plane overhead, aggregates the MCP tool-call and 2,000-route control-plane benchmarks, and is candid that vendor headline numbers are condition-specific.", "date": "2026-07-01", "type": "written", "title": "Envoy AI Gateway & Tetrate Agent Router Performance Benchmarks (2026)", "publisher": "Tetrate" } ], "sources": { "name": "https://aigateway.envoyproxy.io/", "cap_mcp": "https://aigateway.envoyproxy.io/docs/capabilities/mcp/", "fit_mcp": "https://aigateway.envoyproxy.io/docs/capabilities/mcp/", "license": "https://api.github.com/repos/envoyproxy/ai-gateway", "category": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md", "sdk_note": "https://aigateway.envoyproxy.io/docs/cli/aigwrun", "url_docs": "https://aigateway.envoyproxy.io/docs/", "url_home": "https://aigateway.envoyproxy.io/", "zdr_note": "https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture", "byok_mode": "https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth", "deployment": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "eval_hooks": "https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing", "url_github": "https://api.github.com/repos/envoyproxy/ai-gateway", "zdr_status": "https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture", "cap_budgets": "https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy", "log_content": "https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing", "log_opt_out": "https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing", "api_surfaces": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "cap_failover": "https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback", "github_stars": "https://github.com/envoyproxy/ai-gateway", "product_fate": "https://aigateway.envoyproxy.io/release-notes/v1.0", "regions_note": "https://aigateway.envoyproxy.io/docs/getting-started/prerequisites", "byok_fee_note": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE", "cache_support": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/prompt-caching", "fit_cloud_iam": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers", "pricing_model": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE", "url_changelog": "https://aigateway.envoyproxy.io/release-notes/", "vendor_status": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md", "byok_supported": "https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth", "certifications": "https://aigateway.envoyproxy.io/docs/capabilities/security/", "fit_eval_shape": "https://aigateway.envoyproxy.io/docs/cli/aigwrun", "fit_helm_chart": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "fit_kubernetes": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "fit_model_gate": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers", "fit_prod_shape": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "latency_claims": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "provider_count": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers", "cap_rate_limits": "https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting", "fit_credit_card": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "logs_by_default": "https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics", "tracing_support": "https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing", "deployment_modes": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "model_count_note": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "token_markup_pct": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE", "cap_observability": "https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics", "fit_openai_dropin": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "fit_vercel_ai_sdk": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "last_release_date": "https://api.github.com/repos/envoyproxy/ai-gateway/releases/latest", "openai_compatible": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "streaming_support": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "throughput_claims": "https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling", "architecture_class": "https://api.github.com/repos/envoyproxy/ai-gateway", "cap_load_balancing": "https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback", "guard_model_policy": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints", "provider_count_low": "https://aigateway.envoyproxy.io/blog/v1.0-release-announcement", "guardrails_location": "https://aigateway.envoyproxy.io/docs/capabilities/security/", "latency_overhead_ms": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "latency_source_note": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "provider_count_high": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers", "requires_vendor_sdk": "https://aigateway.envoyproxy.io/docs/cli/aigwrun", "self_host_cost_note": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "fit_gateway_platform": "https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md", "fit_quickstart_steps": "https://aigateway.envoyproxy.io/docs/getting-started/", "self_host_data_plane": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "fit_self_host_install": "https://aigateway.envoyproxy.io/docs/getting-started/installation", "cap_conditional_routing": "https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints" }, "verified_at": { "name": "2026-09-02", "cap_mcp": "2026-09-02", "fit_mcp": "2026-09-02", "license": "2026-09-02", "category": "2026-09-02", "sdk_note": "2026-09-02", "url_docs": "2026-09-02", "url_home": "2026-09-02", "zdr_note": "2026-09-02", "byok_mode": "2026-09-02", "deployment": "2026-09-02", "eval_hooks": "2026-09-02", "url_github": "2026-09-02", "zdr_status": "2026-09-02", "cap_budgets": "2026-09-02", "log_content": "2026-09-02", "log_opt_out": "2026-09-02", "api_surfaces": "2026-09-02", "cap_failover": "2026-09-02", "github_stars": "2026-09-17", "product_fate": "2026-09-02", "regions_note": "2026-09-02", "byok_fee_note": "2026-09-02", "cache_support": "2026-09-02", "fit_cloud_iam": "2026-09-02", "pricing_model": "2026-09-02", "url_changelog": "2026-09-02", "vendor_status": "2026-09-02", "byok_supported": "2026-09-02", "certifications": "2026-09-02", "fit_eval_shape": "2026-09-02", "fit_helm_chart": "2026-09-02", "fit_kubernetes": "2026-09-02", "fit_model_gate": "2026-09-02", "fit_prod_shape": "2026-09-02", "latency_claims": "2026-09-02", "provider_count": "2026-09-02", "cap_rate_limits": "2026-09-02", "fit_credit_card": "2026-09-02", "logs_by_default": "2026-09-02", "tracing_support": "2026-09-02", "deployment_modes": "2026-09-02", "model_count_note": "2026-09-02", "token_markup_pct": "2026-09-02", "cap_observability": "2026-09-02", "fit_openai_dropin": "2026-09-02", "fit_vercel_ai_sdk": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "throughput_claims": "2026-09-02", "architecture_class": "2026-09-02", "cap_load_balancing": "2026-09-02", "guard_model_policy": "2026-09-02", "provider_count_low": "2026-09-02", "guardrails_location": "2026-09-02", "latency_overhead_ms": "2026-09-02", "latency_source_note": "2026-09-02", "provider_count_high": "2026-09-02", "requires_vendor_sdk": "2026-09-02", "self_host_cost_note": "2026-09-02", "fit_gateway_platform": "2026-09-02", "fit_quickstart_steps": "2026-09-02", "self_host_data_plane": "2026-09-02", "fit_self_host_install": "2026-09-02", "cap_conditional_routing": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-17T18:50:21.725425+00:00", "vendor_status": "foundation", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed and freshly stable. v1.0.0 shipped 23 June 2026 as the first GA release with a committed-stable `v1beta1` control-plane API; v1.1.0 followed on 21 August 2026, and the repository was last pushed 2 September 2026 with 286 open issues and 1,987 stars ([v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0); [releases API](https://api.github.com/repos/envoyproxy/ai-gateway/releases/latest); [GitHub API](https://api.github.com/repos/envoyproxy/ai-gateway), 2026-09-02). Cadence tracks Envoy Gateway - \"approximately every 2-3 months\" - with end-of-life two releases after a version ships, and the community meets weekly on the Envoy Slack ([RELEASES.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/RELEASES.md); [README](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md)). Governance is multi-vendor: maintainers from Tetrate, Bloomberg, Tencent, Netflix and Nutanix, several of whom also maintain Envoy Proxy, Envoy Gateway, KServe or sit on Kubeflow's steering committee ([MAINTAINERS.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/MAINTAINERS.md); [1.0 announcement](https://aigateway.envoyproxy.io/blog/v1.0-release-announcement)). Key risk to watch: `QuotaPolicy` is still `v1alpha1`-only and therefore outside the stability guarantee, with its `serviceQuota` field \"accepted but not yet enforced end-to-end\" ([RELEASES.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/RELEASES.md)).", "last_release_date": "2026-08-21", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "v1.1.0, published 2026-08-21T18:23:16Z, not a prerelease ([releases API](https://api.github.com/repos/envoyproxy/ai-gateway/releases/latest)). It is the first minor release on the stable 1.x API and added `/tokenize` across providers, `credentialOverride` for per-request credentials, `GatewayConfig.spec.forwardProxy` for HTTP CONNECT egress, `streamIdleTimeout`, MCP `hostnames` and CEL `backendSelector`, the `gen_ai` tracing convention plus a Grafana dashboard, JSON controller logs, and Helm PDB/topology-spread support ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "prompt_transits_vendor": "no", "prompt_path_note": "Prompts never reach a project-operated service. Traffic terminates in Envoy proxies and an ext_proc container inside your own cluster, and the control plane only reconciles CRDs and pushes xDS config ([system architecture](https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture); [data plane](https://aigateway.envoyproxy.io/docs/concepts/architecture/data-plane), 2026-09-02). The only third parties on the path are the upstream providers you configure.", "logs_by_default": "metadata_only", "logs_note": "Out of the box you get Prometheus metrics following OpenTelemetry GenAI semantic conventions - `gen_ai.client.token.usage`, `gen_ai.server.request.duration`, `gen_ai.server.time_to_first_token`, `gen_ai.server.time_per_output_token` - and Envoy access logs carrying token-cost dynamic metadata; that is metadata, not content ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [access logs](https://aigateway.envoyproxy.io/docs/capabilities/observability/accesslogs), 2026-09-02). The load-bearing caveat is one level up: as soon as you set `OTEL_EXPORTER_OTLP_ENDPOINT`, tracing defaults to OpenInference conventions **with full request and response content included by default**, and every `OPENINFERENCE_HIDE_*` redaction flag defaults to false ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing)).", "retention_days": null, "retention_note": "Retention is entirely yours - the project stores nothing and ships no datastore for request data. Metrics, traces and access logs land in the Prometheus, OTLP collector and log sinks you operate ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [access logs](https://aigateway.envoyproxy.io/docs/capabilities/observability/accesslogs)).", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "No vendor ever receives prompts, so there is nothing to train on; no training policy is published, and none would be meaningful for self-hosted Apache-2.0 software ([system architecture](https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture)).", "subprocessors_url": null, "regions_note": "Region is whatever your Kubernetes cluster and chosen upstreams are - there is no vendor region list. Kubernetes v1.32+ with Envoy Gateway v1.7.0+ is the only placement requirement stated ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [compatibility](https://aigateway.envoyproxy.io/docs/compatibility), 2026-09-02).", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "No SOC 2 statement on any page fetched; the project is Apache-2.0 software you run yourself and publishes no compliance attestations ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/), [SECURITY.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/SECURITY.md)).", "status": "not_published" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "No DPA or controller/processor language exists because the project never receives your data; compliance sits with the operator.", "status": "not_published" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "detail": "No BAA is possible - there is no vendor service to sign one with.", "status": "not_published" } }, "guardrails_location": "none", "guardrails_note": "There is no guardrail feature. The security documentation covers client-to-gateway access control by delegating to Envoy Gateway SecurityPolicy - JWT validation, JWT claim-based authorization, mTLS, external authorization, OIDC, basic auth, API key, IP allow/deny - and gateway-to-provider credential handling; content inspection is absent ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/); [upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth), 2026-09-02). No page in the site map carries the word guardrail, and neither the [v1.0](https://aigateway.envoyproxy.io/release-notes/v1.0) nor [v1.1](https://aigateway.envoyproxy.io/release-notes/v1.1) release notes introduce one.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "oss_no_vendor_price", "pricing_model_note": "Apache-2.0 open source with no vendor-priced tier at all: the site has no pricing page and the only artifacts are Helm charts, container images and CLI binaries ([LICENSE](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE); [installation](https://aigateway.envoyproxy.io/docs/getting-started/installation); [CLI install](https://aigateway.envoyproxy.io/docs/cli/aigwinstall), 2026-09-02). A managed product built on the same data plane is sold separately by Tetrate as Tetrate Agent Router Service, which appears in this project only as one more upstream provider you can route to ([supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers)) - it is a different product and is not scored here.", "min_commitment": "None. Apache-2.0 licence, no contract, no seat count and no vendor relationship ([LICENSE](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/LICENSE)).", "overage_note": "n.a. - there is no vendor meter to overrun. Nothing on the [installation](https://aigateway.envoyproxy.io/docs/getting-started/installation) or [release notes](https://aigateway.envoyproxy.io/release-notes/) pages describes paid units; quota enforcement is something you configure for your own tenants with `QuotaPolicy` ([quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy)).", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "No gateway-side cache and therefore no cache pricing. What exists is provider-agnostic passthrough of Anthropic-style `cache_control: {\"type\": \"ephemeral\"}` breakpoints - native on Anthropic, translated for Claude on GCP Vertex AI and AWS Bedrock, minimum 1,024 cacheable tokens and at most 4 breakpoints, with `prompt_tokens_details.cached_tokens` returned in usage. Any discount is the provider's, not the gateway's ([prompt caching](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/prompt-caching), 2026-09-02).", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not documented as a product feature; you would alert on the exported Prometheus token metrics yourself ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics)).", "tier": null, "supported": null }, "virtual_keys": { "note": "No virtual-key concept for downstream callers on any page fetched; client authentication is delegated to Envoy Gateway SecurityPolicy (JWT, API key, basic auth, mTLS, OIDC, external authorization) ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/)).", "tier": null, "supported": null }, "rate_limiting": { "note": "Token-based global rate limiting through Envoy Gateway's `BackendTrafficPolicy`, keyed on `x-ai-eg-model` plus client headers, with CEL cost expressions; requires a Redis deployment ([usage-based rate limiting](https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting)).", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not documented on any page fetched ([quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy); [metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics)).", "tier": null, "supported": null }, "model_allowlists": { "note": "Effectively by routing: only models declared in an `AIGatewayRoute` rule are reachable and `GET /v1/models` returns exactly that set ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints)).", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "`QuotaPolicy` per-model token budgets with `clientSelectors` bucket rules keyed on request headers, so a per-tenant or per-key budget is expressible; shadow mode evaluates rules without enforcing ([quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy)).", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Same mechanism - bucket rules on a tenant header carve per-team budgets; there is no team object in the product ([quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy)).", "tier": null, "supported": true } }, "cost_attribution": "Per-request token and cost metadata rather than a billing console: `llmRequestCosts` on `AIGatewayRoute` exposes InputToken, CachedInputToken, OutputToken, TotalToken and CEL-computed costs as Envoy dynamic metadata under `io.envoy.ai_gateway`, surfaced in access logs as `gen_ai.*` fields; OpenTelemetry GenAI metrics carry provider and model attributes and can be labelled with arbitrary request headers (for example a tenant header) via `controller.metricsRequestHeaderAttributes`. Per-team or per-key rollups are yours to build in Prometheus or your log store.", "cost_export": "No cost CSV, invoice or billing API - the project has no billing. Cost signals leave through Prometheus metrics, OTLP traces and Envoy access logs that you own ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [access logs](https://aigateway.envoyproxy.io/docs/capabilities/observability/accesslogs)).", "product_kind_note": "An infrastructure-layer AI gateway, not a platform: \"an open source project for using Envoy Gateway to handle request traffic from application clients to Generative AI services\", positioned explicitly as \"an additive layer designed to expand use cases for Envoy Proxy and Envoy Gateway without changing existing deployment or control patterns\" ([README](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md); [GOALS.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/GOALS.md), 2026-09-02). Its personas are platform engineers and infrastructure administrators; there is no dashboard, no prompt studio and no evaluation product - control surfaces are Kubernetes CRDs and telemetry.", "deployment_modes": [ "self_host" ], "deployment_note": "Install order matters: Envoy Gateway first (`oci://docker.io/envoyproxy/gateway-helm` with the project's `envoy-gateway-values.yaml`, requiring v1.7.0+), then `ai-gateway-crds-helm`, then `ai-gateway-helm` into `envoy-ai-gateway-system`, waiting on `deployment/ai-gateway-controller` ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [installation](https://aigateway.envoyproxy.io/docs/getting-started/installation)). Kubernetes v1.32+ and Gateway API v1.5.x are required; the compatibility matrix pairs AI Gateway v1.0.x with Envoy Gateway v1.8.1+ and Envoy Proxy v1.38.x, and lists v0.5-v0.7 as still supported with earlier versions EOL ([compatibility](https://aigateway.envoyproxy.io/docs/compatibility), 2026-09-02). Token rate limiting and quotas additionally need Redis plus rate-limit configuration on the Envoy Gateway install ([usage-based rate limiting](https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting)); InferencePool needs the Gateway API Inference Extension CRDs ([InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "One OpenAI-compatible front door plus provider-native passthrough surfaces, all mounted on the same Gateway: OpenAI paths under `/`, Cohere under `/cohere`, Anthropic under `/anthropic`, with the prefixes overridable through the Helm `endpointConfig` value ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints), 2026-09-02). Requests are matched on the model name, which the ext_proc filter extracts from the body and re-emits as the `x-ai-eg-model` header; the same Gateway also serves `/mcp` for Model Context Protocol traffic ([MCP](https://aigateway.envoyproxy.io/docs/capabilities/mcp/)). v1.1 added a vLLM-compatible `/tokenize` endpoint across providers ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a. - no page fetched states a model count. Checked [supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers), [supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints), [home](https://aigateway.envoyproxy.io/) and the [v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0); models come from whichever upstream providers the operator configures with their own credentials.", "provider_count_low": 16, "provider_count_high": 19, "provider_count_source_note": "Range 16-19 depending on the page. The supported-providers table lists 19 rows - OpenAI, AWS Bedrock, Azure OpenAI, Google Gemini on AI Studio, Google Vertex AI, Anthropic on GCP Vertex AI, Groq, Grok, Together AI, Cohere, Mistral, DeepInfra, DeepSeek, Hunyuan, Tencent LLM Knowledge Engine, Tetrate Agent Router Service, SambaNova, Self-hosted-models and Anthropic ([supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers), 2026-09-02) - while the 1.0 announcement and the v1.0/v1.1 release-note headlines count \"16 AI providers\" and the repo README shows 16 provider logos ([1.0 announcement](https://aigateway.envoyproxy.io/blog/v1.0-release-announcement); [README](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md)). \"Support\" here means an API-schema mapping on `AIServiceBackend` plus an upstream-auth type on `BackendSecurityPolicy`, not a hosted model catalogue.", "first_party_vs_partner": "All third-party or operator-hosted: the project owns no models and no inference hardware. It routes to 18 named external providers plus self-hosted servers such as vLLM, and to Gateway API InferencePool endpoints you run yourself ([supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers); [InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support), 2026-09-02).", "custom_endpoint_support": "Yes: a `Self-hosted-models` row in the provider table maps any OpenAI-schema server (vLLM is the named example) onto an `AIServiceBackend` pointing at an Envoy Gateway `Backend` or Kubernetes Service, with optional API-key auth ([supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers); [connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers)). Self-hosted fleets can instead be fronted by a Gateway API InferencePool with an endpoint picker that routes on live KV-cache usage, queue depth and LoRA adapter state ([InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support)). Locally, `aigw run` auto-configures against any `OPENAI_BASE_URL`, with Ollama as the documented example ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02).", "byok_mode": "byok_only", "merchant_of_record": "The upstream AI provider. You supply provider credentials in Kubernetes Secrets and are billed directly by OpenAI, AWS, Azure, GCP and so on; the project never takes payment or proxies billing ([upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth); [connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers)).", "key_management_note": "Credentials are centralised in the control plane, away from applications. `BackendSecurityPolicy` takes an API key from a Kubernetes Secret (key name `apiKey`), or cloud identity: AWS Bedrock via OIDC to STS - including EKS Pod Identity and IRSA with just a region set - Azure OpenAI via Entra ID, GCP Vertex AI via workload identity federation and Google STS, each minting short-lived tokens per request; long-lived keys stay in Secrets under operator control, and the docs note secret updates are \"picked up automatically in a few seconds\" ([upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth); [connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers); [OpenAI guide](https://aigateway.envoyproxy.io/docs/getting-started/connect-providers/openai), 2026-09-02). v1.1 added `BackendSecurityPolicy.spec.credentialOverride` so a per-request credential can come from dynamic metadata or a header, which is how you do per-tenant keys ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1)).", "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` is listed \"Fully Supported\" with streaming, function/tool calling, JSON-schema structured output, audio and video inputs, the injected `x-ai-eg-model` routing header, token-usage and cost tracking, and fallback plus load balancing; the quickstart calls `$GATEWAY_URL/v1/chat/completions` ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage), 2026-09-02).", "api_anthropic_messages_note": "Yes, two ways: a native `POST /anthropic/v1/messages` surface listed \"Fully Supported\", and cross-provider translation of Anthropic Messages into OpenAI Chat Completions and into AWS Bedrock Converse/InvokeModel including streaming, tool use, reasoning blocks and images ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0), 2026-09-02). v1.1 added `/anthropic/v1/messages/count_tokens` and `/anthropic/v1/models` ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1)).", "api_openai_responses_note": "Yes: `POST /v1/responses` is listed \"Fully Supported\" with MCP tools, reasoning and multimodal input, and v1.1 added `/v1/responses/input_tokens` ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "api_embeddings_note": "Yes: `POST /v1/embeddings` is a supported endpoint with its own metrics surface, and embeddings translation is documented per provider in the model-virtualization table ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [model name virtualization](https://aigateway.envoyproxy.io/docs/capabilities/traffic/model-name-virtualization), 2026-09-02).", "api_images_note": "Yes: `POST /v1/images/generations` appears as a supported endpoint, and \"images\" is listed in the 1.0 endpoint coverage summary ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0), 2026-09-02).", "api_audio_note": "Yes: `POST /v1/audio/transcriptions` and `POST /v1/audio/translations` are supported endpoints, and audio is part of the documented 1.0 endpoint surface ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0), 2026-09-02).", "api_batch_note": "n.a. - no batch or async bulk endpoint appears on any page fetched: [supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints) enumerates chat, completions, embeddings, images, audio, responses, rerank, models and tokenize surfaces with no `/v1/batches`, and neither the [v1.0](https://aigateway.envoyproxy.io/release-notes/v1.0) nor [v1.1](https://aigateway.envoyproxy.io/release-notes/v1.1) release notes mention batch.", "deployment_modes_note": "Self-host only, in two shapes. The primary one is Kubernetes: two Helm charts (CRDs plus controller) on top of Envoy Gateway, driving Envoy proxies in your cluster ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation)). The second is a single-process local mode - `aigw run` runs the same configuration API with no Docker or Kubernetes on Linux and macOS, listening on `localhost:1975`, intended for testing config and for local development against provider-agnostic apps ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02). There is no vendor-hosted or hybrid-VPC offering from the project; the managed product on this data plane is Tetrate's separate Agent Router Service.", "byok_mode_note": "byok_only by necessity: there are no platform credits or vendor billing, so every upstream call uses credentials you supply through `BackendSecurityPolicy` - API keys from Kubernetes Secrets, or cloud identity for AWS, Azure and GCP ([upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth); [connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers), 2026-09-02).", "access_meta": { "qualified": [ "provider_count" ], "model_count_kind": null, "provider_count_kind": "counted_list" }, "rel_timeout_config": "config_file", "rel_timeout_note": "Config-file only, and split across two APIs. Request timeouts come from Envoy Gateway's `BackendTrafficPolicy` (the fallback example sets `timeout: 30s` with `perRetry.backOff.baseInterval 100ms` and `maxInterval 10s`) ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback)); v1.1 added an AI-specific `AIGatewayRouteRule.streamIdleTimeout` that triggers failover if it fires before the first token and returns 504 if it fires mid-stream ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02). No default value for either is stated on the pages fetched.", "rel_retry_config": "config_file", "rel_retry_note": "Retries are Envoy Gateway's, configured in YAML next to the AI routes: the documented example sets `numRetries: 5`, `numAttemptsPerPriority: 1`, exponential backoff `100ms`/`10s`, `retryOn.httpStatusCodes: [500]` and triggers `connect-failure` and `retriable-status-codes`, with `numAttemptsPerPriority` controlling how many tries each priority group gets before failing over ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback), 2026-09-02).", "rel_retry_default_note": "The numbers above are the values in the documentation's example manifest, not stated defaults; no page fetched publishes a default retry count or backoff for AI routes ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback), 2026-09-02).", "rel_fallback_config": "config_file", "rel_fallback_note": "Ordered priority groups: `backendRefs` in an `AIGatewayRoute` rule carry `priority: 0`, `priority: 1` and so on, and traffic moves to the next priority when the current one exhausts its attempts. Because the ext_proc filter chain is split into router-level and upstream-level filters, a failover to a different provider re-runs request translation and upstream auth for the new backend - which is what makes cross-provider fallback work rather than just cross-endpoint retry ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback); [data plane](https://aigateway.envoyproxy.io/docs/concepts/architecture/data-plane), 2026-09-02). Weighted fallback is not documented.", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": "n.a. - no active health check or circuit breaker for AI backends on any page fetched ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback), [system architecture](https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture), [data plane](https://aigateway.envoyproxy.io/docs/concepts/architecture/data-plane), [capabilities index](https://aigateway.envoyproxy.io/docs/)). Failure detection is reactive: retries and priority failover on connect failures and retriable status codes. The one exception is InferencePool, where an endpoint picker selects self-hosted endpoints from live metrics such as KV-cache usage and queue depth ([InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support), 2026-09-02).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "n.a. - no cross-region or multi-cluster failover construct is documented; the two-tier gateway pattern in the README is about tier-one ingress versus tier-two self-hosted model access, not geography ([README](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/README.md); [provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback), 2026-09-02).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Config-file. Multiple `backendRefs` at the same priority are load-balanced by Envoy, and Envoy Gateway's `BackendTrafficPolicy` governs the algorithm ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback)); for self-hosted fleets an InferencePool endpoint picker does inference-aware selection on real-time KV-cache usage, queued requests and LoRA adapter state ([InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support), 2026-09-02).", "architecture_class": "compiled_binary", "architecture_note": "Go control plane and Go ext_proc filter alongside the Envoy C++ data plane: repo language bytes are Go 7,634,320, MDX 1,427,966, CSS 44,361 and TypeScript 43,527, built against Go 1.26.4 ([GitHub API](https://api.github.com/repos/envoyproxy/ai-gateway); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02). Requests traverse Envoy Proxy plus an External Processor container that does model extraction, routing, upstream auth, request/response translation and token accounting, with a Rate Limit Service and Redis for token limits; the filter chain is deliberately split into router-level and upstream-level stages so a retry to another provider re-runs translation and auth. Dynamic Modules are noted as a possible future alternative to ext_proc ([system architecture](https://aigateway.envoyproxy.io/docs/concepts/architecture/system-architecture); [data plane](https://aigateway.envoyproxy.io/docs/concepts/architecture/data-plane)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Everything is published: Helm charts as OCI artifacts (`oci://docker.io/envoyproxy/ai-gateway-crds-helm` and `oci://docker.io/envoyproxy/ai-gateway-helm`, pinned in the docs to `--version v1.0.0`) ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation)); a CLI container image at `envoyproxy/ai-gateway-cli` and per-release `aigw` binaries for Linux and macOS on the GitHub releases page, or `go install ./cmd/aigw` from source ([CLI installation](https://aigateway.envoyproxy.io/docs/cli/aigwinstall), 2026-09-02); the v1.1.0 release itself is on GitHub ([releases API](https://api.github.com/repos/envoyproxy/ai-gateway/releases/latest)).", "latency_claims": [ { "url": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "unit": "ms", "scope": "gateway_overhead", "value": "~2", "metric": "data-plane gateway overhead (~0.01% of end-to-end latency)", "publisher": "third_party", "conditions": "Attributed to an independent Broadcom/VMware Cloud Foundation validation under sustained enterprise LLM load with real GPU inference; overhead reported flat at peak saturation. The millisecond figure appears only in Tetrate's summary page, and Tetrate co-maintains the project; Broadcom's own post states no ms value. Percentile not stated." }, { "url": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "unit": "ms", "scope": "gateway_overhead", "value": "160-390", "metric": "added latency on an MCP tool call vs direct unproxied call", "publisher": "vendor", "conditions": "December 2025 Go microbenchmark of a simple MCP echo tool, standalone single process, 100 key-derivation iterations for encryption setup; average difference against a competing implementation ~0.2 ms. Summarised by Tetrate; original benchmark post not fetched." }, { "url": "https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling", "unit": "s", "scope": "capacity", "value": "~5", "metric": "route readiness (AIGatewayRoute creation to serving traffic)", "publisher": "vendor", "conditions": "Project-run control-plane benchmark at 2,000 routes; the delay is the 5-second default poll interval of the ext_proc config watcher, not request-path latency. Mock cassette backend, no headerMutation." } ], "throughput_claims": [ { "url": "https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling", "unit": "AIGatewayRoutes", "scope": "capacity", "value": "2000", "metric": "control-plane configuration scale, all routes verified serving", "publisher": "vendor", "conditions": "Required raising the Envoy Gateway extension-manager gRPC message size to 25Mi and the controller `maxRecvMsgSize` to 26214400; runs excluded `headerMutation`; effective ceiling bounded by the ~1 MB Kubernetes Secret size limit for aggregated filter config." }, { "url": "https://tetrate.io/learn/ai/ai-gateway-benchmarks", "unit": "concurrent users", "scope": "capacity", "value": "224", "metric": "saturation point (TTFT climbs sharply)", "publisher": "third_party", "conditions": "Broadcom/VMware run as summarised by Tetrate; described as a GPU compute ceiling on a four-H100 test cluster rather than a gateway limit. Not an RPS figure." } ], "streaming_support": "yes", "streaming_note": "Yes - SSE streaming is listed as supported on chat completions and Anthropic messages, with translation preserved across providers including tool use and reasoning blocks ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [v1.0 release notes](https://aigateway.envoyproxy.io/release-notes/v1.0)). Two operational caveats the docs raise: Envoy Gateway's default 32 KB buffer limit \"is not enough for most AI model responses\", so the example manifest raises it to 50 MB via `ClientTrafficPolicy` ([basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage)); and `streamIdleTimeout` returns 504 if it fires mid-stream, while firing before the first token triggers failover instead ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "guard_pii": "not_documented", "guard_pii_note": "n.a. - no PII detection, masking or redaction of request content on any page fetched ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/); [header and body mutations](https://aigateway.envoyproxy.io/docs/capabilities/traffic/header-body-mutations); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1)). The nearest thing is trace-level redaction: `OPENINFERENCE_HIDE_INPUTS` / `HIDE_OUTPUTS` / `HIDE_EMBEDDINGS_TEXT` / `HIDE_EMBEDDINGS_VECTORS` suppress prompt content in spans, all defaulting to false ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing), 2026-09-02) - that protects your telemetry, not the model call.", "guard_injection": "not_documented", "guard_injection_note": "n.a. - no prompt-injection or jailbreak detection documented ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/); [capabilities index](https://aigateway.envoyproxy.io/docs/); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "guard_content": "not_documented", "guard_content_note": "n.a. - no moderation or toxicity filtering; nothing on the [security](https://aigateway.envoyproxy.io/docs/capabilities/security/) page or in the [v1.0](https://aigateway.envoyproxy.io/release-notes/v1.0) and [v1.1](https://aigateway.envoyproxy.io/release-notes/v1.1) release notes inspects prompt or completion content.", "guard_custom": "not_documented", "guard_custom_note": "n.a. as a policy-driven content check. You can rewrite headers and bodies (`headerMutation`, `bodyMutation` on `AIServiceBackend`) and enforce arbitrary CEL expressions for MCP tool authorization ([header and body mutations](https://aigateway.envoyproxy.io/docs/capabilities/traffic/header-body-mutations); [MCP](https://aigateway.envoyproxy.io/docs/capabilities/mcp/)), and Envoy Gateway's external-authorization hook can front the gateway ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/)) - but no custom guardrail evaluator is documented.", "guard_model_policy": "yes", "guard_model_policy_note": "Yes, by construction: only models declared in an `AIGatewayRoute` rule are routable, `GET /v1/models` returns exactly the declared set, and MCP tool exposure is filtered with `toolSelector.include` / `includeRegex` plus a CEL `backendSelector` that defaults to Deny ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [MCP](https://aigateway.envoyproxy.io/docs/capabilities/mcp/); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "guard_execution_location": "not_documented", "guard_third_party": null, "guard_failure_mode": "not_documented", "guard_failure_mode_note": "n.a. - no guardrail exists, so no fail-open/fail-closed behaviour is stated ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/)). For the request path proper, the documented failure behaviour is Envoy retry and priority failover plus 429 on quota exhaustion ([provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback); [quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy)).", "log_content": "configurable", "log_content_note": "Content capture is a switch, and the default depends on which semantic convention you pick. OpenInference (the default) records inputs and outputs unless you set `OPENINFERENCE_HIDE_INPUTS`, `OPENINFERENCE_HIDE_OUTPUTS`, `OPENINFERENCE_HIDE_EMBEDDINGS_TEXT` or `OPENINFERENCE_HIDE_EMBEDDINGS_VECTORS`; switching to `AI_GATEWAY_TRACING_SEMCONV=gen_ai` makes content opt-in via `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing), 2026-09-02).", "log_opt_out": "yes", "log_opt_out_note": "Yes, and it is total: with no OTLP endpoint configured nothing is exported at all, and per-field redaction flags plus the `gen_ai` semantic-convention mode let you keep spans without message content ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing), 2026-09-02).", "tracing_support": "otel", "tracing_note": "Full OpenTelemetry: set `OTEL_EXPORTER_OTLP_ENDPOINT` (globally or per-Gateway through `GatewayConfig.spec.extProc.kubernetes.env`) and the ext_proc emits spans, defaulting to **OpenInference** conventions with `AI_GATEWAY_TRACING_SEMCONV=gen_ai` as the alternative; MCP `CallTool` and `ListTools` operations get spans too, and a `session.id` header mapping groups multi-turn conversations. Arize Phoenix is the documented consumer, giving LLM-as-judge evaluation over production spans ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [gateway config](https://aigateway.envoyproxy.io/docs/capabilities/gateway-config), 2026-09-02).", "export_destinations": [ "OpenTelemetry (OTLP)", "Prometheus", "Envoy access logs", "Arize Phoenix", "Grafana" ], "export_destinations_note": "OTLP for traces to any collector ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing)); Prometheus scrape for GenAI token and latency metrics, with custom labels lifted from request headers ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics)); Envoy access logs carrying `gen_ai.*` token costs and MCP fields such as `mcp_tool_name` and `mcp_backend`, configured on `EnvoyProxy.spec.telemetry.accessLog` ([access logs](https://aigateway.envoyproxy.io/docs/capabilities/observability/accesslogs)). Named downstreams in the docs are Arize Phoenix via its Helm chart and a bundled Grafana dashboard at `examples/monitoring/grafana-dashboard.json` ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "feedback_capture": "no", "feedback_capture_note": "n.a. - no thumbs-up/score ingestion API or feedback header on any page fetched ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [API reference](https://aigateway.envoyproxy.io/docs/api/), 2026-09-02).", "eval_hooks": "partial", "eval_hooks_note": "Partial and second-hand: the gateway emits OpenInference-compatible spans with full message content, and the docs walk you through deploying Arize Phoenix so \"LLM-as-judge\" evaluations can run against production spans - the evaluation itself belongs to Phoenix, not to the gateway, which ships no dataset, scorer or experiment surface ([tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing), 2026-09-02).", "observability_requires_saas": "No. Metrics are scraped from your own Prometheus and traces go to any OTLP collector you run; there is no vendor dashboard, account or telemetry egress in the product ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing), 2026-09-02).", "log_retention_note": "n.a. - no vendor-side log retention exists. The only project-side storage described anywhere is the Kubernetes Secret the controller writes to hand filter configuration to ext_proc, which is configuration rather than traffic ([control-plane scaling](https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling)).", "production_meta": { "sources": {}, "qualified": [ "latency_overhead_ms", "guard_pii", "guard_injection", "guard_content", "guard_custom" ], "latency_prose": "No project-published data-plane overhead figure exists. The ~2 ms overhead claim is Tetrate's summary of a Broadcom/VMware validation ([Tetrate benchmark index](https://tetrate.io/learn/ai/ai-gateway-benchmarks)); Broadcom's own post gives methodology but no ms number ([VMware CF blog](https://blogs.vmware.com/cloud-foundation/2026/07/02/beyond-benchmarks-engineering-a-science-grounded-validation-for-the-envoy-ai-gateway/)).", "multi_surface": {}, "throughput_prose": "No requests-per-second claim exists for this gateway. Published figures are configuration scale (2,000 routes, [control-plane scaling](https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling)) and a 224-concurrent-user GPU saturation point ([Tetrate benchmark index](https://tetrate.io/learn/ai/ai-gateway-benchmarks)).", "benchmark_independence": "The 2 ms and 224-user figures originate with Broadcom's VMware Cloud Foundation team (third party) but are quoted from a Tetrate page, and Tetrate co-created and maintains the project; the 2,000-route and MCP figures are project or maintainer-published. None is a neutral head-to-head against other gateways with policies enabled." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Reliability is inherited rather than reinvented: this is Envoy Proxy's retry, timeout, priority and load-balancing machinery driven by Gateway API resources, with AI-specific additions where translation matters (per-priority attempt counts, upstream-level filters that re-auth on failover, `streamIdleTimeout` failover before first token, 429 on token quota exhaustion). The gaps are health checking, circuit breaking and multi-region failover, none of which appear in the AI Gateway docs. Operationally the controller supports leader election plus a horizontally scalable read-only extension server, with an HPA example at 70% CPU, min 2 / max 10 replicas ([scaling](https://aigateway.envoyproxy.io/docs/capabilities/scaling); [provider fallback](https://aigateway.envoyproxy.io/docs/capabilities/traffic/provider-fallback); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "guard_notes": "The honest headline: **Envoy AI Gateway has no content guardrails at all** - no PII, moderation, injection or custom-evaluator surface appears anywhere on the docs site, and no page in the 469-URL site map mentions guardrails ([security](https://aigateway.envoyproxy.io/docs/capabilities/security/); [capabilities](https://aigateway.envoyproxy.io/docs/), 2026-09-02). What it does have is infrastructure-grade control: Envoy Gateway SecurityPolicy for client auth (JWT claims, mTLS, OIDC, external authz, API key, IP lists), centralised upstream credentials with short-lived tokens from AWS STS, Entra ID and Google STS, route-scoped model allowlisting, CEL-gated MCP tool authorization with OAuth 2.0 authorization-code + PKCE, and token quotas. If you need content inspection you must put it in front of, or behind, this gateway.", "fit_agent_features_note": "Agent-shaped features are the project's growth area: the Responses API surface supports MCP tools, reasoning and multimodal input ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints)), and the MCP gateway multiplexes many MCP servers behind one `/mcp` endpoint with tool-name prefixing (`github__issue_read`), include/regex tool filtering, OAuth 2.0 authorization-code + PKCE with JWT scope and CEL claim checks, upstream API-key injection, header forwarding and streamable HTTP transport per the June 2025 MCP spec, all traced and metered ([MCP](https://aigateway.envoyproxy.io/docs/capabilities/mcp/), 2026-09-02). v1.1 added `MCPRoute.spec.hostnames` (max 16) and a CEL `backendSelector` defaulting to Deny ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1)). Locally, `aigw run --mcp-config` accepts the same `mcpServers` JSON that Claude Desktop, Cursor and VS Code use ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun)). There is no agent loop, no tool executor and no prompt/agent registry - tool execution stays with the client.", "fit_byok_required": "yes", "fit_byok_required_note": "Yes for any real traffic: each `AIServiceBackend` needs a `BackendSecurityPolicy` with your provider credential ([connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers)). The exception is evaluation - the quickstart's `basic.yaml` routes to a mock backend (`some-cool-self-hosted-model`) so you can make a first call with no provider account at all ([basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage), 2026-09-02).", "fit_client_sdk_langs": null, "fit_client_sdks_note": "No client library is published or required. The documented pattern is pointing an OpenAI-compatible client at the gateway; `aigw run` reads the OpenAI SDK's environment variables so existing code needs no change, and the InferencePool page cites \"seamless integration with OpenAI SDKs\" ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun); [InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support), 2026-09-02). No per-language SDK list, snippet set or package name appears on the pages fetched, so `fit_client_sdk_langs` is left empty rather than inferred from the curl examples.", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Native cloud identity on all three hyperscalers: AWS Bedrock through the default credential chain including EKS Pod Identity and IRSA (region only, no key material) or OIDC-to-STS temporary credentials; Azure OpenAI through Entra ID short-lived tokens; GCP Vertex AI through Application Default Credentials, service-account keys or Workload Identity Federation with Google STS ([connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers); [upstream auth](https://aigateway.envoyproxy.io/docs/capabilities/security/upstream-auth); [supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers), 2026-09-02).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. - no Cloudflare Workers or edge-runtime story; the data plane is Envoy in your cluster or the `aigw` binary on Linux/macOS ([data plane](https://aigateway.envoyproxy.io/docs/concepts/architecture/data-plane); [aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02). Cloudflare is not even listed as an upstream provider ([supported providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers)).", "fit_credit_card": "not_required", "fit_credit_card_note": "Not required - there is no account, sign-up or billing relationship anywhere in the install path; you need a Kubernetes cluster or a local machine plus, for real calls, your own provider key ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation); [aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02).", "fit_ecosystem_notes": "The ecosystem is Kubernetes-native and standards-first rather than SaaS-integration-first: Gateway API v1.5.x, Gateway API Inference Extension v1.0.2, Envoy Gateway v1.8.1 and Envoy Proxy v1.38.1 as of v1.1.0, with the MCP Go SDK v1.7.0 for MCP support ([v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1); [compatibility](https://aigateway.envoyproxy.io/docs/compatibility)). Observability plugs into Prometheus, OTLP collectors, Arize Phoenix and a bundled Grafana dashboard ([metrics](https://aigateway.envoyproxy.io/docs/capabilities/observability/metrics); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing)). Named adopters on the home page are Alan by Comma Soft, Bloomberg, LY Corporation, National Research Platform, Nutanix, Paper Compute Co., Simplifai, Stacklok, Tencent Cloud, Tetrate and Unwrap ([home](https://aigateway.envoyproxy.io/), 2026-09-02). What is absent is application-framework glue: no LangChain, LlamaIndex, Vercel AI SDK or Terraform documentation exists on the site.", "fit_eval_prereqs": "Two paths. Local: the `aigw` binary or `envoyproxy/ai-gateway-cli` image on Linux or macOS, one environment variable (`OPENAI_API_KEY`, or `OPENAI_BASE_URL` for Ollama and other OpenAI-compatible servers), listening on `localhost:1975` ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun); [CLI installation](https://aigateway.envoyproxy.io/docs/cli/aigwinstall)). Kubernetes: `kubectl`, `helm` and `curl`, a cluster on v1.32+, and Envoy Gateway v1.7.0+ installed first with the project's values file; the quickstart then works against a mock backend with no provider key ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage), 2026-09-02).", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes. Locally: `OPENAI_API_KEY=sk-your-key aigw run`, then `curl -H \"Content-Type: application/json\" -XPOST http://localhost:1975/v1/chat/completions -d '{\"model\": \"qwen2.5:0.5b\", \"messages\": [{\"role\": \"user\", \"content\": \"Say this is a test!\"}]}'` ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun)). On Kubernetes: `kubectl apply -f .../examples/basic/basic.yaml`, set `$GATEWAY_URL`, then curl `$GATEWAY_URL/v1/chat/completions` ([basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage), 2026-09-02).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It is the gateway - but uniquely among these entries it is not a standalone one: Envoy Gateway is a hard prerequisite (v1.7.0+ for install, v1.8.1+ in the v1.0.x compatibility matrix) and AI Gateway is an extension server plus ext_proc filter on top of it, described as \"an additive layer\" ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [compatibility](https://aigateway.envoyproxy.io/docs/compatibility); [GOALS.md](https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/GOALS.md), 2026-09-02).", "fit_helm_chart": "oci://docker.io/envoyproxy/ai-gateway-helm plus oci://docker.io/envoyproxy/ai-gateway-crds-helm (Envoy Gateway's oci://docker.io/envoyproxy/gateway-helm required first)", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Kubernetes is the primary target, not a deployment option: two official OCI Helm charts (CRDs and controller), a documented CRD-ownership migration with `--take-ownership`, controller leader election with a horizontally scalable read-only extension server, an HPA example (70% CPU, min 2 / max 10), and v1.1 Helm additions for PodDisruptionBudget, topology spread constraints, pod labels and a restricted controller security context ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation); [scaling](https://aigateway.envoyproxy.io/docs/capabilities/scaling); [v1.1 release notes](https://aigateway.envoyproxy.io/release-notes/v1.1), 2026-09-02).", "fit_mcp": "mcp_gateway", "fit_mcp_note": "A genuine MCP gateway, configured with an `MCPRoute` CRD: multiple MCP servers multiplexed behind one endpoint, tool names namespaced by backend (`github__issue_read`), tool filtering by name or regex, OAuth 2.0 authorization-code flow with PKCE plus JWT scope/claim and CEL-based authorization with a configurable `defaultAction`, header forwarding, upstream API-key injection, streamable HTTP per the June 2025 MCP spec, and OTel spans plus Prometheus metrics for MCP traffic ([MCP](https://aigateway.envoyproxy.io/docs/capabilities/mcp/), 2026-09-02). The same thing runs locally via `aigw run --mcp-config` on `http://localhost:1975/mcp` ([aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun)). A dedicated `MCPBackend` CRD is on the post-1.0 roadmap ([1.0 announcement](https://aigateway.envoyproxy.io/blog/v1.0-release-announcement)).", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_terraform", "fit_vercel_ai_sdk", "fit_python_frameworks", "fit_client_sdk_langs" ] }, "fit_model_gate": "none", "fit_model_gate_note": "None: no approval, waitlist, enablement or quota step exists on the project side - whatever your provider credential can reach is reachable once you declare it in a route ([connect providers](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/connect-providers); [supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints), 2026-09-02). Any gating you experience is your upstream provider's.", "fit_onboarding_notes": "Three things bite newcomers, and all are in the docs. Envoy Gateway must be installed first, with the project's own `envoy-gateway-values.yaml`, or nothing reconciles ([prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites)). Envoy Gateway's default 32 KB client buffer is too small for model responses, which is why the basic example ships a `ClientTrafficPolicy` raising it to 50 MB ([basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage)). And token rate limiting or quotas need a Redis deployment plus rate-limit configuration chosen at Envoy Gateway install time, not afterwards ([usage-based rate limiting](https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting); [quota policy](https://aigateway.envoyproxy.io/docs/capabilities/traffic/quota-policy)). Also note the docs' own caution about `v0.0.0-latest` chart tags being unstable and the advice to pin a commit ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation), 2026-09-02).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes: OpenAI paths are served at the gateway root, the quickstart is a plain curl to `$GATEWAY_URL/v1/chat/completions`, and `aigw run` intentionally consumes OpenAI SDK environment variables so an existing client only needs its base URL changed ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [basic usage](https://aigateway.envoyproxy.io/docs/getting-started/basic-usage); [aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02).", "fit_prod_prereqs": "Kubernetes v1.32+ with Gateway API v1.5.x and Envoy Gateway v1.8.1+ (Envoy Proxy v1.38.x) for AI Gateway v1.0.x; `kubectl`, `helm` and `curl` to install; provider credentials in Kubernetes Secrets or cloud identity wired to `BackendSecurityPolicy`; Redis plus Envoy Gateway rate-limit configuration if you want token limits or quotas; an OTLP collector and Prometheus if you want traces and metrics; Gateway API Inference Extension CRDs for InferencePool routing ([compatibility](https://aigateway.envoyproxy.io/docs/compatibility); [prerequisites](https://aigateway.envoyproxy.io/docs/getting-started/prerequisites); [usage-based rate limiting](https://aigateway.envoyproxy.io/docs/capabilities/traffic/usage-based-ratelimiting); [tracing](https://aigateway.envoyproxy.io/docs/capabilities/observability/tracing); [InferencePool support](https://aigateway.envoyproxy.io/docs/capabilities/inference/inferencepool-support), 2026-09-02). Plan capacity from the documented controller footprint - 100m/128Mi requests, 500m/256Mi limits, HPA to 10 replicas - and note the ~1 MB Kubernetes object limit on the aggregated filter-config Secret when running thousands of routes ([scaling](https://aigateway.envoyproxy.io/docs/capabilities/scaling); [control-plane scaling](https://aigateway.envoyproxy.io/blog/benchmarking-control-plane-scaling)).", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "infra_rollout", "fit_python_frameworks": null, "fit_python_frameworks_note": "n.a. - no LangChain, LlamaIndex, DSPy or other framework integration is documented; the 469-URL site map contains no framework page and none of the docs pages fetched name one ([capabilities index](https://aigateway.envoyproxy.io/docs/); [supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints), 2026-09-02). In practice you would use each framework's OpenAI-compatible client with the gateway's base URL.", "fit_quickstart_steps": 4, "fit_quickstart_steps_caveat": "The getting-started guide numbers four sections (Prerequisites, Installation, Basic Usage, Connect Providers) rather than four commands; the Installation page itself has two numbered Helm steps and Prerequisites adds the Envoy Gateway install, so there is no single steps-to-first-call figure ([getting started](https://aigateway.envoyproxy.io/docs/getting-started/); [installation](https://aigateway.envoyproxy.io/docs/getting-started/installation), 2026-09-02).", "fit_quickstart_url": "https://aigateway.envoyproxy.io/docs/getting-started/", "fit_self_host_docs": "command_published", "fit_self_host_install": "`helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm --version v1.0.0 --namespace envoy-ai-gateway-system --create-namespace` then `helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm --version v1.0.0 --namespace envoy-ai-gateway-system --create-namespace`, after Envoy Gateway v1.7.0+; locally `docker run --rm -p 1975:1975 -e OPENAI_API_KEY=... envoyproxy/ai-gateway-cli run` or `go install ./cmd/aigw`", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. - no Terraform provider, module or example is published: nothing in the repository root or the 469-URL site map mentions Terraform, and the documented install path is Helm ([installation](https://aigateway.envoyproxy.io/docs/getting-started/installation); [GitHub repo](https://github.com/envoyproxy/ai-gateway), 2026-09-02).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "openai_compat_only", "fit_vercel_ai_sdk_note": "No Vercel AI SDK provider package or integration page exists - the term appears nowhere on the pages fetched or in the site map. What is documented is generic OpenAI compatibility at the gateway root, which is what an `@ai-sdk/openai` `baseURL` override would target ([supported endpoints](https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-endpoints); [aigw run](https://aigateway.envoyproxy.io/docs/cli/aigwrun), 2026-09-02).", "fit_vercel_ai_sdk_package": null }, { "slug": "higress", "name": "Higress", "company": "Alibaba Group; governed as a CNCF Sandbox project (\"Copyright Higress a Series of LF Projects, LLC\")", "tagline": "Envoy/Istio-based Kubernetes and microservice gateway from Alibaba, now a CNCF Sandbox project, whose Wasm plugin set turns it into an LLM and MCP gateway.", "category": "open_source", "deployment": "both", "license": "Apache-2.0", "founded_year": null, "hq_country": null, "url_home": "https://higress.ai", "url_docs": "https://higress.ai/en/docs/latest/overview/what-is-higress/", "url_pricing": "https://higress.ai/en/ai-gateway/", "url_status": null, "url_github": "https://github.com/higress-group/higress", "url_changelog": "https://github.com/higress-group/higress/releases", "url_models_api": null, "github_stars": 9458, "model_count": 100, "model_count_note": "Vendor-stated \"Supports 100+ LLM models\" on the AI gateway product page ([AI gateway page](https://higress.ai/en/ai-gateway/)) and \"unified protocol conversion for 100+ common models\" on the home page ([home page](https://higress.ai/en/)) and the multi-model proxy guide ([multi-model proxy](https://higress.ai/en/docs/ai/scene-guide/multi-proxy/)). No enumerated model list is published, so 100 is a floor rather than a count.", "provider_count": 31, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No fee of any kind: the software is Apache-2.0 and nothing meters your traffic, so BYOK is the only mode and it costs nothing beyond your provider bills and your own compute ([GitHub API](https://api.github.com/repos/higress-group/higress), [ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "seat_fee_usd_month": null, "free_tier": "The entire gateway is free: Apache-2.0 licensed ([GitHub API](https://api.github.com/repos/higress-group/higress)) with the Community edition described as \"Free & Open Source, Community Support\" and \"Local deployment, verify core capabilities in 5 minutes\" ([AI gateway editions](https://higress.ai/en/ai-gateway/)). There is no request, seat or token cap in the open-source edition, and no Higress account exists to create.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "Your own infrastructure only — no licence, no seat fee, no metered tier. Sizing signals from the docs: gateway defaults to 2 replicas, the console to 1, and the optional o11y stack adds Grafana, Prometheus, Loki and PromTail to the same cluster ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)). Redis is a hard dependency for ai-cache, ai-token-ratelimit, ai-quota and MCP hosting, so budget for it ([ai-cache](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/), [ai-token-ratelimit](https://higress.ai/en/docs/latest/plugins/ai/api-consumer/ai-token-ratelimit/), [MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)). Autoscaling is your job: \"Higress is based on K8s HPA and supports elastic scaling. The gateway is stateless and is a deployment.\" ([FAQ](https://higress.ai/en/docs/latest/overview/faq/))", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "Everything is in your own systems by construction: configuration lives in Kubernetes Ingress/CRD objects, in the global ConfigMap, or — in the non-Kubernetes install — in local files or a Nacos namespace you nominate with `-c file:///opt/higress/conf` or `nacos://host:8848` ([Docker Compose options](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/), [quick start](https://higress.ai/en/docs/latest/user/quickstart/)). Telemetry lands in your Prometheus, Loki and OTLP collectors ([Prometheus guide](https://higress.ai/en/docs/latest/user/prometheus/)). No vendor account holds anything to export.", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Embeddings", "Anthropic Messages", "MCP (Streamable HTTP and SSE)", "Kubernetes Ingress API", "Kubernetes Gateway API", "Gateway API Inference Extension", "Higress Console API (Swagger, disabled by default)" ], "sdk_note": "No Higress-branded client SDK exists on any fetched page; you call the gateway with the OpenAI wire format, and every ai-proxy example is a raw `curl` against `/v1/chat/completions` ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). The published SDKs are for writing gateway plugins, not for calling it: the Wasm plugin SDKs cover Go, Rust and JavaScript ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "Vendor-published and not a gateway-overhead figure. No added-latency-per-request number (ms or percentile) is published anywhere on [the README](https://github.com/higress-group/higress/blob/main/README.md), [what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/) or [the AI gateway page](https://higress.ai/en/ai-gateway/); the only quantified latency claim is the 50% time-to-first-token reduction attributed to LLM-aware load balancing rather than to the proxy hop ([Higress engineering post](https://medium.com/@higress_ai/no-increase-in-gpu-the-first-token-latency-decreases-by-50-new-practices-in-llm-service-load-5583192f9442)). Alibaba Cloud's comparison page quotes 内核优化 \"90%+\" and 软硬一体 \"50%+\" for its commercial edition against \"self-tune\" for open source, with no baseline or unit defined, so those are not recorded as Higress figures ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "n.a. as a vendor commitment — Higress has no hosted plane, so EU residency is a property of where you deploy it ([Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)). No EU, GDPR or data-residency statement appears on [the home page](https://higress.ai/en/), [the documentation index](https://higress.ai/llms.txt) or [the FAQ](https://higress.ai/en/docs/latest/overview/faq/). Note that the published container registries are Alibaba Cloud registries in cn-hangzhou, us-west-1 and ap-southeast-7 with no EU mirror, so EU operators should mirror images themselves ([README](https://github.com/higress-group/higress/blob/main/README.md)).", "zdr": null, "zdr_note": "not_applicable: there is no vendor store to opt out of. Retention is entirely yours — access logs are metadata-only by default ([log description](https://higress.ai/en/docs/latest/ops/log/)), prompt/completion logging via ai-statistics is opt-in through `apply_to_log` (default false) ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)), and ai-cache stores prompts and answers in your own Redis with `cacheTTL: 0` meaning never expire, which is the setting most likely to surprise a privacy reviewer ([ai-cache plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/)).", "air_gapped": null, "best_for": "Teams already running Kubernetes who want one Envoy-based gateway for north-south traffic, microservices and LLM/MCP calls, and who are happy to assemble reliability and guardrail behaviour from plugin fields themselves.", "strengths": [ "One Apache-2.0 binary covers Kubernetes Ingress, microservice routing, LLM proxying and MCP hosting; nginx-Ingress annotation compatibility makes it a credible ingress-nginx replacement", "31 provider types behind the OpenAI contract with path-based protocol detection for chat completions, Anthropic Messages and embeddings, plus `protocol: original` passthrough", "Genuine MCP gateway: REST-to-MCP with no code, MCP proxying, per-tool allowlists and separate downstream/upstream security schemes, with `passthroughAuthHeader` off by default", "Cache is both exact and semantic in one plugin, and all state (Redis, vector DB, Prometheus, Loki) stays in your own infrastructure", "Token-level governance is real: per-consumer quotas with an admin API, Redis-backed token-per-second/minute/hour/day limits, and ai-statistics metrics split by gateway, route, service and model", "CNCF Sandbox project since 15 March 2026 with a named enterprise adopter list, active releases and a documented plugin-authoring SDK in Go, Rust and JavaScript" ], "weaknesses": [ "Reliability primitives are off by default and shallow: ai-proxy `retryOnFailure` disabled with `maxRetries: 1` and no backoff, `failover` disabled, retries only on non-streaming requests", "`denyCode` defaults to 200 in both ai-security-guard and ai-data-masking, so blocked traffic returns HTTP 200 unless you change it; ai-security-guard also ships with `checkRequest`/`checkResponse` false", "The qwen3guard plugin is explicitly fail-open and the vendor states forced fail-close \"cannot be described as satisfied\" in the current version", "Alibaba Cloud's own matrix marks automatic fault detection and recovery as unsupported, and multi-AZ deployment, rate-limit degradation, monitoring/alerting and enterprise observability as build-it-yourself, for the open-source edition — with no SLA at all versus 99.99% for the paid product", "No published model catalogue or `/v1/models` endpoint: the only figure is a repeated \"100+ models\" marketing claim", "Documentation is unevenly bilingual — the `failover`, `retryOnFailure` and full 31-provider tables appear on the Chinese ai-proxy page but not the English one", "No image, audio, video, rerank, batch or Responses API surface, and no Terraform, LangChain, LlamaIndex or Vercel AI SDK integration is documented", "No evaluation or feedback capture of any kind, and the LLM request timeout defaults to no timeout at the Ingress layer" ], "avoid_if": "You want an LLM-first router with a published model catalogue, per-request routing policy and turnkey defaults; you need English-only documentation; or you need a vendor SLA, SOC 2 report or fail-closed guardrails, none of which exist for the open-source edition.", "reviews": [], "sources": { "company": "https://www.cncf.io/blog/2026/03/25/higress-joins-cncf-delivering-an-enterprise-grade-ai-gateway-and-a-seamless-path-from-nginx-ingress/", "fit_mcp": "https://higress.ai/en/docs/ai/mcp-server/", "license": "https://api.github.com/repos/higress-group/higress", "category": "https://higress.ai/en/docs/latest/overview/what-is-higress/", "url_docs": "https://higress.ai/en/docs/latest/overview/what-is-higress/", "url_home": "https://higress.ai/en/", "byok_mode": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "guard_pii": "https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/", "rel_notes": "https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison", "api_images": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "deployment": "https://higress.ai/en/docs/latest/user/quickstart/", "eval_hooks": "https://higress.ai/llms.txt", "url_github": "https://api.github.com/repos/higress-group/higress", "zdr_status": "https://higress.ai/en/docs/latest/ops/log/", "guard_notes": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/", "log_content": "https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/", "log_opt_out": "https://higress.ai/en/docs/latest/ops/log/", "model_count": "https://higress.ai/en/ai-gateway/", "url_pricing": "https://higress.ai/en/ai-gateway/", "api_surfaces": "https://github.com/higress-group/higress/blob/main/README.md", "github_stars": "https://github.com/higress-group/higress", "product_fate": "https://api.github.com/repos/higress-group/higress", "cache_support": "https://higress.ai/en/docs/ai/scene-guide/semantic-cache/", "cost_controls": "https://higress.ai/en/docs/latest/plugins/ai/api-consumer/ai-token-ratelimit/", "fit_cloud_iam": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "guard_content": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/", "pricing_model": "https://higress.ai/en/ai-gateway/", "url_changelog": "https://api.github.com/repos/higress-group/higress/releases/latest", "vendor_status": "https://www.cncf.io/projects/higress/", "api_embeddings": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "certifications": "https://higress.ai/llms.txt", "fit_kubernetes": "https://higress.ai/en/docs/latest/ops/deploy-by-helm/", "latency_claims": "https://medium.com/@higress_ai/no-increase-in-gpu-the-first-token-latency-decreases-by-50-new-practices-in-llm-service-load-5583192f9442", "provider_count": "https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/", "deployment_note": "https://higress.ai/en/docs/latest/ops/deploy-by-helm/", "guard_injection": "https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/", "logs_by_default": "https://higress.ai/en/docs/latest/ops/log/", "tracing_support": "https://higress.ai/en/docs/latest/user/configmap/", "rel_retry_config": "https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/", "token_markup_pct": "https://higress.ai/en/ai-gateway/", "guard_third_party": "https://higress.ai/en/docs/ai/scene-guide/application-safety/", "last_release_date": "https://api.github.com/repos/higress-group/higress/releases/latest", "openai_compatible": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "streaming_support": "https://higress.ai/en/docs/latest/overview/what-is-higress/", "throughput_claims": "https://github.com/higress-group/higress/blob/main/README.md", "architecture_class": "https://api.github.com/repos/higress-group/higress", "fit_quickstart_url": "https://higress.ai/en/docs/ai/quick-start/", "guard_failure_mode": "https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/", "rel_timeout_config": "https://higress.ai/en/docs/latest/user/annotation/", "export_destinations": "https://higress.ai/en/docs/latest/user/configmap/", "guardrails_location": "https://higress.ai/en/docs/ai/scene-guide/application-safety/", "key_management_note": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "rel_fallback_config": "https://higress.ai/en/docs/ai/scene-guide/multi-proxy/", "api_openai_responses": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "fit_self_host_install": "https://github.com/higress-group/higress/blob/main/README.md", "guard_content_default": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/", "api_anthropic_messages": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "prompt_transits_vendor": "https://github.com/higress-group/higress/blob/main/README.md", "custom_endpoint_support": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "rel_health_check_config": "https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/", "rel_multi_region_config": "https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison", "self_host_artifact_note": "https://github.com/higress-group/higress/blob/main/README.md", "rel_load_balancing_config": "https://higress.ai/en/docs/latest/user/annotation/", "api_openai_chat_completions": "https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/", "observability_requires_saas": "https://higress.ai/en/docs/latest/user/prometheus/" }, "verified_at": { "company": "2026-09-02", "fit_mcp": "2026-09-02", "license": "2026-09-02", "api_audio": "2026-09-02", "api_batch": "2026-09-02", "byok_mode": "2026-09-02", "guard_pii": "2026-09-02", "api_images": "2026-09-02", "deployment": "2026-09-02", "eval_hooks": "2026-09-02", "url_github": "2026-09-02", "zdr_status": "2026-09-02", "log_content": "2026-09-02", "log_opt_out": "2026-09-02", "model_count": "2026-09-02", "github_stars": "2026-09-26", "cache_support": "2026-09-02", "fit_terraform": "2026-09-02", "guard_content": "2026-09-02", "pricing_model": "2026-09-02", "api_embeddings": "2026-09-02", "certifications": "2026-09-02", "fit_kubernetes": "2026-09-02", "latency_claims": "2026-09-02", "provider_count": "2026-09-02", "guard_injection": "2026-09-02", "logs_by_default": "2026-09-02", "tracing_support": "2026-09-02", "rel_retry_config": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "throughput_claims": "2026-09-02", "architecture_class": "2026-09-02", "guard_failure_mode": "2026-09-02", "rel_timeout_config": "2026-09-02", "guardrails_location": "2026-09-02", "rel_fallback_config": "2026-09-02", "api_openai_responses": "2026-09-02", "self_host_data_plane": "2026-09-02", "guard_content_default": "2026-09-02", "api_anthropic_messages": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "rel_health_check_config": "2026-09-02", "rel_multi_region_config": "2026-09-02", "rel_load_balancing_config": "2026-09-02", "api_openai_chat_completions": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:17:00.406232+00:00", "vendor_status": "foundation", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed: v2.2.4 was released on 13 August 2026, the default branch was last pushed on 2 September 2026, and the repo carries 9,273 stars with 1,080 open issues ([GitHub API](https://api.github.com/repos/higress-group/higress)). It entered the CNCF Sandbox on 15 March 2026 ([CNCF project page](https://www.cncf.io/projects/higress/)), and the CNCF announcement names Alibaba Group, Ant Group, BOSS Zhipin, Cathay Insurance, Ctrip, DJI, Kuaishou, Sealos and Vipshop as adopters ([CNCF announcement](https://www.cncf.io/blog/2026/03/25/higress-joins-cncf-delivering-an-enterprise-grade-ai-gateway-and-a-seamless-path-from-nginx-ingress/)). Caveat: Alibaba Cloud's own comparison page marks the open-source edition as having no SLA at all, versus 99.99% for its commercial AI Gateway ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "last_release_date": "2026-08-13", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Latest non-prerelease is v2.2.4, published 2026-08-13T23:49:46Z via the GitHub releases API; the default branch was last pushed 2026-09-02T12:30:09Z ([releases/latest](https://api.github.com/repos/higress-group/higress/releases/latest)).", "prompt_transits_vendor": "no", "prompt_path_note": "No Higress-operated hop exists: the gateway is software you deploy, and requests go from your gateway straight to the provider endpoints configured in ai-proxy ([README](https://github.com/higress-group/higress/blob/main/README.md), [ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). Two things do leave your perimeter if you enable them — ai-security-guard sends prompt and completion text to Alibaba Cloud's content-moderation API ([content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/)), and semantic caching sends prompt text to whichever embedding service and vector database you configure, the worked example being Alibaba Cloud BaiLian plus DashVector ([semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/)).", "logs_by_default": "metadata_only", "logs_note": "Access logs are on and JSON-formatted by default, and they are metadata: the 22 documented fields are `authority`, `bytes_received`, `bytes_sent`, `duration`, `method`, `path`, `request_id`, `response_code`, `response_flags`, `route_name`, `trace_id`, `upstream_host`, `upstream_service_time`, `user_agent`, `x_forwarded_for` and similar — request and response bodies are represented only by their byte counts ([log description](https://higress.ai/en/docs/latest/ops/log/)). Token counts, model name, `llm_service_duration` and `llm_first_token_duration` are added by the ai-statistics plugin ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)).", "retention_days": null, "retention_note": "n.a. as a vendor-set value — Higress is self-hosted software with no vendor-side store. The bundled `global.o11y` stack installs Grafana, Prometheus, Loki and PromTail into your cluster and you own the retention settings ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)); ai-cache entries live in a Redis you run, with `cacheTTL` defaulting to 0 meaning never expire ([ai-cache plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/)).", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "not_applicable by construction rather than by policy: Higress is self-hosted Apache-2.0 software with no vendor telemetry endpoint on any fetched page ([GitHub API](https://api.github.com/repos/higress-group/higress), [README](https://github.com/higress-group/higress/blob/main/README.md)), so there is no Higress-side corpus to train on. No separate training or data-use policy is published — [the documentation index](https://higress.ai/llms.txt), [the FAQ](https://higress.ai/en/docs/latest/overview/faq/) and [the home page](https://higress.ai/en/) contain none.", "subprocessors_url": null, "regions_note": "You choose, because you run it. There is no vendor region list; placement is wherever your Kubernetes cluster or host sits ([Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/), [Docker Compose guide](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/)). The only region-shaped facts published are container-registry mirrors in cn-hangzhou, us-west-1 and ap-southeast-7 ([README](https://github.com/higress-group/higress/blob/main/README.md)) and Alibaba Cloud's note that multi-AZ deployment is \"build it yourself\" for the open-source edition ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "certifications": { "itar": { "detail": "Not mentioned on any fetched page.", "status": "not_published" }, "soc2": { "detail": "No SOC 2 claim on higress.ai, the documentation index or the CNCF project page. Higress is self-hosted software; certification would attach to your own deployment.", "status": "not_published" }, "fedramp": { "detail": "Not mentioned on any fetched page.", "status": "not_published" }, "gdpr_dpa": { "detail": "No DPA, privacy policy or GDPR statement on the fetched pages; there is no vendor data processor to contract with.", "status": "not_published" }, "iso27001": { "detail": "Not mentioned on any fetched Higress or CNCF page.", "status": "not_published" }, "hipaa_baa": { "detail": "Not mentioned on any fetched page.", "status": "not_published" } }, "guardrails_location": "both", "guardrails_note": "Enforcement is always in your own gateway process, but the classifier can be remote: ai-security-guard delegates to Alibaba Cloud's content-moderation API, which requires activating that Alibaba Cloud service ([content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/)), whereas qwen3guard calls a Qwen3Guard-Gen model you can host yourself behind an OpenAI-compatible endpoint via vLLM or SGLang ([Higress content-security post](https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/)), and ai-data-masking runs entirely locally in Wasm ([ai-data-masking](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/)).", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "oss_with_managed_tier", "pricing_model_note": "Apache-2.0 software with no price of its own, alongside a separately-sold managed edition: the AI gateway page presents Community (\"Free & Open Source, Community Support\", local deployment), Serverless via Alibaba Cloud AI Gateway, and a Feitian Exclusive Edition with negotiable SLA and ticket/DingTalk support ([AI gateway editions](https://higress.ai/en/ai-gateway/)). The quick start states \"Serverless Standard starts at ¥0 and charges only for actual usage\" and recommends Alibaba Cloud AI Gateway Enterprise for production without Kubernetes ([quick start](https://higress.ai/en/docs/latest/user/quickstart/)). No dollar or yuan rate card is published on the Higress site itself, and the managed edition is an Alibaba Cloud product, not a Higress-branded SaaS.", "min_commitment": "None for the open-source edition — no account, contract or licence key is involved ([GitHub API](https://api.github.com/repos/higress-group/higress)). The managed alternatives are Alibaba Cloud commitments: Serverless Standard \"starts at ¥0 and charges only for actual usage\", while the Feitian Exclusive Edition is a negotiated commercial agreement ([quick start](https://higress.ai/en/docs/latest/user/quickstart/), [AI gateway editions](https://higress.ai/en/ai-gateway/)).", "overage_note": "n.a. — there is no meter to overrun. Higress bills nothing; the costs you incur are your own infrastructure plus the upstream provider bills against the API keys you configure in ai-proxy ([AI gateway editions](https://higress.ai/en/ai-gateway/), [ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Both shapes, self-hosted, and Redis is mandatory. The ai-cache plugin does exact-match caching keyed by default on `messages.@reverse.0.content` with prefix `higress-ai-cache:` and `cacheTTL` defaulting to 0, i.e. never expire; `redis.serviceName` is required and `x-higress-skip-ai-cache: on` bypasses the cache per request ([ai-cache plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/)). Semantic caching is the same plugin pointed at an embedding service plus a vector database — the worked example uses Alibaba Cloud BaiLian `text-embedding-v3` at 1024 dimensions with DashVector and Cosine distance ([semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/)). No cached-token pricing exists because no vendor meters tokens here; a hit simply skips the upstream call, so you save the provider's own bill.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated. Token metrics are Prometheus-scrapable so alerting is possible in your own Alertmanager, but no alert feature is documented; Alibaba Cloud lists 监控告警 as \"build it yourself\" for the open-source edition.", "tier": null, "supported": null }, "virtual_keys": { "note": "Console \"Consumer Management\" issues per-consumer credentials verified by key-auth against the `x-api-key` header, and routes carry an allowed-consumer list; the term \"virtual key\" is not used. Errors are 401 for a missing/invalid key and 403 for an unauthorised consumer.", "tier": null, "supported": true }, "rate_limiting": { "note": "ai-token-ratelimit does Redis-backed token-per-second/minute/hour/day limits, keyed by header, query param, consumer, cookie or IP, with rule-level global thresholds; rejects with 429 and \"Too many requests\". Requires ai-statistics. Request-rate limits exist separately as `higress.io/route-limit-rpm`/`-rps` annotations and the cluster/local rate-limit plugins.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated on any fetched page.", "tier": null, "supported": null }, "model_allowlists": { "note": "Enforced indirectly through ai-proxy `modelMapping` (including a `*` catch-all and regex mapping) and through route-level model matching with an allowed-consumer list in AI Route Config.", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "ai-quota holds a fixed token quota per consumer in Redis under the `chat_quota:` prefix, with an admin API at `admin_path` (default `/quota`) for GET/refresh/delta. Requires key-auth or jwt-auth plus ai-statistics.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Not stated. Quotas and rate limits are per consumer, header, cookie, param or IP; no team or org object exists in the fetched docs.", "tier": null, "supported": null } }, "cost_attribution": "Per gateway, route, service and model: ai-statistics exposes input tokens, output tokens, time to first token (streaming) and total request time \"observable across four dimensions: gateway, route, service, model\", with `input_token`, `output_token`, `model`, `llm_service_duration` and `llm_first_token_duration` in the log line ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)). Per-consumer attribution comes from key-auth's `X-Mse-Consumer` header plus ai-quota's per-consumer Redis counters ([key-auth](https://higress.ai/en/docs/latest/user/plugins/authentication/key-auth/), [ai-quota](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-quota/)). Attribution is in tokens, not currency — no price table or cost field is computed anywhere.", "cost_export": "n.a. — no CSV, billing API, webhook, S3 or warehouse cost export on [ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/), [ai-quota](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-quota/) or [the token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/). What you can export is token telemetry, via Prometheus scraping and OTLP ([Prometheus guide](https://higress.ai/en/docs/latest/user/prometheus/)).", "product_kind_note": "General-purpose API gateway first, LLM router second: the docs describe Higress as \"an open source AI-native API gateway\" built on Istio and Envoy that covers AI gateway, Kubernetes Ingress and microservice-gateway use cases in one binary ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)). The AI behaviour is delivered by Wasm plugins (ai-proxy, ai-cache, ai-token-ratelimit, ai-quota, ai-security-guard, ai-data-masking, ai-statistics) inside a 41-plugin marketplace whose other 24 plugins are ordinary gateway concerns such as WAF, CORS, JWT auth and DeGraphQL ([plugin marketplace](https://higress.ai/en/plugins/)).", "deployment_modes": [ "self_host", "saas" ], "deployment_note": "Two components: `higress-controller` (config aggregation and distribution) and `higress-gateway` (data plane, `replicas` default **2**, deployable as Deployment or DaemonSet), plus a `higress-console` at 1 replica. Notable defaults: `global.ingressClass` `higress`, `global.enableIstioAPI` **false**, `global.enableGatewayAPI` **false**, `global.o11y.enabled` **false**; maximum supported Gateway API version is 1.4.0 on 2.2.x and 1.0.0 on 2.1.x and earlier ([Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)). Images come from `higress-registry.cn-hangzhou.cr.aliyuncs.com` with us-west-1 and ap-southeast-7 mirrors selectable through `global.hub` ([README](https://github.com/higress-group/higress/blob/main/README.md)). MCP hosting needs Higress >= 2.1.0, Redis for caching, and Nacos >= 3.0 with Higress >= 2.1.2 for the Nacos MCP registry ([MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Protocol is inferred from the request path rather than declared: ai-proxy auto-detects OpenAI `/v1/chat/completions`, Anthropic `/v1/messages` and OpenAI `/v1/embeddings`, and `protocol: original` lets you pass a provider's native wire format straight through instead of the OpenAI contract ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). On the north-south side the same binary serves the Kubernetes Ingress API, the Gateway API and the Gateway API Inference Extension ([README](https://github.com/higress-group/higress/blob/main/README.md)), and hosts MCP servers over Streamable HTTP and SSE ([MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)). The console's own REST API is Swagger-documented but `higress-console.swagger.enabled` defaults to false ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)).", "model_count_low": 100, "model_count_high": 100, "model_count_source_note": "Single figure, repeated verbatim on three vendor pages (100+); the \"View Complete Model Integration Directory\" link on [the AI gateway page](https://higress.ai/en/ai-gateway/) did not resolve to a fetchable list on 2026-09-02, and no `/v1/models` catalogue endpoint is documented, so the number could not be independently counted.", "provider_count_low": 26, "provider_count_high": 31, "provider_count_source_note": "Counted from the `provider.type` tables, not from marketing. The Chinese ai-proxy page enumerates 31 named provider types — openai, azure, moonshot, qwen, baichuan, yi, zhipuai, deepseek, groq, grok, openrouter, fireworks, baidu, ai360, github, mistral, minimax, claude, ollama, hunyuan, stepfun, cloudflare, spark, gemini, deepl, cohere, together-ai, dify, vertex, bedrock, triton ([ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)). The English page carries 26 provider-specific config sections and omits GitHub Models, Cohere and Dify ([ai-proxy, English](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)), hence the 26-31 range.", "first_party_vs_partner": "All third-party or self-run. Higress hosts no models: ai-proxy forwards to external provider APIs using credentials you supply, and self-hosted backends are first-class via the `ollama` provider (`ollamaServerHost`/`ollamaServerPort`) and the NVIDIA Triton Inference Server example ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "custom_endpoint_support": "Yes, several ways: `openaiCustomUrl` points the OpenAI provider at any compatible service, `azureServiceUrl` at an Azure deployment, `ollamaServerHost`/`ollamaServerPort` at a local Ollama, and there is a worked NVIDIA Triton Inference Server configuration ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). Below the plugin, arbitrary upstreams are registered as console \"Service Sources\" of type Domains or Static Addresses, including non-LLM services such as Redis ([token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/)).", "byok_mode": "byok_only", "merchant_of_record": "The upstream model provider, always. Higress forwards requests using the `apiTokens` you configure per provider and never intermediates payment ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)); if you buy the managed edition instead, Alibaba Cloud is the merchant ([quick start](https://higress.ai/en/docs/latest/user/quickstart/)).", "key_management_note": "Provider keys are plugin configuration, not a managed secret store. `apiTokens` is a list and ai-proxy \"randomly selects\" one per request, with Azure OpenAI limited to a single token; `failover` can quarantine a token that returns errors until health checks recover it ([ai-proxy, English](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)). In the non-Kubernetes install, `-k/--data-enc-key` sets a 32-character key used to encrypt sensitive configuration data and \"for cluster deployment, this must be set\", otherwise a random key is generated ([Docker Compose options](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/)). Console consumer credentials are separate from provider keys and are checked against the `x-api-key` header ([token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/)). Hardware-backed KMS integration is a commercial-edition feature, listed as 深度集成阿里云 KMS 产品 versus 自行构建 for open source ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "api_openai_chat_completions_note": "Yes: the ai-proxy plugin \"implements AI proxy functionality based on OpenAI API contract\" and auto-detects the OpenAI Chat Completions protocol when the request path is `/v1/chat/completions`, converting to each upstream provider's own format ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "api_anthropic_messages_note": "Yes: ai-proxy recognises `/v1/messages` as the Anthropic Claude Messages protocol and performs \"intelligent conversion\" to OpenAI format for providers that do not support Claude natively ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)), observed 2026-09-02.", "api_openai_responses_note": "n.a. — `/v1/responses` appears on none of the fetched pages: [ai-proxy (EN)](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [ai-proxy (CN)](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/), [plugin marketplace](https://higress.ai/en/plugins/) and the full documentation index at [higress.ai/llms.txt](https://higress.ai/llms.txt).", "api_embeddings_note": "Yes: ai-proxy auto-detects `/v1/embeddings` as the OpenAI text-embedding protocol, with worked examples for Qwen and 360 Brain ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). Semantic caching also depends on calling an embedding service through the gateway ([semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/)).", "api_images_note": "n.a. as a gateway path — no `/v1/images/generations` or image-generation surface on [ai-proxy (EN)](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [ai-proxy (CN)](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/) or [plugin marketplace](https://higress.ai/en/plugins/). The only multimodal content is a vision-input example headed \"Multimodal Model API Request Example (Applicable to qwen-vl-plus and qwen-vl-max Models)\" that still posts to chat/completions.", "api_audio_note": "n.a. — no `/v1/audio/*`, TTS or STT path on [ai-proxy (EN)](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [ai-proxy (CN)](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/) or [plugin marketplace](https://higress.ai/en/plugins/). DeepL is supported as a provider but only for text translation.", "api_batch_note": "n.a. — no batch or async bulk endpoint on [ai-proxy (EN)](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [ai-proxy (CN)](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/) or the documentation index [higress.ai/llms.txt](https://higress.ai/llms.txt).", "deployment_modes_note": "Self-host is the primary mode, in three shapes: Helm on Kubernetes ([Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)), Docker Compose or a shell installer without Kubernetes, with configuration in local files or Nacos ([Docker Compose guide](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/), [quick start](https://higress.ai/en/docs/latest/user/quickstart/)), and a single all-in-one container ([README](https://github.com/higress-group/higress/blob/main/README.md)). The SaaS option is not a Higress service — it is Alibaba Cloud AI Gateway, recommended in Higress's own quick start for production without Kubernetes, in Serverless Standard, Serverless Enterprise and Feitian Exclusive tiers ([quick start](https://higress.ai/en/docs/latest/user/quickstart/), [AI gateway editions](https://higress.ai/en/ai-gateway/)).", "byok_mode_note": "`byok_only`: every provider block in ai-proxy requires your own credentials — `apiTokens` for token-based providers, `vertexAuthKey`/`vertexRegion`/`vertexProjectId` for Google Vertex AI, `awsAccessKey`/`awsSecretKey`/`awsRegion` for AWS Bedrock ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). The installer prompts for an Alibaba Cloud DashScope or other API key at first run and the console's \"LLM Provider Management\" page is where the keys live ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)). Higress sells no credits and hosts no models.", "access_meta": {}, "rel_timeout_config": "config_file", "rel_timeout_note": "`config_file`, but the LLM request timeout is the weak spot: ai-proxy's `timeout` (default **120000** ms) is documented as applying only to the context-data retrieval call, not to the model request ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). The request timeout you actually want is the Ingress annotation `higress.io/timeout`, in seconds, with **no timeout by default** ([annotations](https://higress.ai/en/docs/latest/user/annotation/)). Connection-level values live in the ConfigMap: downstream `idleTimeout` 180 s, upstream `idleTimeout` 10 s ([global configuration](https://higress.ai/en/docs/latest/user/configmap/)). MCP proxying has its own `server.timeout`, default 5000 ms ([mcp-server plugin](https://higress.ai/en/docs/ai/mcp-server/)).", "rel_retry_config": "config_file", "rel_retry_note": "Off by default at the AI layer and capped at one extra attempt when switched on; no backoff strategy or jitter is documented anywhere ([ai-proxy retryOnFailure](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)). The generic Ingress retry annotation is separate and retries to a different upstream host rather than to a different model ([annotations](https://higress.ai/en/docs/latest/user/annotation/)). No retry keys appear in the global ConfigMap ([global configuration](https://higress.ai/en/docs/latest/user/configmap/)).", "rel_retry_default_note": "`config_file` in two independent places. ai-proxy `retryOnFailure`: `enabled` default **false**, `maxRetries` default **1**, `retryTimeout` default **30000** ms, `retryOnStatus` default `[\"4.*\",\"5.*\"]`, and \"only non-streaming requests can be retried\" ([ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)). Ingress-level: `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` default **3**, with `proxy-next-upstream-timeout` having no timeout by default ([annotation compatibility](https://higress.ai/en/docs/latest/user/annotation/)).", "rel_fallback_config": "dashboard_only", "rel_fallback_note": "Model-level fallback is a console concept, not a plugin field: the home page states Higress \"supports model-level Fallback\" ([home page](https://higress.ai/en/)) and the multi-model proxy guide configures it in `AI Route Config`, where a DeepSeek route falls back to Alibaba Cloud `qwen-turbo` when the primary model fails or is rate-limited ([multi-model proxy guide](https://higress.ai/en/docs/ai/scene-guide/multi-proxy/)). The guide shows a single fallback target, not an ordered chain, and no YAML/CRD field for it appears in [ai-proxy (EN)](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/) or [ai-proxy (CN)](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/). Alibaba Cloud's own comparison confirms cross-model failover exists in the open-source edition (\"多模型间 Failover: 支持\") ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "rel_fallback_shape": "single", "rel_health_check_config": "config_file", "rel_health_check_note": "Health checking exists only at the API-key level, not the upstream level. ai-proxy `failover` is `enabled: false` by default with `failureThreshold` **3**, `successThreshold` **1**, `healthCheckInterval` **5000** ms, `healthCheckTimeout` **5000** ms, a required `healthCheckModel` and `failoverOnStatus` `[\"4.*\",\"5.*\"]`; an unhealthy `apiToken` is removed from rotation and re-added once probes pass ([ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)), which the AI quick-start describes in prose as pausing requests on a token until health checks recover it ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)). No circuit-breaker or outlier-detection annotation appears in [the annotation reference](https://higress.ai/en/docs/latest/user/annotation/) and no health-check keys appear in [the global ConfigMap](https://higress.ai/en/docs/latest/user/configmap/) — and Alibaba Cloud's comparison page explicitly marks \"故障自动检测及恢复\" (automatic fault detection and recovery) as 不支持 for the open-source edition ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "n.a. — no cross-region or multi-AZ failover configuration on [the Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/), [the global ConfigMap](https://higress.ai/en/docs/latest/user/configmap/) or [the annotation reference](https://higress.ai/en/docs/latest/user/annotation/). Alibaba Cloud's comparison page lists 多可用区部署 (multi-availability-zone deployment) as 自行构建, i.e. build it yourself, for the open-source edition ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)). Container images are mirrored to cn-hangzhou, us-west-1 and ap-southeast-7 registries, but that is image distribution, not traffic failover ([README](https://github.com/higress-group/higress/blob/main/README.md)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Two unrelated layers. Classic HTTP balancing is an Ingress annotation: `nginx.ingress.kubernetes.io/load-balance` defaults to `round_robin` with `least_conn` and `random` also supported, `ewma` is explicitly **not** supported and silently falls back to round-robin, and `upstream-hash-by` gives consistent hashing on `$request_uri`, `$host`, `$remote_addr`, a header or a query arg ([annotation compatibility](https://higress.ai/en/docs/latest/user/annotation/)). Across LLM credentials, ai-proxy simply picks an `apiToken` at random per request ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). LLM-aware algorithms — \"global least connections\", \"prefix matching\" and \"GPU-aware\" load balancing — are described in a vendor engineering post rather than in the plugin reference ([Higress engineering post](https://medium.com/@higress_ai/no-increase-in-gpu-the-first-token-latency-decreases-by-50-new-practices-in-llm-service-load-5583192f9442)), and no `ai-load-balancer` page exists in the documentation index ([higress.ai/llms.txt](https://higress.ai/llms.txt)).", "architecture_class": "compiled_binary", "architecture_note": "Envoy data plane plus a Go control plane: the project is \"built on Istio and Envoy\" and extended with Wasm plugins written in Go, Rust or JavaScript ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)). Repo language bytes are Go 9,245,785, C++ 1,269,908, Rust 193,762, Shell 122,023, Python 88,069 and TypeScript 53,579 ([GitHub languages API](https://api.github.com/repos/higress-group/higress)), i.e. a compiled control plane and a compiled proxy, with plugin code compiled to Wasm rather than interpreted per request.", "self_host_data_plane": "yes", "self_host_artifact_note": "Container images and a Helm chart, both published: `higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one:latest` runs console, HTTP and HTTPS on 8001/8080/8443 from one `docker run` ([README](https://github.com/higress-group/higress/blob/main/README.md)), the chart is `higress.io/higress` installed into `higress-system` ([README](https://github.com/higress-group/higress/blob/main/README.md), [Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)), and a Docker Compose bundle of apiserver/controller/pilot/gateway/console is generated by an installer script for non-Kubernetes hosts ([Docker Compose guide](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/)). Alibaba Cloud Compute Nest deployment is also documented ([documentation index](https://higress.ai/llms.txt)). The FAQ notes there is no pre-built package beyond the images: \"There is no existing one, you need to build it yourself. Currently, all Docker images are provided and can be pulled and used by yourself.\" ([FAQ](https://higress.ai/en/docs/latest/overview/faq/))", "latency_claims": [ { "url": "https://medium.com/@higress_ai/no-increase-in-gpu-the-first-token-latency-decreases-by-50-new-practices-in-llm-service-load-5583192f9442", "unit": "%", "scope": "llm_load_balancing", "value": "50% lower", "metric": "time to first token", "publisher": "vendor", "conditions": "Vendor engineering post on LLM-aware load balancing. Stated conditions: NVIDIA GenAI-Perf as the load generator, llama3 on vLLM across 3 ECS nodes, concurrency 20, 60 sessions of 5 dialogue rounds, ~200 input and ~800 output tokens per round. No before/after absolute values, no percentile and no GPU model are given in the post; headline is \"No increase in GPU, the first token latency decreases by 50%\"." } ], "throughput_claims": [ { "url": "https://github.com/higress-group/higress/blob/main/README.md", "unit": "requests/second", "scope": "capacity", "value": "hundreds of thousands", "metric": "production traffic handled", "publisher": "vendor", "conditions": "README/overview marketing statement: \"Born from Alibaba's internal product with over 2 years of production validation, supporting large-scale scenarios with hundreds of thousands of requests per second.\" No hardware, replica count, payload size or measurement method given, and it is aggregate Alibaba production traffic rather than a per-instance benchmark." }, { "url": "https://github.com/higress-group/higress/blob/main/README.md", "unit": "x", "scope": "control_plane", "value": "10x", "metric": "speed of route configuration changes vs ingress-nginx", "publisher": "vendor", "conditions": "README cites a comparison published by sealos; \"route changes take effect ten times faster than ingress-nginx\" with resource overhead \"significantly decreased\". Cluster size, route count and method not stated in the README." } ], "streaming_support": "yes", "streaming_note": "Yes, and it is a headline feature: Higress claims \"true streaming processing\" for SSE and supports the AI streaming (SSE) scenario ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)), and ai-cache handles both streaming and non-streaming responses ([ai-cache](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/)). Two documented streaming caveats matter: retries only apply to non-streaming requests ([ai-proxy retryOnFailure](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)), and on SSE the ai-data-masking plugin may fail to restore a masked word split across chunks and may leak part of a sensitive word to the client ([ai-data-masking](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/)).", "guard_pii": "sync_block", "guard_pii_note": "`sync_block` by default when the plugin is attached: ai-data-masking's `system_deny` (built-in sensitive-word rules, sourced from `github.com/houbb/sensitive-word`) and `deny_openai` both default to **true**, and `deny_code` defaults to **200** with the message \"Sensitive words found in the question or answer have been blocked\". A softer mode exists — `replace_roles` with regex/GROK patterns and `type: replace` or `hash`, plus `restore: true` to put the original values back into the model's answer ([ai-data-masking](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/)). Documented failure mode on SSE: a masked word split across chunks may not be restored, and part of a sensitive word can reach the user.", "guard_injection": "sync_block", "guard_injection_note": "`sync_block`, but only through the newer `qwen3guard` plugin, and only as a by-product: the vendor engineering post states Qwen3Guard's official safety policy covers violence, illegal acts, sexual content, PII, self-harm, unethical behaviour, politically sensitive topics and copyright infringement \"并在输入审核中包含 Jailbreak 检测\" (jailbreak detection is included in input moderation). The plugin enables `checkRequest` and `checkResponse` by default, calls `Qwen/Qwen3Guard-Gen-4B`, uses `riskLevelBar: Unsafe` by default, and when the bar is met returns a refusal without calling the model at all ([Higress content-security post](https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/)). Caveats stated by the vendor: only the `Safety` verdict drives the decision, per-category actions are not implemented, and streaming interception cannot honour `denyCode`. No prompt-injection or jailbreak wording appears in the older [ai-security-guard](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/) reference.", "guard_content": "sync_block", "guard_content_note": "`sync_block` via ai-security-guard, which calls Alibaba Cloud's content-moderation service inline for both directions and denies the request when a risk label is returned; `checkRequest` and `checkResponse` both default to **false**, `requestCheckService` defaults to `llm_query_moderation`, `responseCheckService` to `llm_response_moderation`, and `denyCode` defaults to **200** — so a blocked call looks like a success to a naive client ([ai-security-guard](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/)). It emits `ai_sec_request_deny`/`ai_sec_response_deny` metrics and `ai_sec_risklabel`/`ai_sec_deny_phase` span attributes. The console walkthrough registers Alibaba Cloud Content Safety as a service source and attaches \"AI Safety Guard\" to a route ([content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/)).", "guard_custom": "sync_block", "guard_custom_note": "Custom policy is regex/GROK-based: ai-data-masking takes user-supplied `deny_words` and `replace_roles` regex patterns with per-rule `replace`/`hash` actions ([ai-data-masking](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/)), and the generic `request-block` and `request-validation` plugins add path/body blocking and JSON-schema request validation ([plugin marketplace](https://higress.ai/en/plugins/)). Custom rules you add yourself default to replacement rather than denial unless you put them in the deny list, hence `sync_observe` as the default posture for custom rules specifically.", "guard_model_policy": "yes", "guard_model_policy_note": "Yes, at the route: `AI Route Config` matches on model name and carries an allowed-consumer list, so a consumer can only reach the models on routes they are authorised for ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)), and ai-proxy's `modelMapping` rewrites or pins requested model names — including a `*` catch-all and regex forms such as `~gpt(.*): openai/gpt$1` — so clients cannot reach an unmapped model ([ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "guard_execution_location": "both", "guard_third_party": [ "Alibaba Cloud Content Moderation", "Qwen3Guard" ], "guard_failure_mode": "fail_open", "guard_failure_mode_note": "`fail_open`, stated explicitly and unusually candidly for the qwen3guard plugin: \"当前插件选择 fail-open：记录警告并放行\" — on connection failure, `timeoutMs` (default 2000 ms) expiry, a non-200 from the guard model, unparseable JSON or a missing `Safety` field, the request is logged and allowed through. The same post says compliance scenarios needing forced fail-close \"不能被描述为已经满足\" (cannot be described as satisfied) in the current version ([Higress content-security post](https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/)). No fail-open/fail-closed statement appears in the older [ai-security-guard](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/) reference.", "log_content": "configurable", "log_content_note": "Configurable in both directions: `accessLogFormat` under the `mesh` field controls which access-log fields are emitted ([log description](https://higress.ai/en/docs/latest/ops/log/)), and ai-statistics has an explicit \"record questions and answers\" mode where `attributes` entries with `value_source` of `request_body`, `response_body` or `response_streaming_body` and `apply_to_log: true` write prompt and completion text into the log — both flags default to **false**, so content logging is opt-in ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)).", "log_opt_out": "not_documented", "log_opt_out_note": "n.a. as a documented switch: [the log page](https://higress.ai/en/docs/latest/ops/log/) explains how to change `accessLogFormat` and how to view logs but states no way to disable access logging, and [the global ConfigMap](https://higress.ai/en/docs/latest/user/configmap/) and [the annotation reference](https://higress.ai/en/docs/latest/user/annotation/) contain no logging on/off key. In practice a self-hosted operator controls this at the Envoy/ConfigMap level, but Higress does not document it.", "tracing_support": "otel", "tracing_note": "OpenTelemetry, plus SkyWalking and Zipkin, configured in the global ConfigMap under `tracing`: `enable` defaults to **false**, `sampling` to **100.0**, `timeout` to **500** ms, and only one exporter can take effect at a time; the `opentelemetry` block takes a `service` and a gRPC `port` ([global configuration](https://higress.ai/en/docs/latest/user/configmap/)). Spans are enriched by ai-statistics via `apply_to_span` attributes, and `trace_id` is an access-log field ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/), [log description](https://higress.ai/en/docs/latest/ops/log/)).", "export_destinations": [ "OpenTelemetry", "Prometheus", "Grafana", "Loki", "SkyWalking", "Zipkin" ], "export_destinations_note": "Traces go to OpenTelemetry, SkyWalking or Zipkin collectors you name in the ConfigMap ([global configuration](https://higress.ai/en/docs/latest/user/configmap/)); metrics are scraped from the gateway's metrics port (default **15020** in the Docker Compose install) by Prometheus and drawn in Grafana, with `--set global.o11y.enabled=true` installing Grafana, Prometheus, Loki and PromTail for you, or an external Grafana URL pasted into the console instead ([Prometheus guide](https://higress.ai/en/docs/latest/user/prometheus/), [Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/), [Docker Compose options](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/)). There is no vendor SaaS sink and no Datadog/LangSmith/S3 integration on any fetched page.", "feedback_capture": "no", "feedback_capture_note": "No feedback, rating or annotation endpoint. ai-statistics can attach arbitrary attributes to logs and spans from request or response fields, which is the closest available mechanism, but nothing captures a user verdict ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)); nothing appears in [the plugin marketplace](https://higress.ai/en/plugins/) either.", "eval_hooks": "no", "eval_hooks_note": "No evaluation, scoring or experiment feature exists: the 41-plugin marketplace has no eval plugin ([plugin marketplace](https://higress.ai/en/plugins/)) and the documentation index lists none across overview, user guide, ops, developer, AI gateway and scene-guide sections ([higress.ai/llms.txt](https://higress.ai/llms.txt)). Traffic-shaping primitives that could support an offline comparison do exist — canary annotations (`canary-by-header`, `canary-weight`) and 多模型灰度 model-level canary — but no scoring is attached ([annotation reference](https://higress.ai/en/docs/latest/user/annotation/), [Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "observability_requires_saas": "No. The console ships \"a built-in monitoring suite based on Prometheus + Grafana, though it's not installed by default\", enabled with `--set global.o11y.enabled=true`, and if you skip it the Monitoring Dashboard page accepts an external Grafana URL instead ([Prometheus guide](https://higress.ai/en/docs/latest/user/prometheus/)). Token-level AI metrics come from the self-hosted ai-statistics plugin across gateway, route, service and model dimensions ([ai-statistics](https://higress.ai/en/docs/latest/plugins/ai/api-o11y/ai-statistics/)). The one caveat is Alibaba Cloud's own matrix, which lists 企业级可观测 (enterprise-grade observability) and 监控告警 (monitoring and alerting) as \"build it yourself\" for the open-source edition ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)).", "log_retention_note": "No retention period is published or configurable by Higress, because Higress stores nothing centrally: logs go to the gateway pod's stdout and, if you enable the bundled observability stack, into your own Loki ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)). Retention is whatever your log pipeline is set to.", "production_meta": {}, "guard_pii_default": "sync_block", "guard_injection_default": "sync_block", "guard_content_default": "off_by_default", "guard_custom_default": "sync_observe", "rel_notes": "The reliability surface is a general-purpose gateway's, not an LLM router's, and it is split awkwardly: retries and key failover are ai-proxy plugin fields (both off by default), load balancing and request timeout are nginx-compatible Ingress annotations, and model fallback is a console-only setting. Read together with Alibaba Cloud's own feature matrix — which marks automatic fault detection and recovery, multi-AZ deployment, rate-limit degradation, monitoring/alerting and enterprise observability as \"build it yourself\" or unsupported in the open-source edition ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)) — the honest reading is that Higress gives you the primitives and expects you to operate them. Envoy underneath means richer circuit breaking is reachable through Istio APIs, but `global.enableIstioAPI` defaults to false ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/)).", "guard_notes": "**The defaults to check before trusting this stack**: ai-security-guard ships with `checkRequest: false` and `checkResponse: false`, so attaching it without editing those fields inspects nothing ([ai-security-guard](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-security-guard/)); `denyCode` defaults to **200** in both ai-security-guard and ai-data-masking, so blocked traffic returns HTTP 200 with a refusal body unless you change it; and the qwen3guard plugin is deliberately fail-open, so an outage of the moderation model silently disables moderation ([Higress content-security post](https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/)). Streaming is the weakest surface: ai-data-masking may leak fragments of a sensitive word across SSE chunks ([ai-data-masking](https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking/)), and qwen3guard cannot apply `denyCode` once bytes have shipped.", "fit_agent_features_note": "Agent support is plugin-shaped and fairly basic. The `ai-agent` plugin runs a ReAct loop in the gateway against OpenAPI-described tools — `llm` block with `maxIterations` default **15**, `maxExecutionTime` default **50000** ms and `maxTokens` default **1000**, `apis[].api` holding the tool's OpenAPI document, and Chinese/English ReAct prompt templates; the worked example wires Amap, XZWeather and DeepL against `qwen-max-0403` ([ai-agent plugin](https://higress.ai/en/docs/latest/user/plugins/ai/api-dev/ai-agent/)). Note that this is prompt-driven ReAct, not OpenAI tool-calling: the page says nothing about function calling. The stronger agent story is MCP — REST-to-MCP conversion and MCP proxying with per-tool auth ([mcp-server plugin](https://higress.ai/en/docs/ai/mcp-server/)) — plus ai-history for multi-turn context and ai-intent for intent routing ([plugin marketplace](https://higress.ai/en/plugins/)).", "fit_byok_required": "yes", "fit_byok_required_note": "Yes — you cannot make a first call without an upstream key. The installer prompts \"enter the Aliyun Dashscope or other API-KEY\" (you may skip and add it in the console instead) and \"LLM Provider Management\" is where keys for Alibaba Cloud, DeepSeek, Azure OpenAI, OpenAI, DouBao and others are stored ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)); every ai-proxy provider block takes `apiTokens` or cloud credentials ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "fit_client_sdk_langs": null, "fit_client_sdks_note": "n.a. — no Higress client library exists on [the ai-proxy reference](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/), [the documentation index](https://higress.ai/llms.txt) or [the AI quick start](https://higress.ai/en/docs/ai/quick-start/); the documented calling convention is `curl` or any OpenAI-compatible SDK against the gateway route. The Go, Rust and JavaScript SDKs Higress does publish are for authoring Wasm gateway plugins, not for calling the gateway ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)), so no client-language list is recorded here.", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Cloud credentials are passed as static plugin fields, not assumed roles: AWS Bedrock takes `awsAccessKey`, `awsSecretKey` and `awsRegion`, and Google Vertex AI takes `vertexAuthKey`, `vertexRegion`, `vertexProjectId` and `vertexAuthServiceName` ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). No IRSA, workload identity, instance-profile or credential-chain support is documented. IDaaS/OIDC integration for the control plane is listed as a commercial-edition feature (深度集成阿里云 IDAAS 产品) versus \"build it yourself\" for open source ([Alibaba Cloud comparison](https://help.aliyun.com/zh/api-gateway/ai-gateway/product-overview/product-comparison)), although an OIDC plugin exists for data-plane auth ([plugin marketplace](https://higress.ai/en/plugins/)).", "fit_cloudflare_workers": "upstream_model_provider", "fit_cloudflare_workers_note": "Cloudflare appears only as a model source: ai-proxy has a `cloudflare` provider type for Cloudflare Workers AI, configured with `cloudflareAccountId` ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). Higress itself is a self-hosted Envoy process and does not run on Workers ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)).", "fit_credit_card": "not_required", "fit_credit_card_note": "Not required: the Community edition is \"Free & Open Source\" with local deployment ([AI gateway editions](https://higress.ai/en/ai-gateway/)) and the install path is a Docker or Helm command with no account creation ([README](https://github.com/higress-group/higress/blob/main/README.md)). A card only enters the picture if you choose Alibaba Cloud's managed edition ([quick start](https://higress.ai/en/docs/latest/user/quickstart/)).", "fit_ecosystem_notes": "The ecosystem is cloud-native rather than AI-native: service discovery from Nacos, ZooKeeper, Consul and Eureka, and governance interop with Dubbo, Nacos and Sentinel ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)); Wasm plugins in Go, Rust or JavaScript with sandbox isolation and hot updates; Prometheus, Grafana, Loki, SkyWalking, Zipkin and OpenTelemetry for telemetry ([global configuration](https://higress.ai/en/docs/latest/user/configmap/), [Prometheus guide](https://higress.ai/en/docs/latest/user/prometheus/)). The AI-side dependencies lean Alibaba: content moderation, BaiLian embeddings and DashVector in the worked examples, and the recommended managed upgrade path is Alibaba Cloud AI Gateway ([content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/), [semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/), [quick start](https://higress.ai/en/docs/latest/user/quickstart/)). Sibling projects HiMarket (API/agent portal, documented on the same site) and HiClaw are also named ([documentation index](https://higress.ai/llms.txt), [CNCF announcement](https://www.cncf.io/blog/2026/03/25/higress-joins-cncf-delivering-an-enterprise-grade-ai-gateway-and-a-seamless-path-from-nginx-ingress/)).", "fit_eval_prereqs": "Docker and an LLM provider key. The AI gateway quick start is a Docker install that exposes the console on `http://localhost:8001/`, asks you to set an admin account on first login, and prompts for an Alibaba Cloud DashScope or other API key which you may skip and add later; it also warns that \"AI Gateway needs to access Internet resources\" during start-up ([AI quick start](https://higress.ai/en/docs/ai/quick-start/), [token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/)). No Kubernetes cluster, cloud account or credit card is involved.", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, a single command: `docker run -d --rm --name higress-ai -v ${PWD}:/data -p 8001:8001 -p 8080:8080 -p 8443:8443 higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one:latest`, then the console on port 8001 ([README](https://github.com/higress-group/higress/blob/main/README.md)). The AI-specific alternative is `wget https://higress.cn/ai-gateway/install.sh` run with bash ([semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It is the gateway: Higress is described as an AI-native API gateway built on Istio and Envoy that unifies AI gateway, Kubernetes Ingress gateway, microservice gateway and security gateway roles in one deployment ([what-is-Higress](https://higress.ai/en/docs/latest/overview/what-is-higress/)), and the home page splits the AI side into LLM Gateway, MCP Gateway and Model Gateway (the last implementing the Gateway API Inference Extension with InferencePool and Endpoint Picker) ([home page](https://higress.ai/en/)).", "fit_helm_chart": "`higress.io/higress` from the `https://higress.io/helm-charts` repo, installed into namespace `higress-system`; sub-charts `higress-core` (controller + gateway) and `higress-console`", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Kubernetes is the first-class target: `helm repo add higress.io https://higress.io/helm-charts` then `helm install higress -n higress-system higress.io/higress --create-namespace`, with `global.local=true` for a kind cluster and `hgctl` to open the console ([Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/), [quick start](https://higress.ai/en/docs/latest/user/quickstart/)). The README states Higress is a conformant Gateway API implementation, a conformant Gateway API Inference Extension implementation, and is listed in the official Kubernetes Ingress Controllers documentation ([README](https://github.com/higress-group/higress/blob/main/README.md)). It is also nginx-Ingress-annotation compatible, which is its main migration story ([annotation compatibility](https://higress.ai/en/docs/latest/user/annotation/)).", "fit_mcp": "mcp_gateway", "fit_mcp_note": "A real MCP gateway, not just a client. The `mcp-server` plugin both converts REST APIs into MCP tools with no code (`server.type: rest`, `tools[].requestTemplate`) and proxies existing MCP servers (`server.type: mcp-proxy` with `mcpServerURL`, `server.timeout` default 5000 ms), with `allowTools` allowlisting, `securitySchemes` plus `defaultDownstreamSecurity`/`defaultUpstreamSecurity`, and `passthroughAuthHeader` defaulting to **false** so client credentials are not leaked to backends ([mcp-server plugin](https://higress.ai/en/docs/ai/mcp-server/)). Hosting requires Higress >= 2.1.0 and Redis; the Nacos MCP registry needs Nacos >= 3.0 and Higress >= 2.1.2; SSE endpoints are exposed at `/{mcp route prefix}/{nacos name}/sse` and the 2025-03-26 streamable-HTTP protocol needs no ConfigMap entry ([MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)). The home page announces support for the 2026-07-28 MCP revision \"with compatibility for existing protocols\" ([home page](https://higress.ai/en/)), an `openapi-to-mcp` conversion tool and a hosted demo at https://mcp.higress.ai/ ([README](https://github.com/higress-group/higress/blob/main/README.md)).", "fit_meta": null, "fit_model_gate": "none", "fit_model_gate_note": "No gate: you add a provider block with your own `apiTokens` and the model is immediately routable, with `modelMapping` deciding which names clients may use ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)); the console equivalent is adding a key under LLM Provider Management ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)). No approval, enablement, quota or waitlist step appears on either page.", "fit_onboarding_notes": "Fast to a first token, slower to a defensible production posture. One `docker run` gives you console, HTTP and HTTPS on 8001/8080/8443 ([README](https://github.com/higress-group/higress/blob/main/README.md)) and the console walks you through LLM Provider Management, Service Sources and AI Route Config with per-route strategies for auth, rate limiting, RAG, prompt templates and semantic caching ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)). Friction to expect: the installer needs outbound internet access at start-up ([token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/)); Redis must exist before caching, token rate limiting, quotas or MCP hosting work ([ai-cache](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/), [MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)); guardrails require an Alibaba Cloud Content Safety subscription or a self-hosted Qwen3Guard model ([content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/), [Higress content-security post](https://higress.ai/blog/higress-mmse_awbbpb_yafyyrc3t5wh0u55/)); and a meaningful part of the reference material — including the `failover` and `retryOnFailure` field tables — is only on the Chinese pages ([ai-proxy, Chinese](https://higress.cn/docs/latest/plugins/ai/api-provider/ai-proxy/)).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes: point an OpenAI-format client at the gateway route and ai-proxy converts to whichever provider the route is bound to, because the plugin \"implements AI proxy functionality based on OpenAI API contract\" and detects the protocol from `/v1/chat/completions` ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)). Requests are authenticated with your own consumer credential in `x-api-key` rather than a provider key ([token management guide](https://higress.ai/en/docs/ai/scene-guide/token-management/)). No SDK change is needed; `protocol: original` is available when you would rather not be converted at all.", "fit_prod_prereqs": "A Kubernetes cluster with Helm, plus a LoadBalancer service (the quick start documents hostNetwork or MetalLB fallbacks when none is available) ([quick start](https://higress.ai/en/docs/latest/user/quickstart/)). Redis is required for ai-cache, ai-token-ratelimit, ai-quota and MCP hosting ([ai-cache](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-cache/), [ai-token-ratelimit](https://higress.ai/en/docs/latest/plugins/ai/api-consumer/ai-token-ratelimit/), [MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/)). Add-ons as needed: Prometheus/Grafana/Loki via `global.o11y.enabled=true`, Nacos >= 3.0 for the MCP registry, an embedding service plus a vector database for semantic caching, and an Alibaba Cloud Content Safety subscription for ai-security-guard ([Helm values](https://higress.ai/en/docs/latest/ops/deploy-by-helm/), [MCP quick start](https://higress.ai/en/docs/ai/mcp-quick-start/), [semantic cache guide](https://higress.ai/en/docs/ai/scene-guide/semantic-cache/), [content security guide](https://higress.ai/en/docs/ai/scene-guide/application-safety/)).", "fit_prod_selfhost_shape": "cli_or_container", "fit_prod_shape": "infra_rollout", "fit_python_frameworks": null, "fit_python_frameworks_note": "n.a. — neither LangChain nor LlamaIndex is documented anywhere in the official docs index or the plugin marketplace ([higress.ai/llms.txt](https://higress.ai/llms.txt), [plugin marketplace](https://higress.ai/en/plugins/)); a LangChain + Higress + Elasticsearch RAG walkthrough exists only as a third-party blog post, not vendor documentation. RAG is instead offered as a gateway plugin (ai-rag) and via ai-search ([plugin marketplace](https://higress.ai/en/plugins/)).", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "No numbered step list exists to count. The AI gateway quick start is organised as screenshot-driven sections (install by Docker, log in to the console, configure an LLM provider key, configure an AI route, observe the AI dashboard) rather than enumerated steps ([AI quick start](https://higress.ai/en/docs/ai/quick-start/)), and the Kubernetes quick start is structured as three stages — install, configure, validate — with branching environment options rather than a linear count ([quick start](https://higress.ai/en/docs/latest/user/quickstart/)). Recorded as 0 to mean \"vendor publishes no step count\", not \"zero work\".", "fit_quickstart_url": "https://higress.ai/en/docs/ai/quick-start/", "fit_self_host_docs": "command_published", "fit_self_host_install": "One container: `docker run -d --rm --name higress-ai -v ${PWD}:/data -p 8001:8001 -p 8080:8080 -p 8443:8443 higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/all-in-one:latest` (console 8001, HTTP 8080, HTTPS 8443). Kubernetes: `helm install higress -n higress-system higress.io/higress --create-namespace`. Non-Kubernetes AI gateway installer: `wget https://higress.cn/ai-gateway/install.sh` then run with bash.", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. — no Terraform provider, module or registry reference on [the documentation index](https://higress.ai/llms.txt), [the Helm guide](https://higress.ai/en/docs/latest/ops/deploy-by-helm/) or [the Docker Compose guide](https://higress.ai/en/docs/latest/ops/deploy-by-docker-compose/). Infrastructure-as-code is expressed instead as Helm values and Kubernetes Ingress/ConfigMap objects, plus an Alibaba Cloud Compute Nest deployment path ([documentation index](https://higress.ai/llms.txt)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Local deployment, verify core capabilities in 5 minutes\" for the Community edition ([AI gateway editions](https://higress.ai/en/ai-gateway/)). The README separately claims Higress \"can be deployed with a single Docker command\" outside Kubernetes ([README](https://github.com/higress-group/higress/blob/main/README.md)).", "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "n.a. — the Vercel AI SDK is not mentioned on [the documentation index](https://higress.ai/llms.txt), [the ai-proxy reference](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/) or [the AI gateway page](https://higress.ai/en/ai-gateway/). In practice `@ai-sdk/openai` with a custom `baseURL` should work against the OpenAI-compatible route ([ai-proxy plugin](https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy/)), but Higress documents no such integration, so this is recorded as not documented rather than as compatibility.", "fit_vercel_ai_sdk_package": null }, { "slug": "litellm", "name": "LiteLLM", "company": "BerriAI", "tagline": "Open-source AI gateway and Python SDK that puts one OpenAI-compatible API in front of many LLM providers.", "category": "open_source", "deployment": "self_host_only", "license": "MIT", "founded_year": 2023, "hq_country": "United States", "url_home": "https://www.litellm.ai/", "url_docs": "https://docs.litellm.ai/docs/simple_proxy", "url_pricing": "https://www.litellm.ai/#pricing", "url_status": null, "url_github": "https://github.com/BerriAI/litellm", "url_changelog": "https://docs.litellm.ai/release_notes", "url_models_api": null, "github_stars": 59000, "model_count": null, "model_count_note": "No aggregate model total is published. The README describes it as an \"Open Source AI Gateway for 100+ LLMs\" and elsewhere as a unified interface to \"100+ LLM providers\" - the same figure used for two different things - while the providers documentation states no total at all and expresses coverage per provider (\"LiteLLM supports all models on VLLM\"). Provider breadth is the meaningful number here.", "provider_count": 140, "token_markup_pct": 0, "credit_fee_pct": 0, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "Self-hosted only: you configure your own upstream provider keys and pay providers directly; no gateway token fee.", "seat_fee_usd_month": 0, "free_tier": "Open-source gateway is $0 forever: 140+ providers, virtual keys, teams, budgets, load balancing, RPM/TPM limits, guardrails.", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "OSS is free (MIT); you pay only for your own containers plus Postgres and Redis. Enterprise features (SSO, RBAC, JWT auth, SCIM, audit logs, support SLAs) require a paid LiteLLM commercial license whose price is not published.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Responses", "OpenAI Embeddings", "OpenAI Images", "OpenAI Audio", "OpenAI Batches", "Rerank", "Anthropic Messages" ], "sdk_note": "Python SDK plus a self-hosted proxy server; any OpenAI-compatible client works by swapping the base URL.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": null, "cap_batch": true, "latency_overhead_ms": 0.66, "latency_source_note": "Vendor-run: LiteLLM's own \"AI Gateway Bench\" reports 0.66 ms added p99 latency and 2,800+ RPS for its Rust gateway (~22 MB RSS), and in the same self-run test puts Portkey at 2.29 ms and Bifrost at 4.54 ms. The GitHub README separately cites 8 ms p95 at 1k RPS. Third-party write-ups report the Python proxy degrading well before that (see weaknesses).", "throughput_rps": 2800, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Self-hosted, so data residency is wherever you deploy; the vendor states self-hosted deployments send no telemetry back.", "zdr": null, "zdr_note": "Nothing transits BerriAI when you self-host, and content logging is off by default, so there is no vendor retention to switch off.", "air_gapped": true, "best_for": "Platform teams that want a free, self-hosted, maximally broad provider abstraction with virtual keys and budgets, at moderate request volumes.", "strengths": [ "Widest provider and model coverage of any gateway here (140+ providers)", "MIT-licensed with no usage fees, plus Helm/Terraform/Docker install paths and air-gapped support", "Virtual keys, team budgets, RPM/TPM limits and guardrails in the free tier", "Response and semantic caching backed by Redis, S3 or GCS", "Very broad API surface: chat, responses, embeddings, images, audio, batches, rerank, MCP" ], "weaknesses": [ "Third-party operators report the Python proxy hitting a GIL bottleneck past ~300 RPS per instance with P99 blowing out to seconds and memory growth to OOM", "Independent 2026 comparison recommends staying under 250-300 RPS per instance and disputes vendor cache/MCP feature claims", "Vendor benchmark numbers are self-run and conflict sharply with rivals' self-run numbers (Kong measured LiteLLM 859% slower than Kong)", "SSO, RBAC, audit logs and SCIM sit behind an unpublished-price commercial license" ], "avoid_if": "You need a single instance to sustain thousands of RPS with tight tail latency, or you need certified compliance artifacts from a vendor rather than your own deployment.", "reviews": [ { "url": "https://dev.to/polar3130/using-gemini-cli-through-litellm-proxy-1627", "why": "Hands-on writeup of putting Gemini CLI behind LiteLLM Proxy on Cloud Run with Cloud SQL, including the operational gotchas found in testing.", "date": "2025-11-25", "type": "written", "title": "Using Gemini CLI Through LiteLLM Proxy", "publisher": "DEV Community - polar3130" }, { "url": "https://www.youtube.com/watch?v=RWDJKIya-WI", "why": "Conference talk by Alina Dallmann on a production LiteLLM deployment for a privacy-preserving university chatbot: proxy setup, Postgres budgets, Redis load balancing, key management.", "date": "2025-11-23", "type": "video", "title": "One API to Rule Them All? LiteLLM in Production", "publisher": "YouTube - PyData" }, { "url": "https://www.youtube.com/watch?v=WyW4Ifu4rSo", "why": "Sudarshan Koirala's hands-on course covering the SDK, proxy server, Docker deploy, admin UI, teams and virtual keys against OpenAI, Ollama and Groq.", "date": "2025-06-02", "type": "video", "title": "LiteLLM Crash Course | For Complete Beginners", "publisher": "YouTube - Data Science Basics" }, { "url": "https://news.ycombinator.com/item?id=47501426", "why": "Supply-chain incident thread with practitioner impact reports and the maintainers' response, including which deployment paths (pinned Docker image) were unaffected.", "date": "2026-03-24", "type": "community", "title": "Tell HN: Litellm 1.82.7 and 1.82.8 on PyPI are compromised", "publisher": "Hacker News" }, { "url": "https://www.reddit.com/r/LocalLLaMA/comments/1mh99hu/litellm_started_breaking_down_for_us_past_300_rps/", "why": "32-comment scaling thread with concrete tuning data: failures past ~300 RPS and an 84.7% P99 reduction (4.9s to 750ms) on 4 CPUs / 4 instances without caching.", "date": "2025-08-04", "type": "community", "title": "LiteLLM started breaking down for us past 300 RPS, what are folks using in prod?", "publisher": "Reddit - r/LocalLLaMA" } ], "sources": { "cap_mcp": "https://github.com/BerriAI/litellm", "company": "https://www.ycombinator.com/companies/litellm", "license": "https://github.com/BerriAI/litellm", "sdk_note": "https://github.com/BerriAI/litellm", "url_docs": "https://docs.litellm.ai/docs/simple_proxy", "cap_audio": "https://github.com/BerriAI/litellm", "cap_batch": "https://github.com/BerriAI/litellm", "free_tier": "https://www.litellm.ai/#pricing", "air_gapped": "https://www.litellm.ai/#pricing", "hq_country": "https://www.ycombinator.com/companies/litellm", "weaknesses": "https://dev.to/debmckinney/litellm-broke-at-300-rps-in-production-heres-how-we-fixed-it-5ej", "cap_budgets": "https://www.litellm.ai/#pricing", "model_count": "https://www.litellm.ai/", "api_surfaces": "https://github.com/BerriAI/litellm", "cap_failover": "https://www.litellm.ai/#pricing", "founded_year": "https://www.ycombinator.com/companies/litellm", "github_stars": "https://github.com/BerriAI/litellm", "product_fate": "https://www.litellm.ai/", "regions_note": "https://www.litellm.ai/enterprise", "cap_image_gen": "https://github.com/BerriAI/litellm", "url_changelog": "https://docs.litellm.ai/release_notes", "vendor_status": "https://www.ycombinator.com/companies/litellm", "byok_supported": "https://www.litellm.ai/#pricing", "cap_embeddings": "https://github.com/BerriAI/litellm", "cap_guardrails": "https://www.litellm.ai/#pricing", "certifications": "https://docs.litellm.ai/docs/data_security", "config_as_code": "https://docs.litellm.ai/docs/proxy/configs", "credit_fee_pct": "https://www.litellm.ai/#pricing", "provider_count": "https://www.litellm.ai/", "retention_days": "https://docs.litellm.ai/docs/proxy/ui_spend_log_settings", "throughput_rps": "https://www.litellm.ai/", "cap_rate_limits": "https://www.litellm.ai/#pricing", "litellm_caching": "https://docs.litellm.ai/docs/proxy/caching", "litellm_pricing": "https://www.litellm.ai/pricing", "logs_by_default": "https://docs.litellm.ai/docs/proxy/ui_logs", "cap_cache_simple": "https://docs.litellm.ai/docs/proxy/prod", "cap_virtual_keys": "https://www.litellm.ai/#pricing", "model_count_note": "https://github.com/BerriAI/litellm", "token_markup_pct": "https://www.litellm.ai/#pricing", "cap_observability": "https://github.com/BerriAI/litellm", "cap_pii_redaction": "https://www.litellm.ai/#pricing", "eu_residency_note": "https://www.litellm.ai/#pricing", "last_release_date": "https://github.com/BerriAI/litellm/releases/tag/v1.98.0", "openai_compatible": "https://docs.litellm.ai/docs/simple_proxy", "cap_cache_semantic": "https://docs.litellm.ai/docs/proxy/prod", "cap_load_balancing": "https://www.litellm.ai/#pricing", "litellm_enterprise": "https://www.litellm.ai/enterprise", "seat_fee_usd_month": "https://www.litellm.ai/#pricing", "security_incidents": "https://docs.litellm.ai/blog/security-update-march-2026", "guardrails_location": "https://docs.litellm.ai/docs/proxy/guardrails/pii_masking_v2", "latency_overhead_ms": "https://www.litellm.ai/", "latency_source_note": "https://www.litellm.ai/", "requires_vendor_sdk": "https://docs.litellm.ai/docs/proxy/user_keys", "self_host_cost_note": "https://www.litellm.ai/#pricing", "litellm_virtual_keys": "https://docs.litellm.ai/docs/proxy/virtual_keys", "data_export_available": "https://docs.litellm.ai/docs/proxy/cost_tracking", "litellm_cost_tracking": "https://docs.litellm.ai/docs/proxy/cost_tracking", "prompt_transits_vendor": "https://docs.litellm.ai/docs/data_security", "cap_conditional_routing": "https://www.litellm.ai/", "trains_on_customer_data": "https://docs.litellm.ai/docs/data_security" }, "verified_at": { "cap_mcp": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "cap_audio": "2026-08-29", "cap_batch": "2026-08-29", "free_tier": "2026-08-29", "air_gapped": "2026-08-29", "hq_country": "2026-08-29", "weaknesses": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-31", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "github_stars": "2026-09-17", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "throughput_rps": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "cap_virtual_keys": "2026-08-29", "model_count_note": "2026-08-31", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-17T18:50:21.761195+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed, MIT-licensed core maintained by BerriAI with a paid enterprise tier layered on the same code rather than a separate build.", "last_release_date": "2026-08-23", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Latest stable release v1.98.0. A v1.100.0 release candidate was tagged on 30 August 2026; release candidates are excluded here.", "prompt_transits_vendor": "no", "prompt_path_note": "Self-hosted: no personal data is collected or transmitted to LiteLLM's servers, and enterprise supports air-gapped deployment so neither your data nor your keys leave your infrastructure.", "logs_by_default": "metadata_only", "logs_note": "Content logging is off by default — only metadata such as tokens, cost and model is stored. Storing request and response bodies requires opting in with store_prompts_in_spend_logs.", "retention_days": null, "retention_note": "No default deletion. Logs persist according to your configuration until you set something like maximum_spend_logs_retention_period.", "zdr_status": "not_applicable", "trains_on_customer_data": "no", "training_note": "True by construction rather than by policy: no data or telemetry reaches LiteLLM's servers when self-hosted. No separate training policy is published.", "subprocessors_url": null, "regions_note": "Your own infrastructure, including air-gapped. Enterprise offers a multi-region control plane.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "docs say Type II in progress with an ETA of 15 Sep 2026; the enterprise page markets it as done", "status": "conflicting" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "status": "not_published" }, "iso27001": { "detail": "marketed on the enterprise page, unverified", "status": "claimed" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "self_hosted", "guardrails_note": "PII masking requires Presidio analyzer and anonymizer containers that you deploy and address locally, so redaction happens entirely inside your boundary.", "acquired_date_basis": null, "security_incidents": [ { "cve": "CVE-2026-33634", "date": "2026-03-24", "what": "Two releases, 1.82.7 and 1.82.8, were published straight to PyPI carrying a credential stealer. Neither version was ever tagged on GitHub — the attacker bypassed the project's CI/CD entirely and uploaded to PyPI directly, using an API token exposed through the compromised Trivy releases in the wider TeamPCP campaign.", "title": "Malicious releases published to PyPI (CVE-2026-33634)", "action": "If you installed LiteLLM from PyPI inside the 24 March window, treat every credential reachable from that host as compromised and rotate it — model keys, cloud credentials, SSH keys and CI tokens. Check for litellm_init.pth in site-packages, a sysmon.service user unit, and outbound traffic to models.litellm[.]cloud or checkmarx[.]zone.", "impact": "The payload collected environment variables, model API keys, SSH keys, AWS, GCP and Azure credentials, Kubernetes tokens, database passwords, shell history and CI/CD secrets, encrypted them with AES-256 under an RSA-4096 key, and posted them to models.litellm[.]cloud — a domain registered the day before the upload and unconnected to the project. It then polled checkmarx[.]zone for follow-on code and persisted via a systemd user unit named sysmon.service.", "status": "resolved", "sources": { "GitHub issue #24518": "https://github.com/BerriAI/litellm/issues/24518", "PyPI incident report": "https://blog.pypi.org/posts/2026-04-02-incident-report-litellm-telnyx-supply-chain-attack/", "LiteLLM security advisory": "https://docs.litellm.ai/blog/security-update-march-2026", "The Hacker News follow-up": "https://thehackernews.com/2026/08/malicious-litellm-releases-tied-to.html", "Datadog Security Labs analysis": "https://securitylabs.datadoghq.com/articles/litellm-compromised-pypi-teampcp-supply-chain-campaign/" }, "severity": "critical", "assessment": "This was a compromise of the distribution channel, not a flaw in the gateway's own code, and the maintainers did not write the malware. It is recorded here because the failure mode it exposes is structural: a self-hosted gateway installed from a public package index inherits that index's supply-chain risk, and this gateway is one that sits directly on your model keys and cloud credentials. Pin versions and verify hashes rather than tracking latest.", "resolution": "PyPI quarantined the project and both versions were removed; they no longer resolve. 1.82.6 and 1.83.0 are unaffected and remain available.", "scale_note": "Package-level download counts were never published. CloudSEK put the campaign's captured dataset at roughly 434,000 files, mapping potential exposure to more than 2,500 organisations — that is the whole TeamPCP campaign, not LiteLLM alone, and the file count is not a deduplicated count of victims.", "window_note": "How long the packages were installable is disputed by the parties involved. LiteLLM says they were live from 10:39 UTC and quarantined after roughly 40 minutes, while PyPI's own incident report puts LiteLLM's exposure at 2 hours 32 minutes from upload to quarantine, and the GitHub issue tracking it records the upload at around 08:30 UTC. LiteLLM's own advice is the safest reading: treat any install between 10:39 and 16:00 UTC on 24 March 2026 as suspect.", "worse_version": "1.82.8 is the more dangerous of the two. It shipped a litellm_init.pth file, which Python executes at interpreter startup, so the stealer ran on any Python process on the machine whether or not anything imported LiteLLM. In 1.82.7 the payload sat in proxy_server.py and only ran when litellm.proxy was imported." } ], "pricing_model": "oss_with_managed_tier", "pricing_model_note": "OSS $0 free forever, self-hosted. Enterprise is talk-to-sales, sized to annual gateway request capacity, deployment architecture and support needs — 'never per token'. Volume discount tiers exist but no minimum amount is published.", "min_commitment": "Enterprise is annual term; no stated minimum amount.", "overage_note": "None stated for OSS or Enterprise. Practical exposure is self-hosted infrastructure and database cost.", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact-match and semantic self-hosted caches: In-Memory, Disk, Redis, Qdrant Semantic, Redis Semantic, Valkey Semantic, S3 bucket, GCS bucket. LiteLLM prices no tokens; provider cached-token rates flow through in its cost calculations. Saving is 100% the customer's; the cost you add is running Redis/Qdrant yourself.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated on the pages fetched.", "tier": null, "supported": null }, "virtual_keys": { "note": "Free. /key/generate accepts models, user_id, team_id, metadata, aliases, budgets and limits.", "tier": "oss", "supported": true }, "rate_limiting": { "note": "Free, pre-request. tpm_limit, rpm_limit, max_parallel_requests, /key/block and /key/unblock.", "tier": "oss", "supported": true }, "spend_webhooks": { "note": "Not stated on the pages fetched.", "tier": null, "supported": null }, "model_allowlists": { "note": "Free. Keys specify allowed models, evaluated against the key row itself.", "tier": "oss", "supported": true }, "budget_caps_per_key": { "note": "Free, pre-request. max_budget + budget_duration, with admin-side upperbound_key_generate_params.", "tier": "oss", "supported": true }, "budget_caps_per_team": { "note": "Teams carry max_budget; owner budgets apply to keys with no team.", "tier": "oss", "supported": true } }, "cost_attribution": "The strongest of any product here: spend logged per hashed api_key, internal user, end-user/customer, team_id, tags, model_group, api_base, User-Agent (tracked as a tag) and arbitrary custom headers/metadata.", "cost_export": "API + database: /spend/logs, /user/info, /user/daily/activity, /gateway/daily/activity, /global/spend/report?group_by=customer|team|api_key|internal_user, x-litellm-response-cost header, and the LiteLLM_SpendLogs table you own. CSV, webhook, S3 and warehouse export not stated as first-class features.", "product_kind_note": "Titled \"LiteLLM AI Gateway (LLM Proxy)\" and elsewhere \"LiteLLM Proxy (LLM Gateway)\", it is a self-hosted proxy that maps many provider APIs to OpenAI-format input/output ([LiteLLM AI Gateway](https://docs.litellm.ai/docs/simple_proxy), [LiteLLM providers](https://docs.litellm.ai/docs/providers)). Not a model platform: \"input, output and exceptions are mapped to the OpenAI format for all supported models\" ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "deployment_modes": [ "self_host", "hybrid_vpc", "air_gapped" ], "deployment_note": "Official images publish to `ghcr.io/berriai` mirrored at `docker.litellm.ai/berriai` (e.g. `docker.litellm.ai/berriai/litellm:v1.90.2`); the `litellm-helm` chart covers monolithic deployments on EKS/GKE/AKS and a componentized `litellm` chart splits `gateway` (LLM traffic, port 4000), `backend` (management API/UI, 4001) and `ui` (3000); official Terraform modules exist for AWS ECS Fargate and Google Cloud Run, and plain Kubernetes manifests are documented ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)). Licensing is a flag, not a build: `litellm_license` is \"optional, omit for open source\" ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)); Enterprise adds SSO+SCIM and support, self-served as \"an instant key, emailed to you\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap: \"Just change the `base_url`, `api_key` and `model`… LiteLLM Proxy is OpenAI-Compatible, it works with any project that calls OpenAI\", and it is also Azure OpenAI-, Anthropic- and Vertex AI-compatible (`/chat/completion`, `/embedding` for Vertex) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)). No proprietary SDK required — documented clients are the OpenAI SDK, LangChain, LlamaIndex, Instructor and curl ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys), [LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)). Streaming works with `\"stream\": true` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); Assistants and Fine-Tuning endpoints are also passed through ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a.: no aggregate model total is stated. The README says \"100+ LLMs\"; the providers page states no total and lists coverage per provider ([LiteLLM providers](https://docs.litellm.ai/docs/providers); [BerriAI/litellm](https://github.com/BerriAI/litellm)).", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "n.a.: the providers index states no total, though it enumerates a long list of provider sections (OpenAI, Azure OpenAI, Azure AI, Vertex AI, Bedrock, HuggingFace, Nvidia NIM, Perplexity, RunwayML, WatsonX, xAI, Databricks, Snowflake Cortex, Together AI, vLLM, Ollama and many more) ([LiteLLM providers](https://docs.litellm.ai/docs/providers), undated).", "first_party_vs_partner": "All third-party or customer-run: LiteLLM ships no models and integrates provider APIs plus self-hosted servers (vLLM, Ollama, LM Studio, Llamafile, Lemonade, Nvidia Riva, custom torch-serve) ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).", "custom_endpoint_support": "Yes: \"Call your custom torch-serve / internal LLM APIs via LiteLLM\", with documented support for vLLM (\"all models on VLLM\"), Ollama (\"all models from Ollama\"), AWS SageMaker (\"All Sagemaker Huggingface Jumpstart Models\"), LM Studio, Llamafile, and OpenAI-compatible endpoints selected by choosing `openai` as provider with an upstream URL ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).", "byok_mode": "byok_only", "merchant_of_record": "n.a. as an explicit statement: the LiteLLM pages fetched do not state who invoices model usage; LiteLLM Enterprise sells the software licence while the deployment runs on the customer's infrastructure and keys ([LiteLLM Enterprise](https://www.litellm.ai/enterprise), [LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "key_management_note": "Keys stay in the customer's deployment: Kubernetes manifests use a Secret for keys and a ConfigMap for `config.yaml` ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)), and Enterprise adds secret-manager integrations — AWS Secrets Manager-style vaults, Google KMS, Google Secret Manager, HashiCorp Vault, CyberArk and custom secret managers — plus key rotations and read/write to the secret manager ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Access is organized as \"Organizations → Teams → Projects → Keys\"; OSS includes request/response logging, Enterprise adds per-key/per-team routing to Langfuse, Langsmith and Arize, log export to GCS/Azure Blob, and \"disable logging per team — GDPR-friendly opt-out at the team level\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Since it is self-hosted, no vendor plane sees request content by default ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).", "api_openai_chat_completions_note": "Yes: `/chat/completions` (examples use `curl http://0.0.0.0:4000/v1/chat/completions`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "api_anthropic_messages_note": "Yes: \"LiteLLM Proxy is Anthropic-compatible\" with `/messages` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "api_openai_responses_note": "Yes, referenced as a supported surface: Manus agents are called \"through LiteLLM's OpenAI-compatible Responses API\" ([LiteLLM providers](https://docs.litellm.ai/docs/providers)). Explicit `/responses` path: n.a.", "api_embeddings_note": "Yes: `/embeddings` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "api_images_note": "Yes: `/image/generations` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); provider-level image generation and editing are documented for Black Forest Labs, Fal AI and RunwayML ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).", "api_audio_note": "Yes, both directions: `/audio/transcriptions` (STT) and `/audio/speech` (TTS) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); provider examples include Deepgram's `/listen` and Nvidia Riva transcription ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).", "api_batch_note": "Yes: \"Batches API endpoints\" are exposed, plus a separate beta \"batch completions\" that fans one request out to N models via a comma-separated model string (`\"model\"=\"llama3,gpt-3.5-turbo\"`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "deployment_modes_note": "self-host via Docker, Helm, Terraform, Kubernetes, or a local binary ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy), [LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); hybrid/VPC in the customer's own cloud (the AWS Terraform module reuses an existing VPC, subnets, database and Redis) ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)); air-gapped on Enterprise — \"self-hosted or air-gapped, and yours to run\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Hosted SaaS: n.a. (not documented on the pages fetched).", "byok_mode_note": "byok_only: the proxy is configured with the customer's own upstream provider keys, and it can additionally \"allow users to pass in their OpenAI API key\"; callers authenticate with LiteLLM virtual keys (e.g. `sk-test-proxy-key-123`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).", "access_meta": { "qualified": [ "api_openai_responses", "api_audio" ], "model_count_kind": null, "provider_count_kind": null }, "rel_timeout_config": "per_request", "rel_timeout_note": "`per_request` and `config_file`. Config: `litellm_settings.request_timeout: 10` (\"raise Timeout error if call takes longer than 10s\") ([LiteLLM reliability](https://docs.litellm.ai/docs/proxy/reliability)); `router_settings.timeout: 30`, per-deployment `litellm_params.timeout: 300`, and per-request `\"timeout\": 1` in the JSON body (or `extra_body={\"timeout\": 1}` from OpenAI SDKs). Streaming has its own key `stream_timeout` (first-chunk deadline). No default value is stated ([LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).", "rel_retry_config": "per_request", "rel_retry_note": "`per_request` and `config_file`. `num_retries` (e.g. `litellm_settings.num_retries: 3`), with precedence `x-litellm-num-retries` header → request body → deployment `litellm_params` → `litellm_settings`. Default count is not stated: `n.a.` Backoff: exponential for `RateLimitError`, immediate retry for generic errors; minimum delay configurable via `retry_after`. Per-error counts via `RetryPolicy` (e.g. `TimeoutErrorRetries: 2`, `RateLimitErrorRetries: 3`). Note `max_retries` is the provider SDK's internal counter and is forced to `0` on router paths to avoid `(1 + N) ** 2` upstream calls ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).", "rel_retry_default_note": null, "rel_fallback_config": "config_file", "rel_fallback_note": "`config_file` (also per-request). ORDERED: `fallbacks: [{\"gpt-3.5-turbo\": [\"gpt-4\"]}]`, and \"Fallbacks are done in-order\". Specialized variants: `default_fallbacks`, `content_policy_fallbacks`, `context_window_fallbacks`. Fallback fires only after `num_retries` is exhausted ([LiteLLM reliability](https://docs.litellm.ai/docs/proxy/reliability)). WEIGHTED failover also exists inside a model group via `enable_weighted_failover` (requires `routing_strategy=\"simple-shuffle\"`, uses `weight`/`rpm`/`tpm`, capped by `max_fallbacks`, default `5`; async entrypoints only) ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).", "rel_fallback_shape": "weighted", "rel_health_check_config": "config_file", "rel_health_check_note": "`config_file`, and it is cooldown-based ejection (LiteLLM does not call it circuit breaking): `allowed_fails` (default `3`), `cooldown_time` (default `5s`), `disable_cooldowns`, plus `AllowedFailsPolicy` per error type. Cooldown triggers: `429`, >50% failures in the current minute, and non-retryable `401`/`404`/`408`. Cooled deployments are removed from the pool and automatically reintroduced ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).", "rel_multi_region_config": "config_file", "rel_multi_region_note": "\"LiteLLM will handle retrying in different regions if a call fails\", with `region_name` (e.g. `\"eu\"`) auto-inferred for Vertex AI, Bedrock and WatsonX, and weighted failover documented across Azure regional copies (`eastus2`, `swedencentral`). LiteLLM does not run the regions itself — you register per-region deployments ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Deployments sharing a `model_name` form a load-balanced model group; `routing_strategy` selects the algorithm (e.g. `\"least-busy\"`, `\"simple-shuffle\"`), and weights are supported through `weight`/`rpm`/`tpm` in the weighted-failover path ([LiteLLM routing](https://docs.litellm.ai/docs/routing), [LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).", "architecture_class": "interpreted_proxy", "architecture_note": "Currently in transition and the repo states both: the repository description is \"The fastest, litest AI Gateway. Rust core with Python SDK\", while the language breakdown is Python 84%, TypeScript 13.2%, Rust 0.7%. Best single label from the evidence: `interpreted_proxy` with a stated Rust core ([BerriAI/litellm](https://github.com/BerriAI/litellm)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Docker images `ghcr.io/berriai/litellm:` (cosign-signed; `-stable` tags after 12-hour load tests) plus componentized `litellm-gateway`/`-backend`/`-ui` images, Helm charts `litellm-helm` and `litellm` (`oci://ghcr.io/berriai/litellm-helm`), and Terraform modules for AWS ECS Fargate and GCP Cloud Run ([BerriAI/litellm](https://github.com/BerriAI/litellm), [LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).", "latency_claims": [ { "url": "https://github.com/BerriAI/litellm", "unit": "ms", "scope": "gateway_overhead", "value": "8", "metric": "p95", "publisher": "vendor", "conditions": "At 1k RPS. The README does not say whether this is added overhead or end-to-end, and gives no payload or hardware." } ], "throughput_claims": [ { "url": "https://github.com/BerriAI/litellm", "unit": "RPS", "scope": "capacity", "value": "1,000", "metric": "load condition", "publisher": "vendor", "conditions": "Stated only as the load the 8ms p95 figure was taken at; hardware not stated." } ], "streaming_support": "yes", "streaming_note": "Supported via `\"stream\": true`. Documented streaming machinery/caveats: `stream_timeout` bounds only the first chunk and can be used to abort hanging providers and retry another model; SSE keepalives via `keepalive_seconds` (clamped 1–300 s; `0` at deployment level is a hard disable) or proxy-wide `sse_keepalive_ping_interval_seconds`, sending `: ping` comment frames; client-side override requires `allow_client_keepalive_override: true` (header `x-litellm-keepalive-seconds`) ([LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).", "guard_pii": "sync_block", "guard_pii_note": "Presidio integration supports per-entity `MASK` or `BLOCK`, runs in `pre_call` mode, and can skip system/tool messages; masked data is forwarded downstream with `pass_data: true` ([Guardrails quick start](https://docs.litellm.ai/docs/proxy/guardrails/quick_start), [Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))", "guard_injection": "sync_block", "guard_injection_note": "`sync_block` via third parties — e.g. Lakera configured as `guardrail: lakera, mode: pre_call`, plus PromptGuard, Gray Swan, and others; detection is delegated to the vendor's classifier ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder), [Guardrail providers](https://docs.litellm.ai/docs/guardrail_providers))", "guard_content": "sync_block", "guard_content_note": "`sync_block` — moderation guardrails (e.g. OpenAI moderation, Azure Content Safety, Bedrock Guardrails) block on policy failure such as toxic content ([Guardrail providers](https://docs.litellm.ai/docs/guardrail_providers), [Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))", "guard_custom": "sync_block", "guard_custom_note": "Policies are declared in YAML pipelines with `on_pass`, `on_fail`, `on_error` branches per step (`allow`, `block`, `next`, `modify_response` with a custom message), and custom guardrail classes can be registered; key-level `permissions` control which guardrails apply ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder), [Guardrails quick start](https://docs.litellm.ai/docs/proxy/guardrails/quick_start))", "guard_model_policy": "yes", "guard_model_policy_note": "`models` allowlists on keys and teams restrict callable models ([Model access](https://docs.litellm.ai/docs/proxy/model_access))", "guard_execution_location": "both", "guard_third_party": [ "Microsoft Presidio", "AWS Bedrock Guardrails", "Azure Content Safety", "OpenAI Moderation", "Lakera", "Aporia", "Pangea", "Guardrails AI", "Pillar Security" ], "guard_failure_mode": "configurable", "guard_failure_mode_note": "**Configurable, defaults to the policy branch**: technical errors (timeouts, network errors, guardrail not loaded, connection failure) are handled by `on_error`, and \"If `on_error` is omitted, the pipeline uses `on_fail` for error outcomes\". The docs' own example fails open: \"If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed)\" ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))", "log_content": "configurable", "log_content_note": "\"Store Prompts in Spend Logs\" controls whether request messages and response content are stored; when disabled only metadata (tokens, cost, model) is kept ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))", "log_opt_out": "yes", "log_opt_out_note": "`yes` — leave \"Store Prompts in Spend Logs\" off or set `general_settings.store_prompts_in_spend_logs: false`; additionally `turn_off_message_logging`, per-request header `x-litellm-enable-message-redaction`, and the `no-log` flag suppress message content on callbacks ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings), [Logging](https://docs.litellm.ai/docs/proxy/logging))", "tracing_support": "otel", "tracing_note": "OpenTelemetry is a first-class callback, alongside Langfuse, Arize, Langtrace, MLflow, and Langsmith for chain/agent-level traces ([Logging](https://docs.litellm.ai/docs/proxy/logging))", "export_destinations": [ "OpenTelemetry", "Datadog", "Langfuse", "Langsmith", "Arize Phoenix", "Prometheus", "S3", "GCS", "Athina", "Braintrust", "Helicone", "Lago", "Logfire", "Sentry", "Slack", "Webhook", "Galileo", "OpenMeter", "DynamoDB", "Azure Blob Storage" ], "export_destinations_note": "Langfuse, OpenTelemetry, GCS buckets, Amazon S3, Azure Blob Storage, GCS Pub/Sub, AWS SQS, DynamoDB, Datadog, Azure Sentinel, Lunary, MLflow, Deepeval, Langsmith, Arize, Langtrace, Galileo, OpenMeter, Sentry, Athina ([Logging](https://docs.litellm.ai/docs/proxy/logging))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — no native feedback/score endpoint on the fetched pages; scoring is delegated to callback platforms ([Logging](https://docs.litellm.ai/docs/proxy/logging))", "eval_hooks": "partial", "eval_hooks_note": "Indirectly, and without LiteLLM SaaS: Deepeval, MLflow, Langsmith, Galileo, Athina and Arize callbacks receive traffic for offline/online evaluation in those tools ([Logging](https://docs.litellm.ai/docs/proxy/logging))", "observability_requires_saas": "No — the proxy is self-hosted, spend logs live in the customer's database, and callbacks ship to destinations the customer chooses ([Logging](https://docs.litellm.ai/docs/proxy/logging), [Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))", "log_retention_note": "Configurable via `maximum_spend_logs_retention_period` (e.g. `7d`, `30d`) settable from the Admin UI without a restart; if unset, logs follow config/default behaviour and no fixed default is documented ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "One published figure: **`8ms P95 latency` at `1k RPS`** (metric: P95; the README does not state whether this is gateway overhead or end-to-end, and gives no payload, hardware or cache conditions — conditions not stated; it links to a benchmarks page) ([BerriAI/litellm](https://github.com/BerriAI/litellm)).", "multi_surface": { "rel_retry": [ "per_request", "config_file" ], "rel_timeout": [ "per_request", "config_file" ] }, "throughput_prose": "**`1k RPS`**, stated only as the load condition attached to the 8 ms P95 figure; hardware not stated ([BerriAI/litellm](https://github.com/BerriAI/litellm)).", "benchmark_independence": "The 8 ms / 1k RPS figure is vendor self-published ([BerriAI/litellm](https://github.com/BerriAI/litellm)). **Flag:** the widely circulated LiteLLM latency numbers (P50 38.65 s, ~40 ms overhead, 44.84 req/s) are published by a RIVAL, Maxim AI's Bifrost, not by LiteLLM ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)) — they are not comparable to LiteLLM's own claim and should not be reported as LiteLLM data." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "choose_at_setup", "guard_custom_default": "choose_at_setup", "rel_notes": "Deepest reliability surface of any self-hostable option here (retry precedence chain, per-error-type retry and failure policies, cooldowns, ordered + weighted failover, region inference), and the docs explicitly warn about retry amplification if the provider SDK's own `max_retries` is left on ([LiteLLM routing](https://docs.litellm.ai/docs/routing)). Kubernetes probes are documented at `/health/readiness` and `/health/liveliness` ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).", "guard_notes": "LiteLLM is the only product in this set that gives an explicit, documented three-way distinction between pass, policy failure, and technical error, with independent branch actions for each — the cleanest fail-open/fail-closed control available ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder)).", "fit_agent_features_note": "LiteLLM supports OpenAI-compatible tool/function calls for MCP tools and can automatically execute returned MCP tool calls when `require_approval: \"never\"`, then feed the results back to the model. Multi-step tool-result-to-model flow is documented through `litellm.experimental_mcp_client.call_openai_tool`; sessions, threads, and A2A are not documented. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))", "fit_byok_required": "no", "fit_byok_required_note": "No for the documented first call: it uses `huggingface/bigcode/starcoder` and states `HUGGINGFACE_API_KEY` is \"[OPTIONAL]\" ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start)). Production secrets do include a provider key, e.g. `--from-literal=OPENAI_API_KEY=\"\"` ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))", "fit_client_sdk_langs": [ "Python" ], "fit_client_sdks_note": "Documented clients and integrations include the OpenAI SDK with `openai.OpenAI`, the Anthropic SDK with `anthropic.Anthropic`, the LiteLLM Python SDK, Mistral SDK, LlamaIndex, and LangChain for JavaScript and Python. The quick-start examples also show `langchain.chat_models.ChatOpenAI` and `langchain.embeddings.OpenAIEmbeddings`. ([LiteLLM proxy quick start](https://docs.litellm.ai/docs/proxy/quick_start))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "For AWS Bedrock AgentCore MCP servers, LiteLLM supports AWS SigV4 and can use AWS access keys or the boto3 credential chain, including IAM roles and environment variables. The documented configuration includes `auth_type: \"aws_sigv4\"`, `aws_region_name`, and `aws_service_name: bedrock-agentcore`. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))", "fit_cloudflare_workers": "upstream_model_provider", "fit_cloudflare_workers_note": "LiteLLM documents Cloudflare Workers AI support using the `cloudflare/` model prefix, including `cloudflare/@cf/meta/llama-2-7b-chat-int8` and `cloudflare/@hf/thebloke/codellama-7b-instruct-awq`. ([LiteLLM Cloudflare Workers AI provider](https://docs.litellm.ai/docs/providers/cloudflare_workers))", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented) on either page; no enterprise/licence requirement is stated and `litellm_license` is optional in the Terraform examples ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start), [Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))", "fit_ecosystem_notes": "LiteLLM is documented as either a Proxy Server or a Python SDK; the Proxy Server can run via the LiteLLM CLI or Docker and listens at `http://0.0.0.0:4000`. The product documentation also describes self-hosted deployment on AWS, Google Cloud, Azure, and Kubernetes clusters. ([LiteLLM documentation home](https://docs.litellm.ai/))", "fit_eval_prereqs": "No account, credit card, cloud account, cluster, or provider key is stated; the Hugging Face key in the example is explicitly \"[OPTIONAL]\". Python 3.10+ is required for LiteLLM 1.84.0+ ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `POST http://0.0.0.0:4000/chat/completions` with JSON containing `\"model\"` and `\"messages\"` ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "litellm-helm at oci://ghcr.io/berriai/litellm-helm — the GitHub page describes it as community maintained", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "LiteLLM deployment documentation lists the Helm charts `litellm-helm` at `oci://ghcr.io/berriai/litellm-helm` and `litellm` at `oci://ghcr.io/berriai/litellm/chart/litellm`. ([LiteLLM deployment documentation](https://docs.litellm.ai/docs/proxy/deploy))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "LiteLLM Proxy provides an MCP Gateway with endpoints including `/mcp-rest/tools/list` and `/mcp-rest/tools/call`. It supports listing and calling tools, prompts, resources, Streamable HTTP, SSE, and stdio, with MCP permissions by Key, Team, Organization, and end user. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))", "fit_meta": { "not_documented_fields": [ "fit_credit_card", "fit_gateway_platform", "fit_model_gate", "fit_vercel_ai_sdk" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented) ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))", "fit_onboarding_notes": "Version trap documented: on Python 3.9 pip silently resolves to 1.83.9 without an error, so you must check `python --version`, upgrade to 3.10+ and reinstall; `uv tool install` provisions a compatible Python automatically. Config via `your_config.yaml` and `litellm --config your_config.yaml`. Production adds `LITELLM_SALT_KEY` (set once, never change), a `/health/readiness` check, and an Admin UI at `/ui`; the proxy claims \"1.5k+ requests/second during load tests\" ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start), [Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. LiteLLM documents OpenAI Python SDK drop-in usage with `api_key=\"anything\"` and the exact proxy base URL `base_url=\"http://0.0.0.0:4000\"`, using `openai.OpenAI`. ([LiteLLM documentation home](https://docs.litellm.ai/))", "fit_prod_prereqs": "A Kubernetes cluster (EKS, GKE, AKS or other), external PostgreSQL (required for authentication/tracking) and Redis (required for multiple instances), cloud ingress/load balancer, and secrets management; the Terraform path provisions networking, databases, Redis, object storage, secrets, load balancer and migrations (AWS: Aurora PostgreSQL, ElastiCache, S3, ALB; GCP: Cloud SQL, Memorystore, GCS, HTTPS load balancer) ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "LiteLLM documents integrations with LangChain and LlamaIndex. Examples include LangChain classes such as `ChatOpenAI` and `OpenAIEmbeddings`, and LlamaIndex classes including `AzureOpenAI`, `AzureOpenAIEmbedding`, `VectorStoreIndex`, and `SimpleDirectoryReader`. ([LiteLLM user keys and integrations](https://docs.litellm.ai/docs/proxy/user_keys))", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "The getting-started page is prose plus commands rather than a numbered procedure.", "fit_quickstart_url": "https://docs.litellm.ai/docs/proxy/quick_start", "fit_self_host_docs": "command_published", "fit_self_host_install": "`uv tool install 'litellm[proxy]'` / `pip install 'litellm[proxy]'` for local ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start)); production images `ghcr.io/berriai/litellm` and `docker.litellm.ai/berriai/litellm:v1.90.2`, plus `helm install litellm oci://ghcr.io/berriai/litellm-helm -f values.yaml` and `helm upgrade --install litellm oci://ghcr.io/berriai/litellm/chart/litellm --version 1.89.2 -f values.yaml`, with `kubectl create secret generic litellm-masterkey|litellm-db|litellm-env ...` ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))", "fit_terraform": "official_modules", "fit_terraform_note": "LiteLLM deployment documentation lists official Terraform modules `BerriAI/litellm/aws` and `BerriAI/litellm/google`, along with the resource-management provider `terraform-provider-litellm`. ([LiteLLM deployment documentation](https://docs.litellm.ai/docs/proxy/deploy))", "fit_terraform_ref": "BerriAI/litellm/aws and BerriAI/litellm/google, plus a terraform-provider-litellm", "fit_third_party_documented": { "kubernetes": "community maintainers, per the chart's own GitHub page" }, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "No official Vercel AI SDK provider package is documented; instead LiteLLM documents Vercel AI Gateway as an upstream provider route (`vercel_ai_gateway/`), and Vercel documents LiteLLM as a framework integration using the Python `litellm` package. ([Vercel AI Gateway LiteLLM framework integration](https://vercel.com/docs/ai-gateway/ecosystem/framework-integrations/litellm))", "fit_vercel_ai_sdk_package": null }, { "slug": "llmgateway", "name": "LLM Gateway", "company": "Polar Lights LLC", "tagline": "AGPL-licensed OpenAI-compatible gateway available as one self-hosted Docker image or a hosted service with credit fees.", "category": "open_source", "deployment": "both", "license": "AGPL-3.0", "founded_year": null, "hq_country": "United States", "url_home": "https://llmgateway.io/", "url_docs": "https://docs.llmgateway.io", "url_pricing": "https://llmgateway.io/pricing", "url_status": "https://status.llmgateway.io", "url_github": "https://github.com/theopenco/llmgateway", "url_changelog": null, "url_models_api": null, "github_stars": 1643, "model_count": 200, "model_count_note": "Vendor-stated 200+ models across 40+ providers on the pricing page; the open-source page cites 280+ models and 35+ providers, and new models are added within 48 hours.", "provider_count": 40, "token_markup_pct": 0, "credit_fee_pct": 5, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "BYOK included on the free plan at no fee; self-hosted requests carry no credit fee and no BYOK metering.", "seat_fee_usd_month": 0, "free_tier": "$0 forever: no seats or minimums, BYOK free, 3 free models limited to 20 req/min, 30-day data retention; hosted credits carry a 5% platform fee.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Full request/response payload retention", "amount": "$0.01 per 1M tokens" }, { "label": "Credit purchases on the hosted service", "amount": "5% platform fee" } ], "self_host_cost_note": "The whole platform (gateway, API, dashboard, worker) ships as one Docker image, ghcr.io/theopenco/llmgateway-unified:latest, free forever under AGPLv3 with no credit fee or BYOK metering; the same codebase powers the managed cloud. Code in the repo's ee/ directory requires a separate commercial enterprise license (contact@llmgateway.io).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": null, "config_as_code": false, "portability_note": "Activity page is view-only; no documented export.", "api_surfaces": [ "OpenAI Chat Completions" ], "sdk_note": "Drop-in OpenAI SDK compatibility by pointing the base URL at https://api.llmgateway.io/v1.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No proxy-overhead benchmark published by the vendor and no independent benchmark found; routing is described as scored on live uptime, throughput, price and latency.", "throughput_rps": null, "soc2": true, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Enterprise offers region pinning and routing restricted to providers meeting SOC 2 / ISO / GDPR requirements, but no EU residency guarantee is stated.", "zdr": null, "zdr_note": "Not branded as ZDR. The gateway blocks requests that would reach a non-conforming provider before any data leaves it, and the DevPass and chat tiers are always metadata-only with no way to enable payload storage.", "air_gapped": null, "best_for": "Small teams that want an OpenRouter-style hosted gateway with a genuine zero-cost self-host escape hatch and no token markup.", "strengths": [ "No token markup: you pay each provider's own rates, with a flat 5% only on hosted credit purchases", "Entire platform self-hosts from a single Docker image under AGPLv3, free forever", "Free plan has no seats or minimums and includes BYOK at no fee", "SOC 2 Type II certified despite being a very small operation", "Enterprise tier covers SSO/SAML/SCIM, guardrails, immutable SIEM-ready audit logs and provider compliance routing" ], "weaknesses": [ "Tiny project by comparison (601 GitHub stars, latest tagged release v1.2.0 from September 2025) and operated by a single small Delaware LLC", "Publishes no latency or throughput benchmarks, and no independent benchmark exists", "Model and provider counts are inconsistent between its pricing and open-source pages", "Advanced governance (guardrails, audit logs, region pinning, SLA) is gated behind an unpriced enterprise license plus the ee/ code license" ], "avoid_if": "You need proven scale, published performance data, or vendor depth to support a large regulated deployment.", "reviews": [ { "url": "https://www.youtube.com/watch?v=6mBV7R3ZDmw", "why": "Independent developer show demos llmgateway.io live with its founders: self-hosting, provider keys, base-URL swap, local models and Docker deployment.", "date": "2025-11-10", "type": "video", "title": "DevNTell - Unifying AI Model Access with LLM Gateway feat. Co-Founders Ismail & Luca", "publisher": "YouTube - DevNTell (Developer DAO)" } ], "sources": { "soc2": "https://llmgateway.io/", "company": "https://llmgateway.io/about", "license": "https://github.com/theopenco/llmgateway", "sdk_note": "https://llmgateway.io/", "url_docs": "https://docs.llmgateway.io", "free_tier": "https://llmgateway.io/pricing", "extra_fees": "https://llmgateway.io/pricing", "hq_country": "https://llmgateway.io/about", "url_status": "https://status.llmgateway.io", "zdr_status": "https://llmgateway.io/blog/soc2-type-ii", "cap_budgets": "https://llmgateway.io/pricing", "model_count": "https://llmgateway.io/pricing", "api_surfaces": "https://llmgateway.io/", "cap_failover": "https://llmgateway.io/about", "github_stars": "https://github.com/theopenco/llmgateway", "product_fate": "https://github.com/theopenco/llmgateway/releases", "regions_note": "https://llmgateway.io/legal/privacy", "byok_fee_note": "https://llmgateway.io/blog/open-source-openrouter-alternatives", "vendor_status": "https://llmgateway.io/about", "byok_supported": "https://llmgateway.io/pricing", "cap_guardrails": "https://llmgateway.io/pricing", "certifications": "https://llmgateway.io/blog/soc2-type-ii", "config_as_code": "https://docs.llmgateway.io/custom-providers", "credit_fee_pct": "https://llmgateway.io/pricing", "provider_count": "https://llmgateway.io/pricing", "retention_days": "https://docs.llmgateway.io/features/data-retention", "cap_rate_limits": "https://llmgateway.io/pricing", "logs_by_default": "https://docs.llmgateway.io/features/data-retention", "cap_cache_simple": "https://llmgateway.io/about", "model_count_note": "https://llmgateway.io/open-source", "token_markup_pct": "https://llmgateway.io/pricing", "cap_observability": "https://llmgateway.io/about", "cap_pii_redaction": "https://llmgateway.io/pricing", "eu_residency_note": "https://llmgateway.io/pricing", "last_release_date": "https://github.com/theopenco/llmgateway/releases/tag/v1.14.0", "openai_compatible": "https://llmgateway.io/", "subprocessors_url": "https://llmgateway.io/blog/soc2-type-ii", "cap_load_balancing": "https://llmgateway.io/about", "llmgateway_pricing": "https://llmgateway.io/pricing", "seat_fee_usd_month": "https://llmgateway.io/pricing", "guardrails_location": "https://llmgateway.io/blog/soc2-type-ii", "latency_source_note": "https://llmgateway.io/blog/open-source-openrouter-alternatives", "requires_vendor_sdk": "https://docs.llmgateway.io/", "self_host_cost_note": "https://llmgateway.io/open-source", "prompt_transits_vendor": "https://llmgateway.io/legal/privacy", "cap_conditional_routing": "https://llmgateway.io/pricing", "trains_on_customer_data": "https://llmgateway.io/legal/privacy" }, "verified_at": { "soc2": "2026-08-29", "company": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "free_tier": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "github_stars": "2026-09-17", "byok_fee_note": "2026-08-29", "byok_supported": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-17T18:50:21.789735+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed with weekly minor releases through August 2026. Operated by Polar Lights LLC, a Delaware company; no funding or ownership information is published.", "last_release_date": "2026-08-24", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "depends", "prompt_path_note": "Hosted, where it sits between your application and every provider, or self-hosted with data in your own PostgreSQL. One disclosure worth reading: the hosted service may route some requests through stealth providers whose identity is not publicly disclosed.", "logs_by_default": "metadata_only", "logs_note": "Metadata-only retention is the default and gives usage analytics without storing payloads.", "retention_days": 30, "retention_note": "Thirty days for all users, custom on enterprise. Stored Responses-API responses are also held 30 days unless you set store to false.", "zdr_status": "depends", "trains_on_customer_data": "no", "training_note": "Customer prompts, inputs and returned responses are not used to train any models.", "subprocessors_url": "https://security.llmgateway.io/", "regions_note": "Processed and stored in the EU or the US, with per-project routing overrides to pin regulated workloads to specific regions and providers. Self-hosting supported.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II, independently examined", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA for enterprise", "status": "available" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "both", "guardrails_note": "Prompt-injection detection, PII redaction and content moderation are enforced at the gateway before any prompt leaves your boundary — which in a self-hosted install is literally your boundary.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "hybrid", "pricing_model_note": "Free OSS self-host (AGPLv3) + credit top-up fee on the hosted platform (5% flat, +1.5% international card fee) + optional storage charge + Enterprise quote. No seats, no minimums, no subscription. Provider rates passed through 0% markup. BYOK 0% fee.", "min_commitment": "None — 'no minimums, no subscription'. Enterprise is custom with volume discounts and a 30-day trial.", "overage_note": "The only after-the-fact charge is storage: full data retention bills stored requests at $0.01 per 1M tokens; metadata retention is free. Free-plan retention is 30 days; Enterprise is unlimited. No request-volume overage or export fee stated.", "cache_support": "none", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Prompt caching is offered ('cache prompts for faster responses'); BYOK users 'still get unified analytics, caching, and failover'. Mechanism (exact-match / semantic / provider passthrough) not stated. LLM Gateway's only fee is 5% on credit purchases (0% on BYOK), so cache savings flow to the customer.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "'Real-time webhook alerts to your channels' (Discord and Slack).", "tier": null, "supported": true }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "20 req/min on 3 free models; paid models unlimited; Enterprise custom. Pre-request.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Real-time webhook alerts.", "tier": null, "supported": true }, "model_allowlists": { "note": "Not stated. Closest are provider compliance policies ('route only to SOC 2 / ISO 27001 / GDPR providers') and region pinning.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated as per-key; the published equivalent is 'cost ceilings per project' plus 'Budgets and spend controls'.", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "'Team management' and 'Admin controls' listed without budget specifics.", "tier": null, "supported": true } }, "cost_attribution": "Per project (cost ceilings and per-project routing overrides) with 'Advanced analytics'. Per key/user/team/tag/customer not stated.", "cost_export": "No CSV/API/webhook/S3/warehouse cost export stated; only alerting webhooks and enterprise audit logs.", "product_kind_note": "LLM Gateway calls itself \"an open-source API gateway for Large Language Models (LLMs)\" acting as \"middleware between your applications and various LLM providers\", routing requests, managing provider keys, and tracking token usage and cost ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Its site frames it as \"One API for 40+ providers\" ([llmgateway.io](https://llmgateway.io/)).", "deployment_modes": [ "saas", "self_host" ], "deployment_note": "Self-hosting is offered under AGPLv3 and is free: \"LLM Gateway is open source under AGPLv3, so you can self-host the gateway for free — or use the hosted platform and pay only the credit fee\" ([LLM Gateway pricing](https://llmgateway.io/pricing)); \"the self-hosted version… ensures your LLM traffic never leaves your infrastructure if desired\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Self-hosted deployments are tuned with environment variables such as `EXPLORATION_RATE`, `PREFERRED_PROVIDER_TTL`, `PREFERRED_PROVIDER_UPTIME_THRESHOLD` and `PREFERRED_PROVIDER_SCORE_MARGIN`; per-project routing customization from the dashboard is gated to the Enterprise plan ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). The dedicated self-hosting doc page failed to load (see gaps).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "not_documented", "api_images": "partial", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Drop-in base-URL swap: \"Just swap out your current LLM provider URL with the LLM Gateway API endpoint\" — \"a single drop-in endpoint\" at `https://api.llmgateway.io/v1/…`, \"maintains compatibility with the OpenAI API format\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21; [quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Streaming: \"pass `stream: true` to any request — Gateway will proxy the event stream unchanged\" ([LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Caveats: the latency routing factor applies only to streaming requests (non-streaming falls back to uptime selection), and a non-`auto` `tool_choice` may be downgraded to `auto` when no eligible mapping can honor it ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). Auth is bearer-token with project-scoped API keys ([LLM Gateway chat completions](https://docs.llmgateway.io/v1_chat_completions), 2026-06-08).", "model_count_low": 200, "model_count_high": 200, "model_count_source_note": "\"Route requests across 200+ models\" ([llmgateway.io](https://llmgateway.io/), undated); docs state no number ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).", "provider_count_low": 40, "provider_count_high": 40, "provider_count_source_note": "\"40+ providers\" ([llmgateway.io](https://llmgateway.io/), undated). Docs pages state no number, referring only to \"multiple LLM providers\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21).", "first_party_vs_partner": "All third-party routed; the gateway \"operates between applications and various LLM providers\" and does not state owning infrastructure ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Region pinning is supported (`provider/model:region`) ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).", "custom_endpoint_support": "Partially documented: \"custom providers\" exist, with keys \"scoped by their own catalogue\", and a customer-specific upstream can be routed (setting the Alibaba Model Studio workspace ID \"routes through the customer's own endpoint\"); no vLLM/Ollama/SageMaker/private-URL registration flow is documented ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).", "byok_mode": "both", "merchant_of_record": "Split by mode: with BYOK \"you pay your providers directly\" and there is \"no platform fee\"; on credits, LLM Gateway invoices, charging \"a flat 5% when you buy credits — no seats, no minimums, no subscription\" (plus a +1.5% international card fee), with volume discounts and negotiated terms on Enterprise ([LLM Gateway pricing](https://llmgateway.io/pricing); [LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).", "key_management_note": "Provider keys are added on a provider-keys page and can be labelled, rotated across multiple keys per provider, and are surfaced per attempt as `providerKeyId` / `providerKeyLabel` with an opaque `apiKeyHash` fingerprint; candidate keys appear as `routingMetadata.eligibleProviderKeys` (omitted in credits mode), and the dashboard badges each call \"your key\" or \"LLM Gateway key\" ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). Caller keys are project-scoped ([LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Storage mechanism (vault/KMS): n.a. ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). On hosted, calls appear in the dashboard with latency, cost and provider breakdown; self-hosting means \"your LLM traffic never leaves your infrastructure if desired\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). The vendor states it is \"SOC 2 Type II Certified\" ([llmgateway.io](https://llmgateway.io/)).", "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` under `https://api.llmgateway.io/v1/…` ([LLM Gateway chat completions](https://docs.llmgateway.io/v1_chat_completions), 2026-06-08; [LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22).", "api_anthropic_messages_note": "Yes: \"the Anthropic Messages endpoint (`/v1/messages`)\" is documented and, per the quickstart, can reach \"any LLM model\" ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27; [LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22).", "api_openai_responses_note": "n.a.", "api_embeddings_note": "n.a. (not documented on the pages fetched).", "api_images_note": "Partially: image-generation models are supported in routing (an \"image price\" weight \"replaces the price weight for image-generation models\"), but no image endpoint path is documented ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).", "api_audio_note": "n.a.", "api_batch_note": "n.a.", "deployment_modes_note": "hosted SaaS (\"Hosted Version: For immediate use without setup, visit llmgateway.io to create an account and get an API key\") and self-host (\"Deploy LLM Gateway on your own infrastructure for complete control over your data and configuration\") ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Hybrid/VPC, on-prem, air-gapped: n.a.", "byok_mode_note": "both, and mixable per request: `credentialSource` is either `byok` (\"your own provider key\" — the provider bills the customer and the attempt \"is not deducted from your credits\") or `platform` (\"an LLM Gateway credential\" — deducted from the credit balance); in hybrid mode a BYOK attempt can fail and fall back to an LLM Gateway credential within the same request ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). \"Bring your own keys — free forever\" ([llmgateway.io](https://llmgateway.io/)).", "access_meta": { "qualified": [], "model_count_kind": "vendor_total", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` across all four fetched pages (docs home, self-host, self-host/docker, repo) ([LLM Gateway docs](https://docs.llmgateway.io/)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([LLM Gateway docs](https://docs.llmgateway.io/)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "`not_documented` as a configurable key. Behavior is asserted without configuration: the gateway \"fails over to a healthy provider when one errors\" — ordering, weighting and key name are all unstated ([LLM Gateway docs](https://docs.llmgateway.io/)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "Only the same \"fails over to a healthy provider when one errors\" statement; no health check, breaker or ejection key ([LLM Gateway docs](https://docs.llmgateway.io/)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Self-hosting on AWS/GCP/Azure and any Kubernetes cluster is documented, but no cross-region failover ([LLM Gateway self-host](https://docs.llmgateway.io/self-host)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "`config_file` at the credential level only: \"LLM Gateway supports multiple API keys per provider for load balancing — provide comma-separated values\" via environment variables. No key name for the policy and no weights ([LLM Gateway Docker self-host](https://docs.llmgateway.io/self-host/docker)).", "architecture_class": "interpreted_proxy", "architecture_note": "`interpreted_proxy` (JS runtime). Repo is TypeScript 95.7%; components are `apps/ui` (Vite + React), `apps/api` (Hono backend) and `apps/gateway` ([theopenco/llmgateway](https://github.com/theopenco/llmgateway)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Single unified Docker container bundling UI, API, gateway, PostgreSQL and Redis; Docker Compose for per-service scaling; an \"official Helm chart\" for EKS/GKE/AKS. Image name and chart repo are not stated ([LLM Gateway self-host](https://docs.llmgateway.io/self-host), [LLM Gateway Docker self-host](https://docs.llmgateway.io/self-host/docker)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "not_documented", "streaming_note": "`not_documented` on the fetched pages; no streaming statement or caveat found ([LLM Gateway docs](https://docs.llmgateway.io/)).", "guard_pii": "not_documented", "guard_pii_note": "Data handling is a retention setting (`retentionLevel: none` avoids storing sensitive content), not a request-path scanner ([Data retention](https://docs.llmgateway.io/features/data-retention), [Logging and analytics](https://deepwiki.com/theopenco/llmgateway/5.3-logging-and-analytics))", "guard_injection": "not_documented", "guard_injection_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))", "guard_content": "not_documented", "guard_content_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))", "guard_custom": "not_documented", "guard_custom_note": "Organisation Policies cover data handling and provider routing rather than content rules ([Policies](https://docs.llmgateway.io/learn/policies), [Audit logs](https://docs.llmgateway.io/features/audit-logs))", "guard_model_policy": "yes", "guard_model_policy_note": "`yes`, at provider granularity — routing can be restricted to compliant providers via organisation policy ([Audit logs](https://docs.llmgateway.io/features/audit-logs), [Policies](https://docs.llmgateway.io/learn/policies))", "guard_execution_location": "both", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))", "log_content": "metadata_only", "log_content_note": "\"Metadata Only\" is the default retention level, storing timestamps, model, tokens, and costs \"without full payloads\"; \"Retain All Data\" stores complete request and response payloads including messages, tool calls, and attachments. The log table has `content`, `messages`, and `reasoningContent` columns used only when payload retention is on ([Data retention](https://docs.llmgateway.io/features/data-retention), [Logging and analytics](https://deepwiki.com/theopenco/llmgateway/5.3-logging-and-analytics))", "log_opt_out": "yes", "log_opt_out_note": "Organization Settings → Policies → Data Retention Level; set to metadata-only or `none`. DevPass and chat subscriptions are always metadata-only with no way to enable payload storage. Responses API records are the exception: kept 30 days for `previous_response_id` chaining regardless of policy, unless `store: false` is sent ([Data retention](https://docs.llmgateway.io/features/data-retention))", "tracing_support": "not_documented", "tracing_note": "`n.a.` — no OpenTelemetry or agent tracing documented ([Overview](https://docs.llmgateway.io/overview))", "export_destinations": [], "export_destinations_note": "`n.a.` — activity and logs are exposed through LLMGateway's own management API rather than shipped to external sinks ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` ([Overview](https://docs.llmgateway.io/overview))", "observability_requires_saas": "No — all components including logging can be self-hosted ([Self-host](https://docs.llmgateway.io/self-host), [LLMGateway repo](https://github.com/theopenco/llmgateway))", "log_retention_note": "The pages disagree; see Contradictions. The data-retention page says 30 days for all users with Enterprise custom ([Data retention](https://docs.llmgateway.io/features/data-retention)); the policies page says Free 30 days, Enterprise custom ([Policies](https://docs.llmgateway.io/learn/policies)); the activity API reference says Free 3 days, Pro 90 days, Enterprise up to 1 year ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity)); the repo advertises \"Extended data retention (90 days vs 3 days)\" ([LLMGateway repo](https://github.com/theopenco/llmgateway))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` on all fetched pages ([LLM Gateway docs](https://docs.llmgateway.io/)).", "multi_surface": {}, "throughput_prose": "`n.a.` ([LLM Gateway docs](https://docs.llmgateway.io/)).", "benchmark_independence": "No figures published ([theopenco/llmgateway](https://github.com/theopenco/llmgateway))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Architecture is documented in operational terms (stateless services, PostgreSQL as source of truth, Redis for cache and worker queue, AGPLv3 license), but the reliability control surface is essentially undocumented — a developer choosing on reliability would have to read the source ([LLM Gateway self-host](https://docs.llmgateway.io/self-host), [LLM Gateway docs](https://docs.llmgateway.io/)).", "guard_notes": "Audit logs are an Enterprise feature ([Audit logs](https://docs.llmgateway.io/features/audit-logs)), and log storage is metered — activity records expose `dataStorageCost`, `creditsDataStorageCost`, and `apiKeysDataStorageCost` fields, so retaining bodies has a direct line-item cost ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity)).", "fit_agent_features_note": "LLM Gateway is described as working with AI agents and development tools and provides pre-built templates and agent configurations. Tool calling is supported through the Vercel AI SDK; sessions, threads, multi-step orchestration, and A2A are not documented. ([LLM Gateway documentation home](https://docs.llmgateway.io/))", "fit_byok_required": "hosted_no_selfhost_yes", "fit_byok_required_note": "No for the hosted platform: \"With your own provider keys (BYOK), routing through LLM Gateway is free\" and you may instead \"use the hosted platform and pay only the credit fee\" ([Pricing - LLM Gateway](https://llmgateway.io/pricing)). Yes for self-hosting: prerequisites include \"API keys for the LLM providers you want to use (OpenAI, Anthropic, etc.)\" ([Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))", "fit_client_sdk_langs": [ "TypeScript" ], "fit_client_sdks_note": "Documented integrations include the OpenAI SDK or other OpenAI-compatible SDKs, the Anthropic SDK, the Vercel AI SDK provider `@llmgateway/ai-sdk-provider`, and the TanStack AI adapter `@tanstack/ai-llmgateway`. The documented OpenAI SDK package is `openai`; languages are otherwise not stated, except that the Vercel AI SDK interface is TypeScript. ([LLM Gateway developer documentation](https://docs.llmgateway.io/developers))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "No: \"Start free with no credit card.\" and \"You can start free without a credit card.\"; plan is \"Free $0 forever.\" ([Pricing - LLM Gateway](https://llmgateway.io/pricing))", "fit_ecosystem_notes": "LLM Gateway is available as a hosted service and can be self-hosted under AGPLv3 using Docker, Docker Compose, or Kubernetes; hosted usage is described as pay-as-you-go with a flat 5% platform fee, and BYOK is supported. It provides one API across 40+ providers and routes across 200+ models, with integrations including REST, CLI, MCP, OpenAI-compatible SDKs, the Anthropic SDK, the Vercel AI SDK provider, and the TanStack AI adapter. ([LLM Gateway AI Gateway product page](https://llmgateway.io/products/ai-gateway))", "fit_eval_prereqs": "Dashboard sign-in and an API key from a created Project (exported in a shell or `.env` file). No credit card, cloud account, cluster, or provider key is stated ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start))", "fit_eval_shape": "base_url_swap", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "Partially: the endpoint shape `https://api.llmgateway.io/v1/…` authenticated with `LLM_GATEWAY_API_KEY` is documented and the page has per-language tabs, but no copy-pasteable request body rendered in the fetched content ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "OCI chart on GHCR; deploys gateway, API, UI and worker, and needs PostgreSQL plus Redis", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "LLM Gateway is self-hostable on Kubernetes, and the Kubernetes documentation describes an official Helm chart published as an OCI artifact on GitHub Container Registry. The chart deploys the gateway, API, UI, and worker and requires PostgreSQL and Redis. ([LLM Gateway Kubernetes self-hosting documentation](https://docs.llmgateway.io/self-host/kubernetes))", "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "LLM Gateway provides a hosted MCP server at `https://api.llmgateway.io/mcp` using streamable HTTP and bearer authentication. It is compatible with Claude Code, Codex, Cursor, and other MCP-compatible clients, and exposes `chat`, `generate-image`, `generate-nano-banana`, `list-models`, and `list-image-models` tools. ([LLM Gateway MCP server](https://llmgateway.io/mcp))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_cloudflare_workers", "fit_gateway_platform" ] }, "fit_model_gate": "none", "fit_model_gate_note": "No model gate documented; the licensing gate applies to enterprise features only: \"Enterprise features require a signed license in production,\" with \"An expired license has a seven-day grace period.\" ([Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker), [Self Host LLM Gateway](https://docs.llmgateway.io/self-host))", "fit_onboarding_notes": "Hosted path works in \"any language or framework,\" streaming via `stream: true` is proxied unchanged, every call appears in the dashboard with latency, cost and provider breakdown, and an Anthropic-compatible endpoint is available. Self-hosted setup requires creating an organization and project in the local UI before generating keys, and warns \"Do not bind-mount a host directory directly to `/var/lib/postgresql/data`\" ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start), [Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. The existing OpenAI SDK can be kept while changing the base URL to `https://api.llmgateway.io/v1`; the documented example uses `apiKey: process.env.LLM_GATEWAY_API_KEY`. ([LLM Gateway AI Gateway product page](https://llmgateway.io/products/ai-gateway))", "fit_prod_prereqs": "Stateless gateway, API, UI and background worker services; PostgreSQL for users/projects/keys/usage; Redis for response caching and the worker queue; provider API keys (OpenAI, Anthropic, Google, others); a Kubernetes cluster for the Helm deployment (EKS, GKE, AKS or self-managed); managed PostgreSQL and Redis recommended; cloud service mappings (AWS EKS/RDS/ElastiCache/Secrets Manager; GCP GKE/Cloud SQL/Memorystore/Secret Manager; Azure AKS/Azure Database for PostgreSQL/Azure Cache for Redis/Key Vault) ([Self Host LLM Gateway](https://docs.llmgateway.io/self-host))", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "base_url_swap", "fit_python_frameworks": [], "fit_python_frameworks_note": "n.a. (not documented)", "fit_quickstart_steps": 6, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.llmgateway.io/quick-start", "fit_self_host_docs": "documented_no_command", "fit_self_host_install": "Offered via Docker (all services in one container), Docker Compose, or the official Helm chart on Kubernetes, but no literal command string is printed on the self-host pages; the Docker page's post-install procedure is 4 steps ending \"Test the gateway by making API calls to http://localhost:4001\" (web UI at http://localhost:3002) ([Self Host LLM Gateway](https://docs.llmgateway.io/self-host), [Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))", "fit_terraform": "official_modules", "fit_terraform_note": "The Enterprise plan includes Terraform modules that provision AWS EKS, RDS, ElastiCache, networking, and secrets, and deploy LLM Gateway. The Terraform registry name is not stated. ([LLM Gateway AWS self-hosting documentation](https://docs.llmgateway.io/self-host/aws))", "fit_terraform_ref": "Enterprise-plan modules provisioning EKS, RDS, ElastiCache, networking and secrets; no Registry name", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "LLM Gateway provides the first-party npm package `@llmgateway/ai-sdk-provider` for the Vercel AI SDK, including `generateText`, `streamText`, and `generateObject`. The documentation also states that `@ai-sdk/openai` can use the gateway with a custom base URL. ([LLM Gateway Vercel AI SDK documentation](https://docs.llmgateway.io/developers/ai-sdk))", "fit_vercel_ai_sdk_package": "@llmgateway/ai-sdk-provider" }, { "slug": "mlflow-ai-gateway", "name": "MLflow AI Gateway", "company": "MLflow project (Linux Foundation)", "tagline": "AI gateway that ships inside the MLflow Tracking Server: OpenAI-compatible routing over encrypted provider connections, with guardrails, budgets and every request landing as an MLflow trace.", "category": "open_source", "deployment": "both", "license": "Apache-2.0", "founded_year": 2018, "hq_country": "United States", "url_home": "https://mlflow.org/ai-gateway", "url_docs": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/", "url_pricing": null, "url_status": null, "url_github": "https://github.com/mlflow/mlflow", "url_changelog": "https://mlflow.org/releases/", "url_models_api": null, "github_stars": 27777, "model_count": null, "model_count_note": "No model total is published for the gateway. The endpoint creation UI shows a searchable model selector with capability badges (Tools, Reasoning, Caching, Vision), context window and token costs, and the docs point you at \"the provider dropdown when creating an endpoint\" instead of a number ([Create and manage endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/create-and-manage/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), 2026-09-02).", "provider_count": 14, "token_markup_pct": 0, "credit_fee_pct": 0, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "You store your own provider keys as LLM Connections inside your MLflow server and pay the provider directly; the project charges nothing for the gateway ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "seat_fee_usd_month": 0, "free_tier": "The whole thing is free: \"MLflow is open source under the Apache 2.0 license and governed by the Linux Foundation\" with no paid gateway tier, and the gateway installs with `pip install 'mlflow[genai]'` ([LiteLLM alternative](https://mlflow.org/litellm-alternative/), [Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "enterprise_from_usd_year": null, "extra_fees": [], "self_host_cost_note": "No licence cost; your cost is the MLflow Tracking Server plus a SQL backend store (SQLite, PostgreSQL, MySQL or MSSQL) and, for production, object storage and optionally Redis for shared budget counters ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/), [Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": false, "portability_note": "Traces, token usage and cost live in your own SQL backend and can be read through the Python client, dual-exported over OTLP to another collector, or archived to your object store; there is no vendor lock on the data ([OpenTelemetry export](https://mlflow.org/docs/latest/genai/tracing/opentelemetry/export/), [Token usage and cost](https://mlflow.org/docs/latest/genai/tracing/token-usage-cost/), [Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/)).", "api_surfaces": [ "MLflow unified invocations", "OpenAI Chat Completions", "OpenAI Embeddings", "OpenAI Responses", "Anthropic Messages", "Google Gemini generateContent" ], "sdk_note": "No MLflow-specific client is needed to call the gateway: the documented clients are cURL, `requests`, the OpenAI Python SDK against `/gateway/mlflow/v1`, and provider SDKs (Anthropic, `google-genai`) against the passthrough paths. The `mlflow` Python client is used for the trace side (`mlflow.get_trace`, `client.delete_traces`) ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [Token usage and cost](https://mlflow.org/docs/latest/genai/tracing/token-usage-cost/)).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 28.6, "latency_source_note": "Vendor self-published and comparative: in MLflow's own LiteLLM comparison, MLflow AI Gateway shows P50 latency 78.6 ms / overhead 28.6 ms and P99 184.2 ms / overhead 134.2 ms against LiteLLM's 106.7 ms / 56.7 ms and 388.8 ms / 338.8 ms, with a 50 ms simulated provider delay, 4 workers and 50 concurrent users ([LiteLLM alternative](https://mlflow.org/litellm-alternative/), 2026-09-02). The docs' own benchmarks page publishes methodology and per-request headers rather than a result table, describing \"latency additions in the single-digit-to-tens-of-milliseconds range\" and showing an example `X-MLflow-Gateway-Overhead-Duration-Ms: 3` ([Benchmarks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/)).", "throughput_rps": 598, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": "Not a vendor decision: the gateway runs wherever you deploy the MLflow server, and no project-operated region exists ([Self-hosting](https://mlflow.org/docs/latest/self-hosting/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "zdr": null, "zdr_note": "There is no vendor retention to negotiate away — prompts go from your server straight to the provider you configured, and traces land in your own backend store ([AI Gateway](https://mlflow.org/ai-gateway), [Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/)).", "air_gapped": null, "best_for": "Teams already running MLflow for tracing and evaluation who want governed, OpenAI-compatible LLM access on the same server — no second system to deploy, and every gateway request lands as a trace they can evaluate.", "strengths": [ "Gateway, tracing, evaluation and prompt registry on one Apache-2.0 server: every request becomes an MLflow trace with no extra instrumentation", "Both unified OpenAI-compatible and provider-native passthrough surfaces (OpenAI, Anthropic Messages, Gemini generateContent) on the same server", "Provider keys encrypted in the backend store with a KEK passphrase and zero-downtime `mlflow crypto rotate-kek` rotation", "LLM-judge guardrails (Safety, PII, Custom) with Block or Sanitize actions, plus USD budget policies that alert by webhook or reject with HTTP 429", "Real RBAC over gateway secrets, endpoints and model definitions from MLflow 3.13.0, and an official OCI Helm chart for Kubernetes", "Foundation-governed with no vendor account, no per-request fee and no token markup" ], "weaknesses": [ "No documented request timeout, retry count, health check or circuit breaker — failover is reactive fallback only", "Post-LLM guardrails are not triggered for streaming requests, so a Safety guardrail on its default stage stops enforcing when a client sets stream: true", "\"Rate Limiting\" is ticked in MLflow's own comparison table but no rate-limit mechanism is documented anywhere in the gateway docs — only dollar budgets", "MLflow 3.0 removed the gateway config keys and the standalone deployment server, so endpoint management is UI/database-driven with no documented config-as-code path", "No gateway response cache of any kind, and no usage or cost export beyond reading traces", "Usage tracking is opt-in per endpoint, and the default `local` budget tracker keeps counters per process with a 600-second refresh, so spend caps can lag under multi-worker deployments", "Out-of-the-box authentication is HTTP basic auth; SSO needs a community OIDC plugin or a reverse proxy", "All published performance numbers are MLflow's own, from a page comparing itself with LiteLLM" ], "avoid_if": "You want a lean standalone proxy, or you need documented rate limits, request timeouts, retry policy, health checks, response caching or config-as-code endpoint management — none of which the gateway publishes — or you need post-LLM guardrails to hold on streaming responses.", "reviews": [ { "url": "https://pexon-consulting.de/blog/mlflow-ai-gateway/", "why": "German consultancy write-up positioning the gateway as \"ein datenbank-gestützter Proxy im MLflow-Tracking-Server\" with native provider integrations since 3.11 — useful, but it still shows a `mlflow gateway start --config-path config.yaml` command and a YAML endpoint config, which MLflow 3.0 removed.", "date": "2026-07-06", "type": "written", "title": "MLflow AI Gateway: LLM-Routing mit Tracing 2026", "publisher": "Pexon Consulting" }, { "url": "https://pexon-consulting.de/blog/mlflow-vs-litellm/", "why": "Third-party head-to-head that lands on the same operational caveat found in the docs: \"MLflow bietet out of the box nur HTTP Basic Authentication\", and neither gateway ships a ready GDPR/SSO/HA story.", "date": "2026-07-08", "type": "written", "title": "MLflow vs LiteLLM: welches LLM-Gateway 2026?", "publisher": "Pexon Consulting" }, { "url": "https://dev.to/debmckinney/we-evaluated-13-llm-gateways-for-production-heres-what-we-found-2dkm", "why": "Vendor-adjacent but externally published survey that files MLflow AI Gateway in \"Tier 4 — Niche or Limited\" with \"Limited LLM-specific features\" and \"Heavy for simple routing\"; note it predates the 2026 relaunch of the feature.", "date": "2025-12-14", "type": "written", "title": "We Evaluated 13 LLM Gateways for Production. Here's What We Found", "publisher": "DEV Community (Debby McKinney, Maxim AI)" }, { "url": "https://www.youtube.com/watch?v=h1gA6YvjEs4", "why": "Independent walkthrough wiring SambaNova Cloud through the gateway for enterprise GenAI, covering governance and observability rather than the vendor's own demo path.", "date": "2026-07-02", "type": "video", "title": "Centralize Your LLM Access with MLflow AI Gateway", "publisher": "YouTube — Martin Khristi" } ], "sources": { "cap_mcp": "https://mlflow.org/docs/latest/genai/mcp-registry/", "company": "https://mlflow.org/litellm-alternative/", "license": "https://api.github.com/repos/mlflow/mlflow", "sdk_note": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/", "url_docs": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/", "zdr_note": "https://mlflow.org/ai-gateway", "cap_evals": "https://mlflow.org/blog/mlflow-ai-gateway/", "free_tier": "https://mlflow.org/litellm-alternative/", "deployment": "https://mlflow.org/ai-gateway", "extra_fees": "https://mlflow.org/litellm-alternative/", "hq_country": "https://www.linuxfoundation.org/press/press-release/the-mlflow-project-joins-linux-foundation", "url_github": "https://api.github.com/repos/mlflow/mlflow", "weaknesses": "https://mlflow.org/litellm-alternative/", "zdr_status": "https://mlflow.org/ai-gateway", "api_surfaces": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/", "cap_failover": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/", "founded_year": "https://mlflow.org/litellm-alternative/", "github_stars": "https://api.github.com/repos/mlflow/mlflow", "product_fate": "https://mlflow.org/releases/2.17.0/", "regions_note": "https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/", "byok_fee_note": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/", "url_changelog": "https://mlflow.org/releases/", "vendor_status": "https://mlflow.org/litellm-alternative/", "byok_supported": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/", "cap_embeddings": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/", "cap_guardrails": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/", "certifications": "https://mlflow.org/litellm-alternative/", "config_as_code": "https://mlflow.org/docs/3.0.0rc3/mlflow-3/breaking-changes", "credit_fee_pct": "https://mlflow.org/litellm-alternative/", "provider_count": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/", "throughput_rps": "https://mlflow.org/litellm-alternative/", "logs_by_default": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/", "model_count_note": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/create-and-manage/", "token_markup_pct": "https://mlflow.org/litellm-alternative/", "cap_observability": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/", "eu_residency_note": "https://mlflow.org/docs/latest/self-hosting/", "last_release_date": "https://api.github.com/repos/mlflow/mlflow/releases/latest", "mlflow_github_api": "https://api.github.com/repos/mlflow/mlflow", "openai_compatible": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/", "cap_load_balancing": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/", "seat_fee_usd_month": "https://mlflow.org/litellm-alternative/", "guardrails_location": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/", "latency_overhead_ms": "https://mlflow.org/litellm-alternative/", "latency_source_note": "https://mlflow.org/litellm-alternative/", "requires_vendor_sdk": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/", "self_host_cost_note": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/", "cap_prompt_management": "https://mlflow.org/litellm-alternative/", "data_export_available": "https://mlflow.org/docs/latest/genai/tracing/opentelemetry/export/", "mlflow_release_2_17_0": "https://mlflow.org/releases/2.17.0/", "mlflow_ai_gateway_docs": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/", "mlflow_ai_gateway_home": "https://mlflow.org/ai-gateway", "prompt_transits_vendor": "https://mlflow.org/ai-gateway", "cap_conditional_routing": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/", "trains_on_customer_data": "https://mlflow.org/ai-gateway", "mlflow_3_breaking_changes": "https://mlflow.org/docs/3.0.0rc3/mlflow-3/breaking-changes", "mlflow_litellm_comparison": "https://mlflow.org/litellm-alternative/" }, "verified_at": { "cap_mcp": "2026-09-02", "company": "2026-09-02", "license": "2026-09-02", "sdk_note": "2026-09-02", "url_docs": "2026-09-02", "cap_evals": "2026-09-02", "free_tier": "2026-09-02", "deployment": "2026-09-02", "hq_country": "2026-09-02", "url_github": "2026-09-02", "weaknesses": "2026-09-02", "zdr_status": "2026-09-02", "api_surfaces": "2026-09-02", "cap_failover": "2026-09-02", "founded_year": "2026-09-02", "github_stars": "2026-09-02", "product_fate": "2026-09-02", "byok_fee_note": "2026-09-02", "url_changelog": "2026-09-02", "vendor_status": "2026-09-02", "byok_supported": "2026-09-02", "cap_embeddings": "2026-09-02", "cap_guardrails": "2026-09-02", "certifications": "2026-09-02", "config_as_code": "2026-09-02", "credit_fee_pct": "2026-09-02", "provider_count": "2026-09-02", "throughput_rps": "2026-09-02", "logs_by_default": "2026-09-02", "model_count_note": "2026-09-02", "token_markup_pct": "2026-09-02", "cap_observability": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "cap_load_balancing": "2026-09-02", "seat_fee_usd_month": "2026-09-02", "guardrails_location": "2026-09-02", "latency_overhead_ms": "2026-09-02", "latency_source_note": "2026-09-02", "requires_vendor_sdk": "2026-09-02", "self_host_cost_note": "2026-09-02", "cap_prompt_management": "2026-09-02", "data_export_available": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "trains_on_customer_data": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-02T15:23:19+00:00", "vendor_status": "foundation", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed, and the deprecation story that circulates about this product is wrong. The gateway was deprecated and renamed \"MLflow Deployments Server\" in 2.9.x ([2.9.2 migration guide](https://mlflow.org/docs/2.9.2/llms/gateway/migration.html)), but 2.17.0 on 2024-10-11 reversed it: \"MLflow AI Gateway is no longer deprecated - We've decided to revert our deprecation for the AI Gateway feature… and have reconsidered and reverted the naming and namespace back to the original configuration\" ([2.17.0 release notes](https://mlflow.org/releases/2.17.0/)). What did break, separately and genuinely, is MLflow 3.0: \"The MLflow deployment server application and the `start-server` CLI command have been removed (#15327)\" and \"The 'routes' and 'route_type' config keys in the gateway server configuration have been removed (#15331)\", so the gateway now runs inside the Tracking Server and endpoints are managed in the UI/database rather than a YAML file ([MLflow 3 breaking changes](https://mlflow.org/docs/3.0.0rc3/mlflow-3/breaking-changes)). It was then relaunched and heavily extended through 2026 — guardrails and cost tracking in 3.10, one-click coding-agent onboarding in 3.13, native provider integrations without LiteLLM from 3.11 — with v3.15.2 published 2026-08-26 and commits the same week ([blog](https://mlflow.org/blog/mlflow-ai-gateway/), [releases index](https://mlflow.org/releases/), [GitHub releases API](https://api.github.com/repos/mlflow/mlflow/releases/latest)).", "last_release_date": "2026-08-26", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "v3.15.2, published 2026-08-26 per the GitHub releases API; the releases index describes 3.15.2 as a patch release. Repo `pushed_at` was 2026-09-02 ([GitHub releases API](https://api.github.com/repos/mlflow/mlflow/releases/latest), [releases index](https://mlflow.org/releases/)).", "prompt_transits_vendor": "no", "prompt_path_note": "Self-hosted software: \"Your API keys and request data stay under your control\" and the gateway runs as part of your own MLflow Tracking Server, so no project-operated service sees prompts. Managed MLflow on Databricks or AWS is a third-party hosting choice, and gateway feature parity there is not documented on the pages fetched ([AI Gateway](https://mlflow.org/ai-gateway), [blog](https://mlflow.org/blog/mlflow-ai-gateway/)).", "logs_by_default": "not_applicable", "logs_note": "No vendor logging exists; logging is to your own database, and it is opt-in per endpoint. Usage tracking is a per-endpoint toggle, and when enabled \"every request is recorded as an MLflow trace\" with the full request and response payload alongside latency and token counts ([Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/), [blog](https://mlflow.org/blog/mlflow-ai-gateway/)).", "retention_days": null, "retention_note": "You set it. Server-side trace archival is configured in YAML (`MLFLOW_TRACE_ARCHIVAL_CONFIG` or `--trace-archival-config`) with keys `retention` (example `30d`), `location` (e.g. `s3://mlflow-trace-archive`), `interval_seconds` and `long_retention_allowlist`; retention resolves global default → workspace override → experiment override, and an experiment asking for longer than the workspace policy only gets it if its ID is allowlisted. Traces can also be deleted outright with `client.delete_traces()` by timestamp or ID ([Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/), [Delete traces](https://mlflow.org/docs/latest/genai/tracing/observe-with-traces/delete-traces/)).", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "not_applicable by construction: the project ships software rather than a hosted service, so no project-operated system receives prompts to train on ([AI Gateway](https://mlflow.org/ai-gateway)).", "subprocessors_url": null, "regions_note": "Anywhere you run it — Docker, Kubernetes via the official Helm chart, or a cloud VM; there are no project-managed regions ([Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/), [Self-hosting](https://mlflow.org/docs/latest/self-hosting/)).", "certifications": { "itar": { "status": "not_applicable" }, "soc2": { "status": "not_applicable" }, "fedramp": { "status": "not_applicable" }, "gdpr_dpa": { "status": "not_applicable" }, "iso27001": { "status": "not_applicable" }, "hipaa_baa": { "status": "not_applicable" } }, "guardrails_location": "self_hosted", "guardrails_note": "Guardrails are configured per endpoint and evaluated by an LLM judge that is itself another gateway endpoint, so enforcement happens in your own server while the judging inference goes to whichever provider backs the judge endpoint ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "oss_no_vendor_price", "pricing_model_note": "Apache-2.0 project under Linux Foundation governance with no pricing page, tier list or paid SKU for the gateway; the landing page contrasts itself with \"per request or per seat\" SaaS gateways and states \"no per-request fees, no usage limits\". Managed MLflow is sold by third parties (Databricks, AWS), not by the project ([AI Gateway](https://mlflow.org/ai-gateway), [LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "min_commitment": "None: the gateway is Apache-2.0 software you run yourself, with no contract, account or licence key ([LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "overage_note": "No vendor meter exists, so there is nothing to overrun. Spend against providers is capped by your own budget policies, which either alert or reject with HTTP 429 ([Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "cache_support": "none", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "No gateway-owned response or semantic cache appears on any fetched gateway page; \"Caching\" appears only as a per-model capability badge meaning \"Model supports prompt caching for efficiency\", and the benchmarks page mentions internal config caching, not response caching ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [Create and manage endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/create-and-manage/), [Benchmarks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/)). Because the gateway never prices tokens, any provider-side cache discount reaches you unchanged.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Budget policies with the Alert action fire once per window when the threshold is crossed ([Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "tier": null, "supported": true }, "virtual_keys": { "note": "Not documented as such. \"LLM Connections\" hold upstream provider keys, not per-consumer virtual keys; client-side access is controlled by MLflow authentication and RBAC rather than issued gateway keys ([API keys](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/create-and-manage/), [RBAC](https://mlflow.org/docs/latest/self-hosting/security/role-based-access-control/)).", "tier": null, "supported": null }, "rate_limiting": { "note": "Contradictory: the vendor comparison table ticks \"Rate Limiting\" for MLflow, but no request-rate or token-rate limit is documented on any gateway docs page fetched — only dollar budgets ([LiteLLM alternative](https://mlflow.org/litellm-alternative/) vs [Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/), [gateway docs index](https://mlflow.org/docs/latest/genai/governance/ai-gateway/)).", "tier": null, "supported": null }, "spend_webhooks": { "note": "The Alert action posts to a webhook, with a payload including `budget_policy_id` and `current_spend` ([Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "tier": null, "supported": true }, "model_allowlists": { "note": "Via RBAC: gateway secrets, endpoints and model definitions are first-class permissioned resources, and the USE permission is what allows \"invoking a gateway endpoint\", so admins decide which teams can call which endpoints. RBAC requires MLflow 3.13.0+ and authentication enabled ([RBAC](https://mlflow.org/docs/latest/self-hosting/security/role-based-access-control/)).", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "Not documented: budget policies are scoped globally or per workspace, not per key or per user ([Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Yes, via workspace scoping: a USD threshold over a daily, weekly or monthly window, applied globally or to a single workspace, with the Reject action returning HTTP 429 ([Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "tier": null, "supported": true } }, "cost_attribution": "Per endpoint, provider and model: the usage dashboard breaks down requests, latency percentiles, token usage, tokens per request, cost breakdown and cost over time, filterable by endpoint and time range. Budget policies scope spend globally or per workspace. Per-user or per-API-key cost attribution is not documented ([Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/), [Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "cost_export": "No CSV or warehouse cost export is documented. Cost lives on the traces themselves — `trace.info.cost` gives input/output/total USD per trace and `span.llm_cost` per LLM call through the Python SDK — and traces can be exported over OTLP ([Token usage and cost](https://mlflow.org/docs/latest/genai/tracing/token-usage-cost/), [OpenTelemetry export](https://mlflow.org/docs/latest/genai/tracing/opentelemetry/export/)).", "product_kind_note": "One feature of a much larger platform, and the vendor frames it that way: \"MLflow AI Gateway was built to fix this without the integration tax. Because it runs as part of the MLflow Tracking Server you're already using for tracing and evaluation, you get governed LLM access in the same place you debug traces and run evaluations\". So platform-level facts — 30 million monthly downloads, 27,777 GitHub stars on `mlflow/mlflow`, autolog for 30+ frameworks — describe MLflow, not gateway adoption; the gateway itself only reached general usability in the 3.9–3.15 releases of 2026 ([blog](https://mlflow.org/blog/mlflow-ai-gateway/), [AI Gateway](https://mlflow.org/ai-gateway), [mlflow/mlflow API](https://api.github.com/repos/mlflow/mlflow)).", "deployment_modes": [ "self_host", "on_prem", "saas" ], "deployment_note": "The gateway is a feature of the Tracking Server, not a deployable of its own — as of MLflow 3.0 \"the MLflow deployment server application and the `start-server` CLI command have been removed\" (#15327), so `mlflow server` is the only entry point. Production guidance is PostgreSQL plus S3/GCS/Azure storage, with TLS, ingress, Prometheus metrics, NetworkPolicy, RBAC and an `mlflow gc` CronJob available in the chart; public ingress also needs `allowed_hosts` set or requests return HTTP 403 ([MLflow 3 breaking changes](https://mlflow.org/docs/3.0.0rc3/mlflow-3/breaking-changes), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/), [Network security](https://mlflow.org/docs/latest/self-hosting/security/network/)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Two shapes on one server. Unified: `POST /gateway/{endpoint}/mlflow/invocations`, plus an OpenAI-compatible base URL at `/gateway/mlflow/v1` where the gateway endpoint name is passed as `model`. Passthrough: provider-native paths such as `/gateway/openai/v1/chat/completions`, `/gateway/anthropic/v1/messages` and `/gateway/gemini/v1beta/models/{endpoint}:generateContent`, so you keep the provider's own SDK while the gateway still holds the credentials and records usage ([Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a. — no model count on any fetched page ([AI Gateway landing](https://mlflow.org/ai-gateway), [gateway docs](https://mlflow.org/docs/latest/genai/governance/ai-gateway/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)); models are whatever the configured provider exposes, plus the optional LiteLLM catalogue.", "provider_count_low": 14, "provider_count_high": 100, "provider_count_source_note": "Three different figures on vendor pages: 14 providers enumerated in the docs tables (OpenAI, Anthropic, Google Gemini, Azure OpenAI, AWS Bedrock, Vertex AI, Cohere, Mistral, Groq, Together AI, Fireworks AI, Ollama, Databricks, Portkey) ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)); \"Select your provider from 100+ supported options\" in the endpoint-creation UI ([Create and manage endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/create-and-manage/)); and \"Access 50+ Model Providers\" on the product page ([AI Gateway](https://mlflow.org/genai/ai-gateway)). The enumerated 14 is used here; the larger numbers depend on the optional LiteLLM catalogue, which \"you will need to install separately\".", "first_party_vs_partner": "All third-party: the project hosts no models. Endpoints forward to provider APIs (OpenAI, Anthropic, Gemini, Azure OpenAI, Bedrock, Vertex AI, Cohere, Mistral, Groq, Together AI, Fireworks AI, Databricks, Portkey) or to your own Ollama/OpenAI-compatible endpoint ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "custom_endpoint_support": "Yes: the product page states the gateway routes to \"any LLM provider — including any OpenAI-compatible API or custom model endpoint\", and Ollama is documented for local models ([AI Gateway](https://mlflow.org/genai/ai-gateway), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "byok_mode": "byok_only", "merchant_of_record": "The upstream LLM provider. Provider credentials are your own LLM Connections and the project bills nothing ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "key_management_note": "Provider keys live in \"LLM Connections\" and are encrypted before being written to the MLflow backend database; a default passphrase is used for local development and production deployments must set `MLFLOW_CRYPTO_KEK_PASSPHRASE`. Rotation is zero-downtime via `mlflow crypto rotate-kek --new-passphrase` plus `MLFLOW_CRYPTO_KEK_VERSION`, and key inputs are masked in the UI ([API keys](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/create-and-manage/), [Key rotation](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/key-rotation/)).", "api_openai_chat_completions_note": "Yes: `client = OpenAI(base_url=\"http://localhost:5000/gateway/mlflow/v1\", api_key=\"\")` then `chat.completions.create(model=\"my-chat-endpoint\", ...)`; the API key is empty because credentials are held server-side ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/)).", "api_anthropic_messages_note": "Yes, as a passthrough surface: `POST /gateway/anthropic/v1/messages`, documented with the Anthropic Python SDK pointed at `base_url=\"https://your-mlflow-server/gateway/anthropic\"` and a dummy API key ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [blog](https://mlflow.org/blog/mlflow-ai-gateway/)).", "api_openai_responses_note": "Yes: the OpenAI passthrough \"exposes the full OpenAI API\" and lists `POST /responses` — Responses API (multi-turn conversations) — alongside `/chat/completions` and `/embeddings` ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "api_embeddings_note": "Yes: embeddings are marked supported for OpenAI, Google Gemini, Azure OpenAI, AWS Bedrock, Vertex AI, Cohere, Mistral, Together AI, Fireworks AI, Ollama, Databricks and Portkey (Anthropic and Groq are marked No), and `POST /embeddings` is part of the OpenAI passthrough ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "api_images_note": "n.a. — no image-generation endpoint on the fetched gateway pages; \"Vision\" appears only as a model capability badge for image input ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/)).", "api_audio_note": "n.a. — no STT/TTS path documented ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/)).", "api_batch_note": "n.a. — no batch or async bulk endpoint documented ([Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "deployment_modes_note": "self-host and on-prem are the first-class paths: pip/uvx locally, Docker Compose, or the official Helm chart on Kubernetes ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)). SaaS only via third parties: \"Deploy on your own infrastructure or use managed versions on Databricks or AWS\" — that sentence is about MLflow generally, and gateway parity on those platforms is not documented on the pages fetched ([AI Gateway](https://mlflow.org/ai-gateway)).", "byok_mode_note": "byok_only: there is no platform credit or hosted key. Step 2 of the quickstart is creating an LLM Connection with \"your API key from the provider\", which is then encrypted in the MLflow backend store ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [API keys](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/create-and-manage/)). The coding-agent flow is the one exception: those endpoints need no key because \"the agent brings its own credentials\" ([Claude Code](https://mlflow.org/docs/latest/genai/governance/ai-gateway/coding-agents/claude-code/)).", "access_meta": { "qualified": [], "model_count_kind": null, "provider_count_kind": "enumerated" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "not_documented for gateway requests: no per-endpoint or per-request timeout key appears on the routing, endpoint or query pages. The only timeouts discussed are artifact upload/download through the tracking server's artifact proxy ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/)).", "rel_retry_config": "not_documented", "rel_retry_note": "No retry count, backoff strategy or default is published. The documented failure behaviour is sequential fallback rather than same-model retry: \"the gateway tries fallback models sequentially until one succeeds\". Default retry count: n.a. Backoff: n.a. ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/)).", "rel_retry_default_note": null, "rel_fallback_config": "dashboard_only", "rel_fallback_note": "ORDERED and UI-configured: fallbacks are added under \"Priority 2 (Fallback)\" on the endpoint page, reordered by dragging, and tried in sequence when the primary hits errors or rate limits. Each fallback carries its own provider, model and API key, so a fallback can point at a different provider or region ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "not_documented", "rel_health_check_note": "not_documented — no active health check, circuit breaker or unhealthy-model ejection is described; failover is reactive, triggered by the error on the request itself ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "not_documented as a gateway feature. \"Regional failover: Route to providers in different geographic regions\" is listed as a fallback use case, but that is provider selection inside one gateway rather than a multi-region gateway deployment; no cross-region MLflow topology is described ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "rel_load_balancing_config": "dashboard_only", "rel_load_balancing_note": "Weights are user-settable but UI-only: traffic splitting assigns each model a 1–100% weight and the weights \"must sum to exactly 100\", intended for A/B tests and gradual rollouts. No latency- or cost-aware balancing is documented ([Traffic routing and fallbacks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/traffic-routing-fallbacks/)).", "architecture_class": "interpreted_proxy", "architecture_note": "Python (the repo's primary language per the GitHub API) running on the FastAPI/uvicorn tracking server; the gateway is not a separate process. It is a database-backed proxy: it requires a SQL backend store (SQLite, PostgreSQL, MySQL or MSSQL) plus the FastAPI server, and file-based tracking stores are not supported. Note that the 27,777 stars on the GitHub API are for all of `mlflow/mlflow` — the whole ML/GenAI platform — and are not a measure of gateway adoption ([mlflow/mlflow API](https://api.github.com/repos/mlflow/mlflow), [Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "self_host_data_plane": "yes", "self_host_artifact_note": "PyPI extra plus an official OCI Helm chart: `pip install 'mlflow[genai]'` then `mlflow server --port 5000`, or `helm install mlflow oci://ghcr.io/mlflow/charts/mlflow --version --namespace mlflow --create-namespace` (Kubernetes 1.23+, Helm 3.8+) ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "latency_claims": [ { "url": "https://mlflow.org/litellm-alternative/", "unit": "ms", "scope": "gateway_overhead", "value": "28.6", "metric": "p50", "publisher": "vendor", "conditions": "Vendor benchmark against LiteLLM. 50 ms simulated provider delay, 4 workers, 50 concurrent users; overhead = measured latency minus the simulated delay. Hardware not stated." }, { "url": "https://mlflow.org/litellm-alternative/", "unit": "ms", "scope": "gateway_overhead", "value": "134.2", "metric": "p99", "publisher": "vendor", "conditions": "Same vendor benchmark run (50 ms simulated provider delay, 4 workers, 50 concurrent users)." }, { "url": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/", "unit": "ms", "scope": "gateway_overhead", "value": "single-digit-to-tens-of-milliseconds", "metric": "range", "publisher": "vendor", "conditions": "Docs prose, not a measured result table. The gateway emits `X-MLflow-Gateway-Duration-Ms` on every response and `X-MLflow-Gateway-Overhead-Duration-Ms` on non-streaming responses; the worked example shows duration 87 ms with 3 ms overhead. Test rig: fake OpenAI server with a fixed 50 ms delay, 4 MLflow instances behind nginx, PostgreSQL, 50 concurrent requests; explicitly excludes provider variance, network, TLS and auth." } ], "throughput_claims": [ { "url": "https://mlflow.org/litellm-alternative/", "unit": "req/s", "scope": "capacity", "value": "598", "metric": "sustained", "publisher": "vendor", "conditions": "Vendor benchmark against LiteLLM (358 req/s), described as \"67% higher throughput\". 50 ms simulated provider delay, 4 workers, 50 concurrent users; hardware not stated." } ], "streaming_support": "yes", "streaming_note": "Supported over SSE by setting `stream: true`. Two documented caveats: post-LLM guardrails \"are not triggered for streaming requests\", and the `X-MLflow-Gateway-Overhead-Duration-Ms` header is only emitted for non-streaming responses ([Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/), [Benchmarks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/)).", "guard_pii": "sync_block", "guard_pii_note": "A PII Detection guardrail type is built in, defaults to the Pre-LLM stage, and its action is chosen at setup: Block returns HTTP 400 naming the guardrail and its rationale, or Sanitize redacts the offending content and lets the request continue ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "guard_injection": "not_documented", "guard_injection_note": "n.a. — no prompt-injection or jailbreak guardrail type is offered; the three documented types are Safety, PII Detection and Custom, so injection detection would have to be written as a Custom judge prompt ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "guard_content": "sync_block", "guard_content_note": "A Safety guardrail type ships built in and defaults to the Post-LLM stage, with the same Block (HTTP 400) or Sanitize actions. Guardrails apply to both unified and passthrough endpoints ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "guard_custom": "sync_block", "guard_custom_note": "Custom guardrails are natural-language judge instructions rather than regex: you pick the stage (Pre-LLM or Post-LLM), the judge model (any other gateway endpoint) and the action (Block or Sanitize). Multiple guardrails run in table order and later ones are skipped once one blocks ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "guard_model_policy": "yes", "guard_model_policy_note": "Yes, through RBAC rather than a gateway-specific allowlist: gateway secrets, endpoints and model definitions are permissioned resource types, and USE is the permission that covers \"invoking a gateway endpoint\". RBAC needs MLflow 3.13.0+ and authentication enabled; before 3.13 only the legacy per-resource permission APIs existed ([RBAC](https://mlflow.org/docs/latest/self-hosting/security/role-based-access-control/), [Basic HTTP auth](https://mlflow.org/docs/latest/self-hosting/security/basic-http-auth/)).", "guard_execution_location": "both", "guard_third_party": null, "guard_failure_mode": "not_documented", "guard_failure_mode_note": "not_documented — the guardrails page describes Block and Sanitize outcomes but says nothing about what happens when the judge endpoint itself errors or times out ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "log_content": "configurable", "log_content_note": "configurable: off unless usage tracking is enabled on the endpoint, and full request/response payloads once it is. PII can be stripped before storage with client-side span processors registered through `mlflow.tracing.configure(span_processors=[...])` ([Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/), [Masking](https://mlflow.org/docs/latest/genai/tracing/observe-with-traces/masking/)).", "log_opt_out": "yes", "log_opt_out_note": "Yes — it is opt-in rather than opt-out: usage tracking is a toggle on each endpoint, and the docs note token and cost metrics are unavailable for some providers and models even when it is on ([Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/)).", "tracing_support": "otel", "tracing_note": "Gateway requests become MLflow traces server-side with no client instrumentation, and the trace layer is OpenTelemetry-based: OTLP export via `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, simultaneous dual export with `MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT=true`, GenAI semantic conventions via `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV`, and W3C TraceContext linking client-side agent traces to gateway traces ([OpenTelemetry export](https://mlflow.org/docs/latest/genai/tracing/opentelemetry/export/), [AI Gateway tracing integration](https://mlflow.org/docs/latest/genai/tracing/integrations/listing/mlflow-ai-gateway/), [LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "export_destinations": [ "OpenTelemetry (OTLP)", "Prometheus", "S3", "GCS", "Azure Blob Storage" ], "export_destinations_note": "OTLP/HTTP for traces and metrics, with dual export so an existing collector keeps receiving spans while MLflow stores them ([OpenTelemetry export](https://mlflow.org/docs/latest/genai/tracing/opentelemetry/export/)); Prometheus metrics and a ServiceMonitor from the official Helm chart ([Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)); and S3/GCS/Azure object storage as the trace-archival and artifact destination ([Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/)).", "feedback_capture": "partial", "feedback_capture_note": "Partial and platform-level, not gateway-level: no feedback widget or scoring call is documented on the gateway pages, but MLflow advertises \"LLM judge alignment with human feedback\" and the MLflow MCP server can \"log feedback and assessments\" against traces ([LiteLLM alternative](https://mlflow.org/litellm-alternative/), [MCP server](https://mlflow.org/docs/latest/genai/mcp/), checked against [Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/)).", "eval_hooks": "yes", "eval_hooks_note": "Yes, and this is the whole pitch: \"Traces captured through the gateway feed directly into `mlflow.genai.evaluate` or Evaluation Dataset APIs, so you can run judges over production traffic without any additional instrumentation\", and a gateway endpoint can itself be referenced as a judge model with `gateway:/my-chat-endpoint` ([blog](https://mlflow.org/blog/mlflow-ai-gateway/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/)).", "observability_requires_saas": "No — the usage dashboard, traces and evaluation all run inside the MLflow server you deployed, and nothing is gated behind a hosted control plane ([Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/), [blog](https://mlflow.org/blog/mlflow-ai-gateway/)). One caveat published by the project: cost computation on Databricks managed MLflow \"requires the client application to install LiteLLM or manually set the cost attributes on spans\", which is not required self-hosted ([Token usage and cost](https://mlflow.org/docs/latest/genai/tracing/token-usage-cost/)).", "log_retention_note": "Determined by your own archival policy and deletion calls; the documented example archives span payloads to object storage after 30 days while keeping traces readable in the UI ([Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/)).", "production_meta": { "sources": {}, "qualified": [ "latency_overhead_ms", "throughput_rps" ], "latency_prose": "Vendor-published: \"MLflow AI Gateway added roughly half the latency overhead of LiteLLM (28.6 ms vs 56.7 ms at P50)\", with P99 overhead 134.2 ms, measured with a 50 ms simulated provider delay, 4 workers and 50 concurrent users ([LiteLLM alternative](https://mlflow.org/litellm-alternative/)). The docs' benchmarks page gives no result table, only the prose \"single-digit-to-tens-of-milliseconds range\" and a worked example of 3 ms overhead on an 87 ms request ([Benchmarks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/)).", "multi_surface": {}, "throughput_prose": "598 req/s versus LiteLLM's 358 req/s in the same vendor run (\"67% higher throughput\"); absolute capacity depends on hardware, which is not stated ([LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "benchmark_independence": "Not independent. Every figure is MLflow's own, published on a page whose purpose is to position MLflow against LiteLLM, and it names a competitor's numbers without an independent replication ([LiteLLM alternative](https://mlflow.org/litellm-alternative/)). The docs benchmarks page is transparent about its harness but publishes no result table ([Benchmarks](https://mlflow.org/docs/latest/genai/governance/ai-gateway/benchmarks/))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "not_documented", "guard_content_default": "choose_at_setup", "guard_custom_default": "choose_at_setup", "rel_notes": "Endpoint changes are hot: \"Add, remove, or modify endpoints dynamically without restarting the server or disrupting running applications\", and key rotation is likewise zero-downtime. The gap is everything below routing — no timeouts, retry counts, health checks or circuit breakers are published, and budget enforcement can lag because the default tracker refresh interval (`MLFLOW_GATEWAY_BUDGET_REFRESH_INTERVAL`) is 600 seconds and the default `local` tracker keeps counters per process rather than sharing them ([gateway docs](https://mlflow.org/docs/latest/genai/governance/ai-gateway/), [Key rotation](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/key-rotation/), [Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/)).", "guard_notes": "The load-bearing caveat is streaming: \"post-LLM guardrails are not triggered for streaming requests\", so a Safety guardrail left on its Post-LLM default silently stops enforcing as soon as a client sets `stream: true`. Guardrails are also LLM-judge based, which means each guarded request costs an extra model call to whichever endpoint backs the judge ([Guardrails](https://mlflow.org/docs/latest/genai/governance/ai-gateway/guardrails/)).", "fit_agent_features_note": "Tool calling and structured output are first-class on the unified endpoint: a `tools` array and `response_format` accepting `text`, `json_object` or `json_schema`. Coding agents are an explicit use case, with one-click endpoints and documented setups for Claude Code (`ANTHROPIC_BASE_URL=http://localhost:5000/gateway/proxy/claude-code`), OpenAI Codex, Gemini CLI and Hermes Agent, each conversation captured as a trace and subject to the same guardrails and budgets ([Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Coding agents](https://mlflow.org/docs/latest/genai/governance/ai-gateway/coding-agents/), [Claude Code](https://mlflow.org/docs/latest/genai/governance/ai-gateway/coding-agents/claude-code/)).", "fit_byok_required": "yes", "fit_byok_required_note": "Yes: nothing works until you create an LLM Connection with your own provider key (quickstart step 2), because the project hosts no models and issues no credits. Exception: coding-agent endpoints, where \"the agent brings its own credentials\" ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [releases index](https://mlflow.org/releases/)).", "fit_client_sdk_langs": [ "Python" ], "fit_client_sdks_note": "Documented client code is Python and cURL: `requests`, the OpenAI Python SDK against `/gateway/mlflow/v1`, the Anthropic Python SDK against the passthrough base URL, and `google-genai`. No JavaScript/TypeScript gateway example appears on the fetched pages, though any OpenAI-compatible client works by construction ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "AWS Bedrock and Vertex AI are listed as supported providers, but the passthrough/authentication detail on the fetched pages is limited to API-key style LLM Connections; no IAM role, SigV4 or workload-identity configuration is documented for the gateway itself ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/), [API keys](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/create-and-manage/)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented) — the gateway is a Python server; no edge-worker deployment appears on the self-hosting pages ([Self-hosting](https://mlflow.org/docs/latest/self-hosting/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "fit_credit_card": "not_required", "fit_credit_card_note": "No card, account or licence for the gateway itself — `pip install 'mlflow[genai]'` and `mlflow server --port 5000` is the whole install. You will need a paid provider key to get a completion out of it ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "fit_ecosystem_notes": "Apache-2.0 and Linux Foundation governed, so there is no vendor account in the path, and the same server carries tracing, evaluation, prompt registry and model registry. The trade is operational: you own PostgreSQL, object storage, TLS, ingress and auth, and the gateway inherits MLflow's server surface — including its authentication model, which out of the box is HTTP basic auth ([LiteLLM alternative](https://mlflow.org/litellm-alternative/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/), [Basic HTTP auth](https://mlflow.org/docs/latest/self-hosting/security/basic-http-auth/)).", "fit_eval_prereqs": "Python with `pip install 'mlflow[genai]'`, plus one provider API key. The default `mlflow server` uses SQLite and FastAPI, \"so no additional configuration is needed for this quickstart\" ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes, both a cURL and an SDK one-liner: `curl -X POST http://localhost:5000/gateway/my-chat-endpoint/mlflow/invocations -H \"Content-Type: application/json\" -d '{\"messages\": [{\"role\": \"user\", \"content\": \"Hello!\"}]}'`, or the OpenAI SDK with `base_url=\"http://localhost:5000/gateway/mlflow/v1\"` and `model=\"my-chat-endpoint\"` ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It is itself the gateway — \"a centralized proxy layer that routes requests to LLM providers through a single, unified API\" — but uniquely here it is a gateway embedded in an MLOps platform rather than a standalone proxy ([AI Gateway](https://mlflow.org/ai-gateway), [blog](https://mlflow.org/blog/mlflow-ai-gateway/)). It can also front another gateway: Portkey is a supported provider ([Model providers](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/model-providers/)).", "fit_helm_chart": "official chart at oci://ghcr.io/mlflow/charts/mlflow", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "Official OCI Helm chart: `helm install mlflow oci://ghcr.io/mlflow/charts/mlflow --version --namespace mlflow --create-namespace`, requiring Kubernetes 1.23+ and Helm 3.8+, with TLS, ingress, Prometheus ServiceMonitor, NetworkPolicy, RBAC and an `mlflow gc` CronJob, and PostgreSQL plus S3/GCS/Azure recommended for production ([Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "fit_mcp": "hosted_mcp_server", "fit_mcp_note": "Two separate things, both experimental. MLflow ships an MCP server (3.5.1+) that exposes trace management to Claude, Cursor and VS Code — search traces, analyse performance, log feedback, manage tags, delete traces — and an MCP Registry (\"experimental feature introduced in MLflow 3.15.0\") for registering and versioning MCP servers via `server.json` and `mlflow.genai.register_mcp_server()` ([MCP server](https://mlflow.org/docs/latest/genai/mcp/), [MCP Registry](https://mlflow.org/docs/latest/genai/mcp-registry/)). The landing page's stronger claim — that the gateway governs \"which MCP servers your agents can reach\" — has no matching page in the AI Gateway docs section ([AI Gateway](https://mlflow.org/ai-gateway)).", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_terraform", "fit_vercel_ai_sdk", "rel_timeout_config", "rel_retry_config", "rel_health_check_config", "rel_multi_region_config", "guard_injection" ] }, "fit_model_gate": "none", "fit_model_gate_note": "None from the project: there is no approval, quota, waitlist or enablement step — you pick a provider and model in the endpoint dialog and supply your own key. Access control is yours to impose, via RBAC on gateway endpoints ([Create and manage endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/create-and-manage/), [RBAC](https://mlflow.org/docs/latest/self-hosting/security/role-based-access-control/)).", "fit_onboarding_notes": "Four steps to a first call, all local, but two constraints bite early: the gateway needs a SQL-backed store (SQLite, PostgreSQL, MySQL or MSSQL) and the FastAPI tracking server — file-based tracking stores are not supported — and usage tracking is off until you toggle it per endpoint, so the observability that justifies the product is opt-in ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/), [Usage tracking](https://mlflow.org/docs/latest/genai/governance/ai-gateway/usage-tracking/)).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes: \"Use any OpenAI-compatible SDK. Point the base URL at the gateway and use your endpoint name as the model\", with `api_key=\"unused\"` because the gateway holds the real credential ([AI Gateway](https://mlflow.org/genai/ai-gateway), [Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "fit_prod_prereqs": "MLflow server with the `[genai]` extra, a SQL backend store (PostgreSQL or MySQL recommended over the SQLite default at concurrency), object storage for artifacts and trace archival, and — for real access control — `pip install 'mlflow[auth]'`, `MLFLOW_FLASK_SERVER_SECRET_KEY`, `mlflow server --app-name basic-auth` and MLflow 3.13.0+ for RBAC. Shared budget enforcement across processes needs Redis via `MLFLOW_GATEWAY_BUDGET_REDIS_URL`, and production key encryption needs `MLFLOW_CRYPTO_KEK_PASSPHRASE` ([Tracking server](https://mlflow.org/docs/latest/self-hosting/architecture/tracking-server/), [Basic HTTP auth](https://mlflow.org/docs/latest/self-hosting/security/basic-http-auth/), [RBAC](https://mlflow.org/docs/latest/self-hosting/security/role-based-access-control/), [Budget alerts and limits](https://mlflow.org/docs/latest/genai/governance/ai-gateway/budget-alerts-limits/), [API keys](https://mlflow.org/docs/latest/genai/governance/ai-gateway/api-keys/create-and-manage/)).", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain", "LangGraph", "DSPy", "OpenAI Agents SDK", "LiteLLM" ], "fit_python_frameworks_note": "The query page ships copy-paste snippets for LangChain (`ChatOpenAI` with the gateway base URL), LangGraph (`create_react_agent`), DSPy (`dspy.LM`), the OpenAI Agents SDK and LiteLLM (`litellm.completion`), all pointed at `/gateway/mlflow/v1`. Separately, MLflow advertises one-line `autolog()` tracing for 30+ frameworks — a platform feature, not a gateway one ([Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/), [LiteLLM alternative](https://mlflow.org/litellm-alternative/)).", "fit_quickstart_steps": 4, "fit_quickstart_steps_caveat": "Four numbered steps (install and start, create an LLM Connection, create an endpoint, query it), but steps 2 and 3 are five-click UI sub-flows each rather than commands ([Quickstart](https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/)).", "fit_quickstart_url": "https://mlflow.org/docs/latest/genai/governance/ai-gateway/quickstart/", "fit_self_host_docs": "command_published", "fit_self_host_install": "`pip install 'mlflow[genai]'` then `mlflow server --port 5000`; on Kubernetes `helm install mlflow oci://ghcr.io/mlflow/charts/mlflow --version --namespace mlflow --create-namespace`", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented) — no Terraform provider or module is referenced on the self-hosting, Kubernetes or gateway pages fetched ([Self-hosting](https://mlflow.org/docs/latest/self-hosting/), [Kubernetes and Helm](https://mlflow.org/docs/latest/self-hosting/kubernetes-helm/)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Set up governed LLM access in minutes. No additional infrastructure required\", broken into per-step estimates of ~30 seconds to start the server, ~1 minute to create an endpoint and ~30 seconds to query it ([AI Gateway](https://mlflow.org/genai/ai-gateway)).", "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "Vercel AI SDK appears in MLflow's docs only as a tracing integration — the `ai` package exporting OpenTelemetry spans into MLflow via `@vercel/otel` and `OTEL_EXPORTER_OTLP_ENDPOINT` — not as a way to call gateway endpoints, and no gateway provider package or `baseURL` example is documented ([Vercel AI SDK tracing](https://mlflow.org/docs/latest/genai/tracing/integrations/listing/vercelai/), checked against [Query endpoints](https://mlflow.org/docs/latest/genai/governance/ai-gateway/endpoints/query-endpoints/)).", "fit_vercel_ai_sdk_package": null }, { "slug": "new-api", "name": "New API", "company": "QuantumNous", "tagline": "Self-hosted Go gateway that fans 30+ upstream model providers into one OpenAI-compatible endpoint, with a full user, token and quota billing system bolted on.", "category": "open_source", "deployment": "self_host_only", "license": "AGPL-3.0", "founded_year": 2023, "hq_country": null, "url_home": "https://www.newapi.ai", "url_docs": "https://www.newapi.ai/en/docs", "url_pricing": null, "url_status": null, "url_github": "https://github.com/QuantumNous/new-api", "url_changelog": "https://www.newapi.ai/en/docs/guide/wiki/changelog", "url_models_api": null, "github_stars": 48314, "model_count": 100, "model_count_note": "\"Choose from 100+ models across multiple providers\" ([newapi.ai](https://www.newapi.ai/)). This is a marketing figure about the upstream catalogue rather than a shipped list - the actual model set on any instance is whatever the operator's channels expose, and the console can sync upstream model lists with an Added/Changed/Deleted preview ([Model Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/model)).", "provider_count": 30, "token_markup_pct": 0, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "No fee: New API charges nothing for relaying your own keys, and the upstream provider bills you directly ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction), [Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)).", "seat_fee_usd_month": 0, "free_tier": "The whole product: \"New API adopts the GNU AGPLv3 open-source license\" and is free to deploy and use provided the licence is honoured; there is no feature-gated tier ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)).", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Commercial licence (AGPLv3 exemption)", "amount": "Price not published; by enquiry to support@quantumnous.com" } ], "self_host_cost_note": "You pay only for your own infrastructure and upstream provider spend: the software is AGPLv3 and \"can be used for free\" so long as the licence is observed, with a paid commercial licence available by email for AGPL-exempt use ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)). Production sizing implies a MySQL and a Redis alongside the gateway ([Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment)).", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Reasonable: usage logs can be exported by users when the Root toggle is on ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)), all state lives in your own SQLite/MySQL/PostgreSQL ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)), and the schema is \"fully compatible with the original One API database\", which makes migration from that predecessor a drop-in ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). There is no declarative export of routing configuration.", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Completions", "OpenAI Responses", "OpenAI Embeddings", "OpenAI Images", "OpenAI Audio", "OpenAI Realtime (WebSocket)", "Cohere/Jina-style Rerank", "OpenAI Moderations", "OpenAI-compatible Video", "Anthropic Messages (native)", "Google Gemini v1beta (native)", "New API Management API" ], "sdk_note": "No vendor SDK is required or published: you point the OpenAI SDK at your instance and pass your New API token as the API key ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)). The same page shows Claude-native and Gemini-native calling patterns, and a built-in Playground is served at `/console/playground` ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)).", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": null, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": true, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "None published. No latency figure of any kind appears on the homepage, README, or the performance page - which covers profiling instrumentation (pprof, Pyroscope) rather than results ([Performance Analysis](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/performance-analysis), [newapi.ai](https://www.newapi.ai/), [README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)); the full docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog) contains no benchmark either. Per-channel \"response time\" shown by the Test action measures your upstream, not gateway overhead ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). The README does point at a related project, new-api-horizon, described only as a \"high-performance optimized version\" with no numbers ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "n.a. as a vendor commitment - retention is entirely your own choice via \"Log Retention Days\" and your database ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)); no zero-data-retention programme exists because there is no vendor-side storage ([Installation & Deployment](https://www.newapi.ai/en/docs/installation)).", "air_gapped": null, "best_for": "Teams that want to self-host one multi-dialect gateway and also *bill* the people behind it - per-token quotas, groups, ratios, top-ups and subscription plans - with no vendor in the request path and no vendor fee ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction), [Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token), [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)).", "strengths": [ "Three request dialects on one instance: OpenAI `/v1` (chat, completions, embeddings, images, audio, rerank, responses, realtime, models), native Anthropic Messages and native Gemini `/v1beta` proxying", "Billing is built in, not bolted on: quota accounting per user/token/model, three-tier Model/Completion/Group ratios at 1 USD = 500,000 quota, pre-consume then reconcile, cache-hit re-pricing", "Operator monetisation path out of the box - EPay, Stripe, Creem and Waffo top-ups, redemption codes, invitation rebates and subscription plans", "Practical reliability controls without a config file: channel priority plus weighted random, multi-key polling that skips failed keys, failure retry, auto-disable with failure threshold and auto-recovery time", "Access control is genuinely granular: per-token model restrictions and IP allowlists, group-based channel isolation, `auto` token group for cross-group failover", "Self-host-only and BYOK-only, so prompts never transit a vendor and there is no control plane to trust", "Very large active community and documented horizontal scaling (shared MySQL plus Redis, `NODE_TYPE` master/slave, external load balancer): 47,090 stars, 11,229 forks, pushed 2026-09-01" ], "weaknesses": [ "Heavy 2026 security record: 14 GitHub-reviewed advisories for the repo, including CVE-2026-71479 (CVSS 9.1, integer overflow in quota billing that let a user credit their own balance - confirmed exploited in the wild on 2026-07-06, fixed in v1.0.0-rc.18) and CVE-2026-64859 (root access token leaked via the user list API)", "Still v1.0.0-rc.30 after nearly three years and 47k stars - the shipping line is release candidates, and the docs themselves say stability is not guaranteed and support may not be provided", "AGPLv3: modify it and offer it as a network service and you must publish your source, unless you buy an unpriced commercial licence", "No SLA, no status page, no SOC 2/ISO 27001/HIPAA/GDPR DPA - expected for self-host-only software, but there is nothing to lean on contractually", "No Kubernetes, Helm, Terraform, MCP or OpenTelemetry support documented anywhere in the docs site or the full changelog", "Routing and channel configuration lives in the database and console, not in a reviewable declarative file", "Guardrails stop at a blocked-word list whose matching semantics, default state and failure mode are undocumented; no PII, injection or custom policy engine", "Documentation is uneven: the Technical Architecture and FAQ pages render essentially empty, the docs changelog lagged five release candidates behind GitHub, and the deepest material is Chinese-first" ], "avoid_if": "You need vendor SLAs or compliance attestations (none published), Kubernetes/Helm/Terraform deployment, MCP or OpenTelemetry, or routing config that lives in Git rather than a database ([Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog), [Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)) - or you cannot commit to tracking a fast-moving release-candidate line with a heavy 2026 advisory record ([GitHub advisories for the repo](https://api.github.com/advisories?affects=github.com/QuantumNous/new-api&ecosystem=go)).", "reviews": [ { "url": "https://hellorecon.com/blog/cve-2026-71479-new-api-quota-integer-overflow", "why": "Independent security write-up that reproduces the billing overflow against the real product (a $0.10 balance became $16,893,488,147,419.20), confirms the rc.18 fix, and gives operators a version-check command against `/api/status`.", "date": "2026-08-18", "type": "written", "title": "Unlimited AI Credits from One Integer: New API's Quota Overflow (CVE-2026-71479) and How to Check Your Gateway", "publisher": "RECON" }, { "url": "https://saas.pet/reviews/new-api-review", "why": "Hands-on operator review running new-api in production across MiniMax, DeepSeek and others, with a useful comparison against one-api, LiteLLM and Vercel AI Gateway. Note it misattributes QuantumNous as \"the team behind the Qwen open-weight models\", so treat its factual claims with care.", "date": "2026-08-20", "type": "written", "title": "new-api review: the self-hosted AI gateway that turns every LLM into one OpenAI-compatible endpoint", "publisher": "saas.pet" }, { "url": "https://www.youtube.com/watch?v=IbuOPbQS_j8", "why": "Project walkthrough covering the aggregation-and-distribution model, the supported dialects (OpenAI-compatible, Responses, Claude Messages, Gemini, Rerank, images, audio, Midjourney Proxy, Suno) and where permissions, routing, billing and format conversion sit.", "date": "2026-05-22", "type": "video", "title": "new-api：多模型 LLM Gateway 與 AI 資產管理中控台｜開源專案介紹", "publisher": "YouTube - Github雷達" }, { "url": "https://www.youtube.com/watch?v=y909K8GS3x0", "why": "End-to-end deployment demo on Debian 13: Docker install, `docker compose up -d`, initialisation, choosing the operating mode, then channel, model-mapping and token configuration - a realistic view of the setup effort.", "date": "2026-01-13", "type": "video", "title": "自建一个统一的 AI 接口：New API + Docker Compose 实操指南", "publisher": "YouTube - Nemo Cheung (lozhang)" } ], "sources": { "name": "https://github.com/QuantumNous/new-api/blob/main/README.en.md", "slug": "https://api.github.com/repos/QuantumNous/new-api", "company": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction", "license": "https://api.github.com/repos/QuantumNous/new-api", "reviews": "https://hellorecon.com/blog/cve-2026-71479-new-api-quota-integer-overflow", "category": "https://api.github.com/repos/QuantumNous/new-api", "url_docs": "https://www.newapi.ai/en/docs/api", "url_home": "https://www.newapi.ai/", "api_audio": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "api_batch": "https://www.newapi.ai/en/docs/api", "byok_mode": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "free_tier": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction", "api_images": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "deployment": "https://www.newapi.ai/en/docs/installation", "url_github": "https://api.github.com/repos/QuantumNous/new-api", "zdr_status": "https://www.newapi.ai/en/docs/installation", "log_content": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/log", "log_opt_out": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced", "model_count": "https://www.newapi.ai/", "api_surfaces": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "founded_year": "https://api.github.com/repos/QuantumNous/new-api", "github_stars": "https://github.com/QuantumNous/new-api", "cache_support": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction", "cost_controls": "https://www.newapi.ai/en/docs/guide/feature-guide/user/token", "fit_terraform": "https://www.newapi.ai/en/docs/installation", "guard_content": "https://www.newapi.ai/en/docs/guide/console/settings/operation-settings", "pricing_model": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction", "url_changelog": "https://www.newapi.ai/en/docs/guide/wiki/changelog", "vendor_status": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction", "api_embeddings": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "certifications": "https://www.newapi.ai/en/docs/legal/acceptable-use", "fit_kubernetes": "https://www.newapi.ai/en/docs/installation", "fit_model_gate": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "provider_count": "https://www.newapi.ai/", "logs_by_default": "https://www.newapi.ai/en/docs/guide/feature-guide/user/log", "tracing_support": "https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/performance-analysis", "deployment_modes": "https://www.newapi.ai/en/docs/installation", "rel_retry_config": "https://github.com/QuantumNous/new-api/blob/main/README.en.md", "fit_byok_required": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "fit_openai_dropin": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "last_release_date": "https://api.github.com/repos/QuantumNous/new-api/releases/latest", "openai_compatible": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "streaming_support": "https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables", "architecture_class": "https://api.github.com/repos/QuantumNous/new-api/languages", "fit_quickstart_url": "https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation", "fit_self_host_docs": "https://github.com/QuantumNous/new-api/blob/main/README.en.md", "guard_model_policy": "https://www.newapi.ai/en/docs/guide/feature-guide/user/token", "rel_fallback_shape": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "rel_timeout_config": "https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables", "security_incidents": "https://api.github.com/advisories/GHSA-8r8v-xf7q-rcpr", "guardrails_location": "https://www.newapi.ai/en/docs/guide/console/settings/operation-settings", "rel_fallback_config": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "api_openai_responses": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api", "self_host_data_plane": "https://www.newapi.ai/en/docs/installation", "data_export_available": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced", "api_anthropic_messages": "https://www.newapi.ai/en/docs/api/ai-model/chat/createmessage", "prompt_transits_vendor": "https://www.newapi.ai/en/docs/installation", "rel_multi_region_config": "https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment", "trains_on_customer_data": "https://www.newapi.ai/en/docs/installation", "rel_load_balancing_config": "https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel", "api_openai_chat_completions": "https://www.newapi.ai/en/docs/guide/feature-guide/user/api" }, "verified_at": { "name": "2026-09-02", "slug": "2026-09-02", "company": "2026-09-02", "license": "2026-09-02", "category": "2026-09-02", "url_docs": "2026-09-02", "url_home": "2026-09-02", "byok_mode": "2026-09-02", "deployment": "2026-09-02", "url_github": "2026-09-02", "log_content": "2026-09-02", "model_count": "2026-09-02", "api_surfaces": "2026-09-02", "founded_year": "2026-09-02", "github_stars": "2026-09-17", "cache_support": "2026-09-02", "cost_controls": "2026-09-02", "guard_content": "2026-09-02", "pricing_model": "2026-09-02", "vendor_status": "2026-09-02", "certifications": "2026-09-02", "provider_count": "2026-09-02", "logs_by_default": "2026-09-02", "tracing_support": "2026-09-02", "deployment_modes": "2026-09-02", "rel_retry_config": "2026-09-02", "last_release_date": "2026-09-02", "openai_compatible": "2026-09-02", "streaming_support": "2026-09-02", "architecture_class": "2026-09-02", "guard_model_policy": "2026-09-02", "rel_fallback_shape": "2026-09-02", "rel_timeout_config": "2026-09-02", "security_incidents": "2026-09-02", "guardrails_location": "2026-09-02", "rel_fallback_config": "2026-09-02", "api_openai_responses": "2026-09-02", "self_host_data_plane": "2026-09-02", "data_export_available": "2026-09-02", "api_anthropic_messages": "2026-09-02", "prompt_transits_vendor": "2026-09-02", "trains_on_customer_data": "2026-09-02", "rel_load_balancing_config": "2026-09-02", "api_openai_chat_completions": "2026-09-02" }, "locked_fields": [], "published": true, "updated_at": "2026-09-17T18:50:21.68859+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed at high velocity: last push 2026-09-01T13:53:35Z, 47,090 stars, 11,229 forks, 1,318 open issues, not archived ([GitHub API](https://api.github.com/repos/QuantumNous/new-api), 2026-09-02). Still on a release-candidate version string after nearly three years - the latest release is v1.0.0-rc.30 of 2026-08-31 ([releases/latest](https://api.github.com/repos/QuantumNous/new-api/releases/latest)). The docs also carry a disclaimer that the project is \"primarily intended for personal learning and research. Stability is not guaranteed, and official technical support may not be provided under the open-source license\" ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)).", "last_release_date": "2026-08-31", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "v1.0.0-rc.30, published 2026-08-31T03:55:59Z. GitHub marks it `prerelease: false` even though the tag is a release candidate, so this is the project's shipping release line rather than a preview ([releases/latest](https://api.github.com/repos/QuantumNous/new-api/releases/latest), 2026-09-02). The docs changelog page lagged behind at v1.0.0-rc.25 (\"Data updated at 2026-8-25\") when checked ([Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog)).", "prompt_transits_vendor": "no", "prompt_path_note": "No vendor is in the path: New API is distributed as software you deploy, so requests go from your client to your instance to the upstream provider whose key you configured ([Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). The AUP describes the intended scenarios as \"self-use, internal team use, and enterprise private deployment\" ([Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "logs_by_default": "metadata_only", "logs_note": "Call logs are on and are per-request metadata: time, model, tokens consumed, quota deducted and status for users ([Usage Logs](https://www.newapi.ai/en/docs/guide/feature-guide/user/log)), with username and channel name columns added for administrators plus filters by time range, username, model, channel and token name ([Log Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/log)). No prompt or completion body storage is described on either page. Error-detail logging is separately gated by `ERROR_LOG_ENABLED`, default `false` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "retention_days": null, "retention_note": "Operator-controlled: \"Log Retention Days - how many days of logs the system automatically cleans up\" is a Root-only setting with no published default ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)); logs live in your own database and can be split out with `LOG_SQL_DSN` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "zdr_status": "not_applicable", "trains_on_customer_data": "not_applicable", "training_note": "not_applicable by construction: there is no hosted service, so no operator-side data reaches QuantumNous ([Installation & Deployment](https://www.newapi.ai/en/docs/installation)). The one documented outbound call from the software is model-metadata synchronisation from `SYNC_UPSTREAM_BASE`, default `https://basellm.github.io/llm-metadata` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)). No separate training policy is published.", "subprocessors_url": null, "regions_note": "n.a. as a vendor concept - the operator chooses where to run it. Multi-node and geographically distributed deployments are documented via a shared primary database plus shared Redis, `NODE_TYPE` master/slave and an external load balancer ([Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment)); the sample container sets `TZ=Asia/Shanghai` ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Self-host-only open-source project; no attestation, trust portal or certification page found on the docs site (checked [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use), [Business Cooperation](https://www.newapi.ai/en/docs/business), [API Reference](https://www.newapi.ai/en/docs/api)).", "status": "not_published" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "No DPA or GDPR page; the AUP pushes compliance obligations onto the deployer, including \"identity management and log retention\" and filing/qualification duties ([Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "status": "not_published" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "self_hosted", "guardrails_note": "Whatever filtering exists runs inside the operator's own instance - there is no vendor service in the path ([Installation & Deployment](https://www.newapi.ai/en/docs/installation)). The AUP is explicit that governance is the deployer's job: \"The deployer is obligated to ensure that services provided through this project comply with content security requirements. Features such as blacklisting, logging, and monitoring should serve content security, abuse governance, and compliance auditing\" ([Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)). The relayed `/v1/moderations` endpoint is likewise framed as \"one of the compliance tools and does not replace the deployer's own safety governance obligations\" ([API Reference](https://www.newapi.ai/en/docs/api)).", "acquired_date_basis": null, "security_incidents": [ { "id": "CVE-2026-71479", "url": "https://github.com/advisories/GHSA-8r8v-xf7q-rcpr", "cvss": 9.1, "date": "2026-07-06", "source": "https://api.github.com/advisories/GHSA-8r8v-xf7q-rcpr", "summary": "Integer overflow in quota billing yielded negative charges, i.e. self-crediting. User-controlled quantity fields (image `n`, task duration, output-token limits) entered quota maths with no upper bound; the wrapped negative charge was applied at settlement, crediting the attacker's balance. Confirmed exploited in the wild: a community operator reported inflated balances and negative consumption entries on 2026-07-06 23:00 (UTC+8) and the maintainers shipped v1.0.0-rc.18 about two hours later, followed by rc.19 with quota-saturation warning logs. Affects <= v1.0.0-rc.17; fixed in v1.0.0-rc.18 by input bounds (MaxImageN, MaxTaskDurationSeconds) and saturating conversions in `common/quota_math.go`.", "severity": "critical" }, { "id": "CVE-2026-64859", "url": "https://github.com/advisories/GHSA-6x2c-phff-wx57", "date": "2026-08-17", "source": "https://api.github.com/advisories?affects=github.com/QuantumNous/new-api&ecosystem=go", "summary": "\"User List API Leaks Root User Access Token Leading to Privilege Escalation\" (GHSA-6x2c-phff-wx57), published 2026-08-17.", "severity": "critical" }, { "id": "CVE-2026-64868", "url": "https://github.com/advisories/GHSA-v828-m3pf-vq9q", "date": "2026-08-17", "source": "https://api.github.com/advisories?affects=github.com/QuantumNous/new-api&ecosystem=go", "summary": "\"Unauthenticated payment webhooks allow memory and disk DoS via unbounded body reads and full-body logging\" (GHSA-v828-m3pf-vq9q).", "severity": "high" }, { "id": "CVE-2026-41432", "url": "https://github.com/advisories/GHSA-xff3-5c9p-2mr4", "date": "2026-04-24", "source": "https://api.github.com/advisories?affects=github.com/QuantumNous/new-api&ecosystem=go", "summary": "\"Stripe Webhook Signature Bypass via Empty Secret Enables Unlimited Quota Fraud\" (GHSA-xff3-5c9p-2mr4).", "severity": "high" }, { "id": "advisory-volume", "url": "https://github.com/QuantumNous/new-api/security/advisories", "date": "2026-09-02", "source": "https://api.github.com/advisories?affects=github.com/QuantumNous/new-api&ecosystem=go", "summary": "Fourteen GitHub-reviewed advisories are published for `github.com/QuantumNous/new-api` (2025-11-24 through 2026-08-17), spanning two critical (quota overflow, root-token leak), five high (SSRF bypasses, Stripe webhook bypass, XSS in the Markdown renderer, SQL LIKE wildcard DoS, webhook DoS) and several medium issues. Operators should pin image tags and track releases actively.", "severity": "context" } ], "pricing_model": "oss_no_vendor_price", "pricing_model_note": "No vendor price at all: the project is AGPLv3 and free to run, with a paid **commercial licence** for AGPL-exempt use available by emailing support@quantumnous.com (price not published) ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction), business enquiries at [Business Cooperation](https://www.newapi.ai/en/docs/business)). Everything the pricing pages of hosted gateways would cover is instead operator-configured: three-tier Model/Completion/Group ratios with 1 USD = 500,000 quota points, pre-consumption followed by post-consumption reconciliation, and a default ratio of 37.5 for unpriced models in self-use mode (an error in commercial mode) ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)). The `/en/docs/guide/pricing` page is about the price table your own instance shows to your users, not a vendor price list.", "min_commitment": "None - no vendor contract exists; you commit only to your own infrastructure and upstream provider spend ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction), [Installation & Deployment](https://www.newapi.ai/en/docs/installation)).", "overage_note": "n.a. - no vendor metering. Overage semantics are yours to configure: tokens auto-disable when their remaining quota is exceeded ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)) and pre-consumption is reconciled after the call completes ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)).", "cache_support": "passthrough", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "New API does not cache responses itself; it re-prices upstream prompt-cache hits. A **Prompt Cache Ratio** between 0 and 1 is set globally or per channel - \"0.5 means cache-hit tokens are billed at 50%\" - and cache billing is documented for OpenAI, Azure, DeepSeek, Claude and Qwen channels ([Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction), [README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). The Redis and `MEMORY_CACHE_ENABLED` caches are for platform state (channels, tokens, quotas), not model responses ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)). No vendor-fixed cached-token discount exists because the operator sets the ratio.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated on [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced) or [Log Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/log); the dashboard shows consumption trends but no alert configuration is described.", "tier": null, "supported": null }, "virtual_keys": { "note": "Tokens are first-class virtual keys with expiry, remaining quota, unlimited-quota flag, model restrictions, IP allowlist and group; the secret is displayed once at creation ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)).", "tier": null, "supported": true }, "rate_limiting": { "note": "Global per-IP requests per minute/hour/day, per-group `{group: [per-minute, per-hour]}` limits with 0 meaning unlimited, plus model rate limiting in Rate Limit Settings covering total and successful request counts ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced), [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction)).", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated; the only documented webhook is the Stripe payment webhook `https://your-domain.com/api/payment/stripe/webhook` ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)).", "tier": null, "supported": null }, "model_allowlists": { "note": "Per-token model restrictions, per-channel model lists and group-based channel isolation ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token), [Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Group Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/group)).", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "Per-token \"Remaining Quota\"; the token is automatically disabled once the quota is exceeded ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)).", "tier": null, "supported": true }, "budget_caps_per_team": { "note": "Groups control channel access and billing multipliers and carry per-group rate limits, but a group-level spend cap is not stated ([Group Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/group), [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)).", "tier": null, "supported": null } }, "cost_attribution": "Per user, per token, per model and per channel, from the log tables and dashboard: user logs show quota deducted per call ([Usage Logs](https://www.newapi.ai/en/docs/guide/feature-guide/user/log)), admin logs add username and channel name ([Log Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/log)), and the console dashboard charts consumption trends. Group multipliers let cost be differentiated by user group ([Group Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/group)). No per-tag or per-customer-metadata dimension is documented.", "cost_export": "\"Enable Log Export: allow users to export their usage logs\" is a Root-only toggle; the export format is not stated on the page, and no API/webhook/S3/warehouse cost export is described ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)). Because the data is in your own MySQL/PostgreSQL, direct SQL is the practical export path ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "product_kind_note": "A gateway *and* a billing/asset-management application, not just a proxy: the README calls it a \"Next-Generation Large Model Gateway and AI Asset Management System\" ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)) and the GitHub description is \"A unified AI model hub for aggregation & distribution... A centralized gateway for personal and enterprise model management\" ([GitHub API](https://api.github.com/repos/QuantumNous/new-api), 2026-09-02). It is an open-source project developed from [One API](https://github.com/songquanpeng/one-api) and is \"Fully compatible with the original One API database\" ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "deployment_modes": [ "self_host" ], "deployment_note": "Docker image `calciumion/new-api:latest` (also published to ghcr.io); single-container form is `docker run --name new-api -d --restart always -p 3000:3000 -e TZ=Asia/Shanghai -v ./data:/data calciumion/new-api:latest`, or clone the repo and `docker-compose up -d` with the bundled compose file that already wires MySQL and Redis ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation)). SQLite is used for local single-node data, remote MySQL >= 5.7.8 or PostgreSQL >= 9.6 for multi-node; amd64/arm64 64-bit only ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "yes", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "One instance exposes three request dialects. The OpenAI surface is a base-URL swap - \"Replace OpenAI's `base_url` with the platform address and use your token as the `api_key`\" - with a published endpoint table: `POST /v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/images/generations`, `/v1/images/edits`, `/v1/audio/transcriptions`, `/v1/audio/speech`, `/v1/rerank`, `/v1/responses`, `GET /v1/realtime` (WebSocket) and `GET /v1/models` ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)). Native Claude Messages format is accepted with an `anthropic-version` header ([Native Claude Format](https://www.newapi.ai/en/docs/api/ai-model/chat/createmessage)) and Gemini requests are proxied at `/v1beta/models/{model}:{action}` ([Gemini Text Chat](https://www.newapi.ai/en/docs/api/ai-model/chat/gemini/geminirelayv1beta)). A separate Management API covers channels, tokens, users, groups, logs, payment and statistics ([API Reference](https://www.newapi.ai/en/docs/api)).", "model_count_low": 100, "model_count_high": null, "model_count_source_note": "Single vendor floor (\"100+ models\"), homepage only ([newapi.ai](https://www.newapi.ai/), observed 2026-09-02). There is no public models endpoint to count from: `GET /v1/models` is served by your own instance ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)).", "provider_count_low": 30, "provider_count_high": null, "provider_count_source_note": "Vendor total, stated as a floor: the homepage says \"30+ Model Providers\" and \"Access 30+ AI providers through a single, unified endpoint\" ([newapi.ai](https://www.newapi.ai/)); the docs repeat \"30+ other model services\" after naming OpenAI, Anthropic, Google Gemini, DeepSeek, Midjourney and Suno ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)). No enumerated channel-type list was found on the fetched pages, so 30 is the vendor floor, not a count.", "first_party_vs_partner": "All third-party: every model is served by an upstream provider whose credentials you supply - OpenAI, Anthropic, Google Gemini, DeepSeek, Midjourney, Suno \"and 30+ other model services\" ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)). QuantumNous claims no model hosting of its own on any fetched page.", "custom_endpoint_support": "Yes: every channel takes a **Base URL** field described as a \"Custom endpoint URL for proxies or self-hosted deployments\", alongside model mapping and parameter override ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). The AUP requires that upstream channels be \"accounts, API Keys, model services, or enterprise contracts legally owned or authorized by the deployer\" ([Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "byok_mode": "byok_only", "merchant_of_record": "For inference, the upstream provider you hold the key with - New API never sells tokens ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). If you resell access, *you* are the merchant of record: the software ships EPay (Alipay/WeChat/bank), Stripe, Creem and Waffo top-up integrations plus redemption codes and subscription plans, and the AUP assigns \"taxation, invoicing, consumer protection, and payment risk control\" to the deployer ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced), [Subscription Plans](https://www.newapi.ai/en/docs/guide/feature-guide/admin/subscription), [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "key_management_note": "Two key layers. Upstream provider keys are stored per channel in your database, with Multi-Key channels holding several keys that are polled automatically and skipped when failing ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)); database content is encrypted with `CRYPTO_SECRET`, which must be identical across nodes sharing Redis ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment)). Downstream, users hold New API tokens shown once at creation, each with expiry (-1 for none), remaining quota, unlimited-quota flag, model restrictions, IP allowlist and group ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)).", "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` is the first row of the supported-endpoints table and the Python example uses the OpenAI SDK against the instance base URL ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)).", "api_anthropic_messages_note": "Yes: a \"Native Claude Format\" surface accepts \"Requests in Anthropic Claude Messages API format\", requires the `anthropic-version` header and accepts `x-api-key` as an alternative to the bearer token, with `system`, `max_tokens`, `tools`, `tool_choice` and `thinking` fields ([Native Claude Format](https://www.newapi.ai/en/docs/api/ai-model/chat/createmessage)); `GET /v1/models` returns Anthropic-shaped output when `x-api-key` plus `anthropic-version` are present ([Get Model List](https://www.newapi.ai/en/docs/api/ai-model/models/list/listmodels)). The literal request path is not printed on the fetched pages - the endpoint table on [Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api) lists no `/v1/messages` row - so treat the path as undocumented even though the format is supported.", "api_openai_responses_note": "Yes: `POST /v1/responses` is listed in the endpoint table ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)) and \"OpenAI Responses API format\" is a headline feature ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). Note the README also lists OpenAI <-> OpenAI Responses *format conversion* as still \"in development\", so cross-dialect conversion into Responses is not finished ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "api_embeddings_note": "Yes: `POST /v1/embeddings` is in the endpoint table ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)) and Embeddings has its own API-reference section ([API Reference](https://www.newapi.ai/en/docs/api)).", "api_images_note": "Yes: `POST /v1/images/generations` and `POST /v1/images/edits` ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)), plus Midjourney-Proxy(Plus) integration as a separate task-based surface ([Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction)).", "api_audio_note": "Yes: `POST /v1/audio/transcriptions` and `POST /v1/audio/speech` ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)); the API reference additionally carries Audio and Real-time Speech sections and Suno music tasks ([API Reference](https://www.newapi.ai/en/docs/api), [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction)).", "api_batch_note": "n.a. - no batch or bulk-async inference endpoint appears in the endpoint table ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)) or the API reference index, where Files and Fine-tuning are explicitly grouped under \"Unimplemented\" ([API Reference](https://www.newapi.ai/en/docs/api)). Asynchronous *task* APIs exist, but they are for video/Midjourney/Suno generation, not OpenAI-style batch ([Create Video](https://www.newapi.ai/en/docs/api/ai-model/videos/sora/createvideo)).", "deployment_modes_note": "Self-host only. Documented methods are Docker Compose (recommended for production), single-container Docker, 1Panel, BaoTa/aaPanel app store, cluster deployment and local development ([Installation & Deployment](https://www.newapi.ai/en/docs/installation)); there is no vendor-hosted tier on any page fetched - the homepage's own framing is \"Open source as our covenant, self-hosted as our ground\" ([newapi.ai](https://www.newapi.ai/)).", "byok_mode_note": "byok_only: a channel *is* one upstream provider API key, and the AUP requires those keys be legally owned or authorised by the deployer ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)). There are no platform credits and no first-party model hosting.", "access_meta": { "qualified": [ "api_anthropic_messages", "model_count", "provider_count" ], "model_count_kind": "vendor_total", "provider_count_kind": "vendor_total" }, "rel_timeout_config": "config_file", "rel_timeout_note": "Environment variables only: `RELAY_TIMEOUT` defaults to `0`, meaning **no timeout**, `STREAMING_TIMEOUT` is 300 s between chunks, `TASK_TIMEOUT_MINUTES` is 1440 for async tasks, and connection pooling is tuned with `RELAY_MAX_IDLE_CONNS` 500 / `RELAY_MAX_IDLE_CONNS_PER_HOST` 100. The docs explicitly warn that setting the relay timeout too low can produce requests the upstream charges for but New API never bills ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "rel_retry_config": "dashboard_only", "rel_retry_note": "\"Automatic retry on failure\" is a listed routing feature and the retry count is a console setting at `Settings -> Operation Settings -> General Settings -> Failure Retry Count`; the default value and any backoff strategy are not published ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). Default retry count: `n.a.` Backoff: `n.a.`", "rel_retry_default_note": null, "rel_fallback_config": "dashboard_only", "rel_fallback_note": "Fallback is channel-level and configured per channel in the console: **Priority** (\"higher value = higher selection priority\", default 0) selects the tier and **Weight** is the \"random weight among same-priority channels\" (default 0), so selection within a tier is weighted random while tiers are ordered ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)); the README's headline phrasing is \"Channel weighted random\" ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). Token groups can be set to `auto`, which \"automatically selects an available group in priority order - useful for cross-group failover scenarios\" ([Group Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/group)). Multi-Key channels additionally poll keys in Round Robin or Weighted Random order and skip a failed key until it recovers ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)).", "rel_fallback_shape": "weighted", "rel_health_check_config": null, "rel_health_check_note": "There is health-driven channel disabling, but no enum here fits: it is configured in the console, not a config file, and it is certainly not \"not_configurable\". Channels have an **Auto Disable** option that \"automatically disables the channel after consecutive failures\" ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)), and Model Behavior Settings expose \"Auto-disable Failed Models\", a \"Failure Threshold\" and an \"Auto-recovery Time (minutes)\" ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)). Manual \"Test\" / \"Test All Channels\" actions report per-channel response time ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). No periodic active health-probe interval or circuit-breaker semantics are published.", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Multi-node clustering is documented (shared MySQL, shared Redis, identical `SESSION_SECRET` and `CRYPTO_SECRET`, `NODE_TYPE` master/slave, `SYNC_FREQUENCY` 60 s) and geographic distribution is *suggested*, but there is no configurable cross-region failover or region-pinning mechanism described ([Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment)).", "rel_load_balancing_config": "dashboard_only", "rel_load_balancing_note": "Weights and priorities are user-settable per channel in `/console/channel`, with model mapping and per-channel parameter override alongside ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)); there is no routing config file - the channel table lives in the database. Cluster-level balancing across nodes is left to an external load balancer such as Nginx or HAProxy ([Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment)).", "architecture_class": "compiled_binary", "architecture_note": "Single Go service with an embedded web console: repo language bytes are Go 6,401,798 (~76%) and TypeScript 1,657,241, plus small JS/CSS/Lua/Shell ([GitHub languages API](https://api.github.com/repos/QuantumNous/new-api/languages), 2026-09-02); the repo primary language is Go and it ships a Dockerfile at root ([GitHub API repo](https://api.github.com/repos/QuantumNous/new-api)). Runtime dependencies are a database (SQLite, MySQL or PostgreSQL) and optionally Redis ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "self_host_data_plane": "yes", "self_host_artifact_note": "Docker image (`calciumion/new-api:latest`, ghcr.io mirror), a bundled `docker-compose.yml` in the repo, and panel one-click installs (1Panel, BaoTa >= 9.2.0) ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Installation & Deployment](https://www.newapi.ai/en/docs/installation)). No Helm chart or Kubernetes manifests exist in the repository root tree ([GitHub API repo](https://api.github.com/repos/QuantumNous/new-api), 2026-09-02).", "latency_claims": null, "throughput_claims": null, "streaming_support": "yes", "streaming_note": "Supported: chat requests take a `stream` boolean ([Native Claude Format](https://www.newapi.ai/en/docs/api/ai-model/chat/createmessage)) and Gemini streaming is proxied via `:streamGenerateContent?alt=sse` ([Gemini Text Chat](https://www.newapi.ai/en/docs/api/ai-model/chat/gemini/geminirelayv1beta)). Operationally, `STREAMING_TIMEOUT` defaults to 300 s (time between streamed chunks), `STREAM_SCANNER_MAX_BUFFER_MB` defaults to 64 and `FORCE_STREAM_OPTION` defaults to true; the docs warn that a too-short relay timeout can leave a request billed upstream but not recorded locally ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "guard_pii": "not_documented", "guard_pii_note": "n.a. - no PII detection, masking or redaction on any fetched page: [Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings), [System Settings](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting), [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced), [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction), [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use), and the full docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog) (93 KB, no occurrence of PII/redaction/guardrail terms).", "guard_injection": "not_documented", "guard_injection_note": "n.a. - no prompt-injection or jailbreak detection is described on [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction), [Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings), [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced) or in the docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog).", "guard_content": "sync_block", "guard_content_note": "A blocked-word (\"blocked words\") facility is part of Operational Settings - \"Here you can navigate to operational settings such as top-up links, documentation addresses, blocked words, logging, monitoring, and quotas\" ([Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings)) - and the AUP counts blacklisting among the content-security features ([Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)). No page fetched documents the matching mode, default state, or the error returned, so treat the semantics as undocumented; the request-time rejection behaviour is visible only in community reports of `message contains sensitive words` errors. Separately, `POST /v1/moderations` is available as a relayed upstream endpoint rather than an enforced gateway policy ([Create Moderation](https://www.newapi.ai/en/docs/api/ai-model/moderations/createmoderation), [API Reference](https://www.newapi.ai/en/docs/api)).", "guard_custom": "not_documented", "guard_custom_note": "n.a. - there is no custom policy/rule engine for request content. The closest configurable hooks are channel-level **Parameter Override** JSON and request-field passthrough selection, which shape requests rather than police them ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog) rc.25 entry on choosing which request fields pass through).", "guard_model_policy": "yes", "guard_model_policy_note": "Yes, and it is one of the product's strong points: tokens carry **Model Restrictions** and an **IP Allowlist** ([Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)), channels declare which models they serve and the model list can hide upstream models, and groups \"isolate channel access permissions and billing multipliers\" across user, token and channel scopes ([Group Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/group), [Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)).", "guard_execution_location": "not_documented", "guard_third_party": null, "guard_failure_mode": "not_documented", "guard_failure_mode_note": "n.a. - no fail-open/fail-closed statement for the blocked-word path on [Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings) or [System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced).", "log_content": "metadata_only", "log_content_note": "Metadata only as documented: the admin and user log pages enumerate time, username, model, channel, token name, tokens, quota and status columns and never mention request/response bodies ([Log Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/log), [Usage Logs](https://www.newapi.ai/en/docs/guide/feature-guide/user/log)). Logs can be written to a separate database with `LOG_SQL_DSN` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "log_opt_out": "not_documented", "log_opt_out_note": "No switch to disable call logging is documented. Other Settings expose \"Enable Log Export\" and \"Log Retention Days\" (automatic cleanup), and Operational Settings lists \"logging\" among its sections without detailing an off switch ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced), [Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings)).", "tracing_support": "not_documented", "tracing_note": "n.a. - no OpenTelemetry, OTLP or distributed-tracing support appears on any fetched page, including the full docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog) (no occurrence of OTel/OpenTelemetry/Prometheus). What exists is profiling: Go `pprof` at `/debug/pprof/` behind `ENABLE_PPROF`, and optional Pyroscope continuous profiling via `PYROSCOPE_URL` / `PYROSCOPE_APP_NAME` with mutex and block rate controls ([Performance Analysis](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/performance-analysis), [Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "export_destinations": null, "export_destinations_note": "n.a. - no log/metric shipping destinations (OTLP collectors, Kafka, S3, webhooks) are documented; the only stated egress is the user-facing log export toggle whose format is unspecified ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)) and a separate log database via `LOG_SQL_DSN` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "feedback_capture": "no", "feedback_capture_note": "n.a. - no rating, thumbs-up/down or feedback ingestion endpoint on [API Reference](https://www.newapi.ai/en/docs/api), [Usage Logs](https://www.newapi.ai/en/docs/guide/feature-guide/user/log) or [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction).", "eval_hooks": "no", "eval_hooks_note": "n.a. - no evaluation, scoring or dataset feature. The API reference index covers model relay plus management APIs only, with Files and Fine-tuning listed as \"Unimplemented\" ([API Reference](https://www.newapi.ai/en/docs/api)); the built-in Playground at `/console/playground` is manual testing, not evaluation ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)).", "observability_requires_saas": "No - the dashboard, logs and consumption charts are part of the self-hosted application and there is no vendor control plane to register with ([Log Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/log), [Installation & Deployment](https://www.newapi.ai/en/docs/installation)).", "log_retention_note": "Retention is a number you choose in Other Settings rather than a vendor policy; there is no vendor-side copy of the data at all because the deployment is entirely yours ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced), [Installation & Deployment](https://www.newapi.ai/en/docs/installation)).", "production_meta": { "sources": {}, "qualified": [ "guard_content", "logs_by_default", "log_content" ], "latency_prose": "No vendor latency claim of any kind was found. The performance documentation is about instrumentation (pprof endpoint behind `ENABLE_PPROF`, optional Pyroscope continuous profiling) and publishes no numbers ([Performance Analysis](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/performance-analysis)); the homepage's only figures are adoption counts - \"40K+ GitHub Stars\", \"30+ Model Providers\", \"500+ Releases Shipped\" ([newapi.ai](https://www.newapi.ai/)).", "multi_surface": { "rel_timeout": [ "config_file" ] }, "throughput_prose": "No throughput or QPS figure published on the README, homepage, performance page or the full docs changelog ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Performance Analysis](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/performance-analysis), [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog)). The related project new-api-horizon is described as a \"high-performance optimized version\" with no measurement attached ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "benchmark_independence": "Nothing to assess: there are no vendor benchmarks, and no third-party benchmark of New API was found. Per-channel \"response time\" in the console measures the operator's upstream provider, not gateway overhead ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Reliability here is operator-assembled rather than declarative: retries, channel priority/weight, multi-key polling and auto-disable thresholds are all console settings backed by the database, while timeouts and cache/sync behaviour are environment variables ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)). Nothing in the fetched docs publishes an SLA, uptime target or status page, which is consistent with self-host-only distribution ([Installation & Deployment](https://www.newapi.ai/en/docs/installation)).", "guard_notes": "Treat New API as a gateway with access control, not a guardrails platform. What it enforces well is *who may call which model with how much quota*; what it does not publish is any content, PII, injection or custom-policy engine beyond a blocked-word list whose behaviour is undocumented ([Operational Settings](https://www.newapi.ai/en/docs/guide/console/settings/operation-settings), [Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token), [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "fit_agent_features_note": "Tool calling passes through in both dialects: the OpenAI surface is a drop-in for the OpenAI SDK ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)) and the native Claude surface accepts `tools`, `tool_choice` and `thinking` ([Native Claude Format](https://www.newapi.ai/en/docs/api/ai-model/chat/createmessage)). Reasoning is controlled by model-name suffixes (`-high`/`-medium`/`-low`, `claude-...-thinking`, `gemini-2.5-flash-nothinking`, `-thinking-128`) with a per-channel `thinking_to_content` option, default false ([Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction)); a rc.25 changelog entry notes usage logs now record reasoning effort consistently ([Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog)). Agent CLIs are supported as clients rather than orchestrated by the gateway ([Verified Apps](https://www.newapi.ai/en/docs/apps)).", "fit_byok_required": "yes", "fit_byok_required_note": "Yes - there is no other way to get a model: a channel is defined by an upstream provider API key, and the AUP insists those keys be legally owned or authorised by the deployer ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Acceptable Use](https://www.newapi.ai/en/docs/legal/acceptable-use)).", "fit_client_sdk_langs": [ "Python" ], "fit_client_sdks_note": "The only language shown by name on the fetched integration page is Python via the OpenAI SDK (with Claude-native and Gemini-native request examples alongside); the code blocks themselves did not render in the fetched text ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)). Verified third-party clients are listed instead of SDKs - AionUi, Cherry Studio, DeepChat, OpenClaw, Claude Code, Codex CLI, Factory Droid CLI and others, all configured with just API address, key and model name ([Verified Apps](https://www.newapi.ai/en/docs/apps)).", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Authentication to upstreams is by provider API key held in the channel record; no cloud IAM role assumption (AWS SigV4, GCP service accounts, Azure managed identity) is documented ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel)). For human access, the console supports Discord, LinuxDO and Telegram OAuth plus OIDC unified authentication and two-factor auth ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [API Reference](https://www.newapi.ai/en/docs/api)).", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. - it is a Go server that needs a database, not an edge worker; Cloudflare Workers appear nowhere on [Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment) or the docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog).", "fit_credit_card": "not_required", "fit_credit_card_note": "Not required: the software is free under AGPLv3 and installs from a public Docker image with no account ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction), [README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). Payment integrations in the product are for charging *your* users, not for paying QuantumNous ([System Settings - Detailed](https://www.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced)).", "fit_ecosystem_notes": "Strong client-side ecosystem, weak platform-side one. Verified apps include AionUi, CC Switch, Cherry Studio, DeepChat, Memoh, OpenClaw, Fluent Read, LangBot, Luna Translator, AstrBot, Claude Code, Codex CLI and Factory Droid CLI, each needing only API address, key and model name ([Verified Apps](https://www.newapi.ai/en/docs/apps)); official Skill plugins target Claude Code, Codex CLI, OpenClaw, Cursor, Windsurf and Cline from the `QuantumNous/skills` repo ([Skills](https://www.newapi.ai/en/docs/skills)). Against that, no Kubernetes, Terraform, MCP or OpenTelemetry story is documented ([Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog)). Multi-language UI covers Chinese, English, French and Japanese, and much of the deeper documentation is Chinese-first ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "fit_eval_prereqs": "Docker, a host to run it on, and an upstream provider key. There is no trial account, sign-up or credit card: `docker run ... -v ./data:/data calciumion/new-api:latest`, then browse to `http://localhost:3000` and set up the administrator account ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation)). SQLite is used by default for a local single-node instance ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "fit_eval_shape": "cli_or_container", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "Partial. A first-call snippet exists in principle - the page has Python (OpenAI SDK), Claude-native and Gemini-native examples - but the code blocks did not render in the fetched page text, so only the endpoint table and the base-URL-swap instruction are verifiable ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)). The deployment commands themselves are fully published ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)).", "fit_gateway_platform": "is_itself_gateway", "fit_gateway_platform_note": "It *is* the gateway: \"an AI API gateway and usage management system designed for legally authorized scenarios\" ([Project Introduction](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/project-introduction)); the GitHub topics include `ai-gateway` ([GitHub API](https://api.github.com/repos/QuantumNous/new-api)).", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. - Kubernetes is never mentioned on the fetched pages: [Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation), [Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment) (which scales with Docker plus an external Nginx/HAProxy load balancer instead) or the full docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog). The repository root tree contains no `helm/` or `k8s/` directory ([GitHub API repo](https://api.github.com/repos/QuantumNous/new-api), 2026-09-02).", "fit_mcp": null, "fit_mcp_note": "No MCP support found, and this is a real gap for agent stacks: \"MCP\" does not occur on any page fetched, including [API Reference](https://www.newapi.ai/en/docs/api), [Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction), [Verified Apps](https://www.newapi.ai/en/docs/apps) and the entire 93 KB docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog). What the project ships instead is its own \"Skill\" plugins - explicitly described as \"a lightweight extension protocol\" - installed with a single `npx` command into Claude Code, Codex CLI, OpenClaw, Cursor, Windsurf and Cline, exposing `/newapi models`, `/newapi groups`, token and balance commands; a `newapi-admin` Skill is marked \"Coming Soon\" ([Skills](https://www.newapi.ai/en/docs/skills)).", "fit_meta": { "not_documented_fields": [ "fit_kubernetes", "fit_terraform", "fit_cloudflare_workers", "fit_vercel_ai_sdk", "fit_mcp" ] }, "fit_model_gate": "enablement_step", "fit_model_gate_note": "Every model must be enabled by an administrator before it is callable: you create a channel with a provider key and the models it serves (optionally syncing the upstream model list with an Added/Changed/Deleted preview), then grant access through groups and per-token model restrictions ([Channel Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/channel), [Model Management](https://www.newapi.ai/en/docs/guide/feature-guide/admin/model), [Token Management](https://www.newapi.ai/en/docs/guide/feature-guide/user/token)). No vendor approval or waitlist exists - the gate is your own.", "fit_onboarding_notes": "First run is guided: after `docker-compose up -d`, visiting `http://Server_IP:3000` \"will automatically redirect to the initialization page\" where you set the administrator account and password ([Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation)). Then the homepage's three-step arc applies - configure channels, deploy and integrate, monitor and optimise ([newapi.ai](https://www.newapi.ai/)). Two documentation caveats: the Technical Architecture page and the support FAQ rendered essentially empty when fetched (149 characters and headings only respectively), and the docs changelog trails GitHub releases ([Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog) showed rc.25 against rc.30 on the API).", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes, stated plainly: \"Replace OpenAI's `base_url` with the platform address and use your token as the `api_key`\", with the OpenAI Python SDK as the first example and an eleven-row `/v1/...` endpoint table ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)). The homepage frames it as a \"100% OpenAI compatible endpoint for all AI providers\" ([newapi.ai](https://www.newapi.ai/)).", "fit_prod_prereqs": "Remote MySQL >= 5.7.8 or PostgreSQL >= 9.6 (SQLite is single-node only), Docker and Docker Compose, and a 64-bit amd64/arm64 host ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)). Multi-node adds shared Redis, an identical `SESSION_SECRET` across all nodes and an identical `CRYPTO_SECRET` when Redis is shared, `NODE_TYPE` master/slave, and an external load balancer such as Nginx or HAProxy ([Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment), [README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)); Redis is the recommended cache backing via `REDIS_CONN_STRING` ([Environment Variables](https://www.newapi.ai/en/docs/installation/config-maintenance/environment-variables)).", "fit_prod_selfhost_shape": "infra_rollout", "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain" ], "fit_python_frameworks_note": "Thin evidence: the homepage's \"Powering AI applications worldwide\" strip lists LangChain, Dify, FastGPT, n8n, Open WebUI, LobeHub, Cherry Studio, Cline, Roo Code and Coze as applications that run on New API ([newapi.ai](https://www.newapi.ai/)), and the Rerank feature is documented as integrating with Dify ([Features Description](https://www.newapi.ai/en/docs/guide/wiki/basic-concepts/features-introduction)). There is no LangChain or LlamaIndex code-level integration page; LlamaIndex is not mentioned at all on the fetched pages.", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": "Counted from the README's three-command quick start (clone the repo, edit `docker-compose.yml`, `docker-compose up -d`) ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md)); the homepage independently frames it as \"3 steps\" - configure channels, deploy and integrate, monitor and optimise ([newapi.ai](https://www.newapi.ai/)). The docs deployment page is prose rather than a numbered list, and first login additionally requires creating the administrator account ([Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation)).", "fit_quickstart_url": "https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation", "fit_self_host_docs": "command_published", "fit_self_host_install": "`docker run --name new-api -d --restart always -p 3000:3000 -e TZ=Asia/Shanghai -v ./data:/data calciumion/new-api:latest`, or `git clone` plus `docker-compose up -d` with the bundled compose file; add `-e SQL_DSN=\"root:123456@tcp(localhost:3306)/oneapi\"` for MySQL ([README.en.md](https://github.com/QuantumNous/new-api/blob/main/README.en.md), [Docker Compose Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/docker-compose-installation)).", "fit_terraform": "not_documented", "fit_terraform_note": "n.a. - no Terraform provider, module or example on [Installation & Deployment](https://www.newapi.ai/en/docs/installation), [Cluster Deployment](https://www.newapi.ai/en/docs/installation/deployment-methods/cluster-deployment) or in the docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog), and no `terraform/` directory in the repository root ([GitHub API repo](https://api.github.com/repos/QuantumNous/new-api)).", "fit_terraform_ref": null, "fit_third_party_documented": null, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "n.a. - neither the Vercel AI SDK nor an `@ai-sdk/*` package is named on [Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api), [Verified Apps](https://www.newapi.ai/en/docs/apps) or the docs [Changelog](https://www.newapi.ai/en/docs/guide/wiki/changelog). In practice the documented OpenAI base-URL swap is what a Vercel AI SDK app would use, but the vendor does not document it ([Using the API](https://www.newapi.ai/en/docs/guide/feature-guide/user/api)).", "fit_vercel_ai_sdk_package": null }, { "slug": "amazon-bedrock", "name": "Amazon Bedrock", "company": "Amazon Web Services", "tagline": "AWS-managed service for calling foundation models from 19 model providers through one AWS API.", "category": "cloud_platform", "deployment": "managed_only", "license": "Proprietary", "founded_year": null, "hq_country": "United States", "url_home": "https://aws.amazon.com/bedrock/", "url_docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html", "url_pricing": "https://aws.amazon.com/bedrock/pricing/", "url_status": null, "url_github": null, "url_changelog": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html", "url_models_api": null, "github_stars": null, "model_count": 100, "model_count_note": "Docs say \"100+ foundation models\"; the product page says \"hundreds of foundation models\" — no exact catalog count is published.", "provider_count": 19, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": null, "free_tier": "No Bedrock-specific free tier; new AWS customers get up to $200 in AWS credits ($100 immediately plus up to $100 earned, over 6 months).", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Intelligent Prompt Routing", "amount": "$1.00 per 1,000 requests" }, { "label": "Guardrails content filters", "amount": "$0.15 per 1,000 text units" }, { "label": "Guardrails sensitive information filter", "amount": "$0.10 per 1,000 text units" }, { "label": "Guardrails Automated Reasoning checks", "amount": "$0.17 per 1,000 text units" }, { "label": "Prompt Optimizer", "amount": "$0.03 per 1,000 tokens" }, { "label": "Knowledge Bases vector index storage", "amount": "$5.00 per GB per month" }, { "label": "Knowledge Bases Retrieve API", "amount": "$1.00 per 1,000 calls (Agentic Retrieval $4.00 per 1,000)" }, { "label": "Human evaluation of model output", "amount": "$0.21 per task" } ], "self_host_cost_note": "Managed only, no self-host option. Bedrock sets its own per-token prices rather than marking up a third party: Standard, Flex and Batch (both 50% below Standard) and Priority (75% above Standard) tiers, plus Provisioned Throughput billed per model unit-hour with no-commitment, 1-month or 6-month terms.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "boto3 remains the primary SDK, but the newer endpoint accepts the OpenAI client.", "api_surfaces": [ "Amazon Bedrock InvokeModel", "Converse API", "OpenAI Chat Completions (/openai/v1)", "OpenAI Responses API", "Anthropic Messages" ], "sdk_note": "The OpenAI SDK works by setting OPENAI_BASE_URL to https://bedrock-runtime.{region}.amazonaws.com/openai/v1; the older bedrock-mantle /v1 route remains supported. Anthropic's client can target https://bedrock-runtime.{region}.amazonaws.com/anthropic.", "cap_failover": null, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": null, "cap_prompt_management": null, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": true, "cap_audio": null, "cap_video": true, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": null, "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": true, "eu_residency": null, "eu_residency_note": "Bedrock supports Regional inference plus Global and Geo cross-Region inference profiles; the specific EU Region list for Bedrock was not confirmed from vendor docs in this research.", "zdr": true, "zdr_note": "Enforceable rather than promised: with retention mode set to none, no request or response is written to durable storage or shared with the provider, and Bedrock blocks requests to models that require retention. Access is evaluated per account and per model with the provider.", "air_gapped": null, "best_for": "Teams already standardized on AWS that need many model vendors behind one IAM-governed, compliance-attested API.", "strengths": [ "19 model providers plus custom model import under one AWS API and one bill", "Deep compliance posture: ISO, SOC, CSA STAR Level 2, GDPR, FedRAMP High, HIPAA-eligible with an AWS BAA", "Built-in Guardrails, evaluations, Knowledge Bases and AgentCore instead of bolt-on tooling", "Cost levers: Batch and Flex at 50% off Standard, prompt caching, Intelligent Prompt Routing, Provisioned Throughput commitments", "OpenAI-compatible Responses and Chat Completions endpoints ease migration" ], "weaknesses": [ "Not a cross-cloud gateway: it only routes to models AWS hosts, so no failover to OpenAI, Azure or Google endpoints", "Deep AWS lock-in — IAM, regions and fine-tuned artifacts do not port to Vertex AI or Foundry", "Pricing is fragmented across tiers, Provisioned Throughput units and per-feature surcharges, making forecasts hard", "Model availability varies by Region, so a model you tested may be unavailable where your data must live" ], "avoid_if": "You want vendor-neutral routing across multiple clouds or a single flat per-token price list.", "reviews": [ { "url": "https://www.doit.com/blog/the-engineering-guide-to-amazon-bedrock-cost-optimization", "why": "Third-party engineering guide covering Bedrock on-demand vs provisioned throughput, batch inference, intelligent prompt routing and prompt caching from customer engagements.", "date": "2026-04-16", "type": "written", "title": "The Engineering Guide to Amazon Bedrock Cost Optimization", "publisher": "DoiT" }, { "url": "https://www.constellationr.com/insights/news/amazon-bedrock-vs-diy-approaches-benchmarked", "why": "Analyst writeup of Holger Mueller's benchmark comparing Bedrock's managed model access against do-it-yourself model hosting on cost and time-to-deploy.", "date": "2024-12-04", "type": "written", "title": "Amazon Bedrock vs. DIY approaches benchmarked", "publisher": "Constellation Research" }, { "url": "https://www.youtube.com/watch?v=y1slHK9R_Vg", "why": "Independent AWS-focused channel deploys an agent to Bedrock AgentCore Runtime end to end and tests Gateway, Identity, Memory and cross-session behaviour.", "date": "2026-03-11", "type": "video", "title": "AWS Bedrock AgentCore - The Complete Beginners Guide", "publisher": "YouTube - Johnny Chivers" }, { "url": "https://news.ycombinator.com/item?id=48473166", "why": "Practitioners describe concrete blast radius of the data-sharing change: zero-data-retention contract clauses, regulated-industry escalations, and cost comparisons with buying Anthropic directly.", "date": "2026-06-10", "type": "community", "title": "AWS Bedrock to require sharing data with Anthropic", "publisher": "Hacker News" }, { "url": "https://www.reddit.com/r/aws/comments/1qt3gne/aws_bedrock_in_production_anyone_else_finding_it/", "why": "28-comment thread from teams running Bedrock ~6 months, focused on throttling, capacity limits and prototypes breaking at scale including during the us-east-1 outage.", "date": "2026-02-01", "type": "community", "title": "AWS Bedrock in production: anyone else finding it a mixed bag?", "publisher": "Reddit - r/aws" } ], "sources": { "zdr": "https://aws.amazon.com/bedrock/", "gdpr": "https://aws.amazon.com/bedrock/", "name": "https://aws.amazon.com/bedrock/", "soc2": "https://aws.amazon.com/bedrock/", "company": "https://aws.amazon.com/bedrock/", "tagline": "https://aws.amazon.com/bedrock/pricing/", "sdk_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html", "url_docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html", "url_home": "https://aws.amazon.com/bedrock/", "zdr_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/data-protection.html", "cap_batch": "https://aws.amazon.com/bedrock/pricing/", "cap_evals": "https://aws.amazon.com/bedrock/pricing/", "cap_video": "https://aws.amazon.com/bedrock/pricing/", "free_tier": "https://aws.amazon.com/free/", "hipaa_baa": "https://aws.amazon.com/compliance/hipaa-eligible-services-reference/", "extra_fees": "https://aws.amazon.com/bedrock/pricing/", "hq_country": "https://en.wikipedia.org/wiki/Amazon_(company)", "zdr_status": "https://docs.aws.amazon.com/bedrock/latest/userguide/data-retention.html", "model_count": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html", "url_pricing": "https://aws.amazon.com/bedrock/pricing/", "api_surfaces": "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-openai-chat-completions.html", "product_fate": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-ug-doc-history.html", "regions_note": "https://aws.amazon.com/bedrock/faqs/", "cap_image_gen": "https://aws.amazon.com/bedrock/pricing/", "url_changelog": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html", "vendor_status": "https://aws.amazon.com/bedrock/sla/", "cap_guardrails": "https://aws.amazon.com/bedrock/pricing/", "certifications": "https://aws.amazon.com/bedrock/faqs/", "config_as_code": "https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-bedrock-intelligentpromptrouter.html", "provider_count": "https://aws.amazon.com/bedrock/pricing/", "retention_days": "https://docs.aws.amazon.com/bedrock/latest/userguide/abuse-detection.html", "sla_uptime_pct": "https://aws.amazon.com/bedrock/sla/", "bedrock_pricing": "https://aws.amazon.com/bedrock/pricing/", "logs_by_default": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html", "tracing_support": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html", "cap_cache_simple": "https://aws.amazon.com/bedrock/pricing/", "model_count_note": "https://aws.amazon.com/bedrock/", "cap_pii_redaction": "https://aws.amazon.com/bedrock/pricing/", "eu_residency_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-regions.html", "last_release_date": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-ug-doc-history.html", "openai_compatible": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html", "subprocessors_url": "https://aws.amazon.com/compliance/sub-processors/", "cap_load_balancing": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-regions.html", "guardrails_location": "https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", "requires_vendor_sdk": "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html", "self_host_cost_note": "https://aws.amazon.com/bedrock/pricing/", "data_export_available": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html", "bedrock_prompt_caching": "https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html", "prompt_transits_vendor": "https://aws.amazon.com/bedrock/faqs/", "cap_conditional_routing": "https://aws.amazon.com/bedrock/pricing/", "trains_on_customer_data": "https://aws.amazon.com/bedrock/faqs/" }, "verified_at": { "zdr": "2026-08-29", "gdpr": "2026-08-29", "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-08-29", "cap_batch": "2026-08-29", "cap_evals": "2026-08-29", "cap_video": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "extra_fees": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "provider_count": "2026-08-29", "tracing_support": "2026-09-05", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-05T20:16:27.577312+00:00", "vendor_status": "hyperscaler", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "A first-party AWS service under continuous development, bought through an existing AWS contract rather than as a new vendor relationship.", "last_release_date": "2026-08-24", "sla_uptime_pct": 99.9, "sla_url": "https://aws.amazon.com/bedrock/sla/", "claim_basis": {}, "last_release_note": "AWS publishes no versioned Bedrock changelog, so this is the most recent entry in the user guide's document history — a documentation update, not a product release.", "prompt_transits_vendor": "yes", "prompt_path_note": "Serverless inside AWS, and you can keep it off the public internet with PrivateLink. Model providers are walled off: they have no access to Bedrock logs or to your prompts and completions.", "logs_by_default": "none", "logs_note": "Model invocation logging is disabled by default, and when you enable it the logs go to your own S3 or CloudWatch in the same account and region. Bedrock's stated baseline is a zero-data-retention model.", "retention_days": 0, "retention_note": "Nothing stored by default. Named exceptions run to 30 days for certain third-party models, including classifier-flagged traffic on some OpenAI models and inputs and outputs on some Anthropic models.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Your content is not used to improve base models and is not shared with model providers. Neither AWS nor the third-party providers use Bedrock inputs or outputs to train any model.", "subprocessors_url": "https://aws.amazon.com/compliance/sub-processors/", "regions_note": "Content is encrypted and stored at rest in the region where you use Bedrock. Cross-region inference moves retained data to the processing region. PrivateLink reaches into your VPC, but the service itself is not customer-hosted.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "in scope for SOC 1, 2 and 3", "status": "certified" }, "fedramp": { "detail": "Moderate/Class C in commercial regions; High/Class D in GovCloud", "status": "certified" }, "gdpr_dpa": { "detail": "AWS DPA and subprocessor notice", "status": "certified" }, "iso27001": { "detail": "in scope, alongside 27017, 27018, 27701 and others", "status": "certified" }, "hipaa_baa": { "detail": "HIPAA-eligible; standard BAA available", "status": "certified" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Sensitive-information filters block or mask PII inside Bedrock, and can be called through the ApplyGuardrail API without invoking a model — but they run on AWS's side.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference, bundled routing: gateway/API layer carries no separately stated fee. Per-token (on-demand), per batch token, or per provisioned model unit hour. Service tiers: Priority +75% to Standard; Flex −50%; Batch −50%.", "min_commitment": "None for on-demand. Provisioned Throughput uses 1-month and 6-month commitments (e.g. Cohere Command $39.60/hr/model unit for 1 mo vs $23.77 for 6 mo; Llama 2 70B $21.18 vs $13.08); pricing requires contacting your account team.", "overage_note": "No log/trace retention overage, request-volume overage or data-export charge. Adjacent charges are usage-metered: model customization, storage, retrieval, guardrails, evaluations, Advanced Prompt Optimizer (charged for on-demand Standard-tier tokens used in optimization).", "cache_support": "hybrid", "cache_read_discount_pct": 90, "cache_write_premium_pct": 25, "cache_pricing_note": "Implicit and explicit prompt caching (prefix-based, not semantic). Explicit uses cache checkpoints (up to 4 per request); TTLs commonly 5 min (Anthropic 1h option, GPT-5.6 default 30m). Implicit caching is best-effort — identical prompts don't guarantee a hit. Cache reads billed at 90% discount for GPT-5.6 (Claude 3.5 Sonnet v2 cache read $0.60 vs $6.00 input = 90% off on the pricing page). Cache writes billed at 1.25× uncached input rate (+25%). Cache saves money conditionally: writes cost 25% more up front and hits are never guaranteed, so a low-hit-rate workload can pay more than it saves. AWS captures nothing extra beyond the write premium.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "rate_limiting": { "note": "Not stated as a cost control on the pricing page.", "tier": null, "supported": null }, "spend_webhooks": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated on the pricing page.", "tier": null, "supported": null } }, "cost_attribution": "Not stated on the pricing page. Cost shaping levers Bedrock does publish: service tier (Flex/Priority), batch, Provisioned Throughput commitments.", "cost_export": "Not stated on the pricing page.", "product_kind_note": "AWS calls Bedrock \"a fully managed service that provides secure, enterprise-grade access to high-performing foundation models from leading AI companies, enabling you to build and scale generative AI applications\" ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html)); the FAQ adds that it is \"serverless,\" so customers \"don't have to manage any infrastructure\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). It is a cloud model platform, not a third-party routing proxy.", "deployment_modes": [ "saas", "hybrid_vpc" ], "deployment_note": "There is no customer-installed gateway: applications call regional `bedrock-runtime` (or `bedrock-mantle`) endpoints ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html); [Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). Customer-owned weights arrive through Custom Model Import, which \"only supports Llama 2/3, Mistral, and Flan architectures\", and models fine-tuned in SageMaker can be brought back only if their architecture is import-supported ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)). Feature gating is by region/eligibility rather than tier — e.g. Guardrails Standard tier \"requires opting into cross-region inference,\" and some models are limited-availability ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/); [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "yes", "api_embeddings": "not_documented", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "yes", "api_surface_note": "Bedrock's primary surface is proprietary (`InvokeModel`, `InvokeModelWithResponseStream`, `StartAsyncInvoke`, `InvokeModelWithBidirectionalStream`, `Converse`), reached through AWS SDKs — \"iOS and Android SDKs, as well as Java, JS, Python, CLI, .Net, Ruby, PHP, Go, and C++\", with \"streaming supported on all the SDKs\" ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html); [Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). An OpenAI-compatible base-URL swap is possible for the chat-completions path using a Bedrock API key ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). The `Converse` API is \"a unified API that abstracts FM differences and enables model switching with a single parameter change\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)).", "model_count_low": 100, "model_count_high": 100, "model_count_source_note": "\"100+ foundation models\" ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html), undated); separately, Bedrock Marketplace offers \"over 100 popular, emerging, or specialized models\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). The former per-model catalog page has been consolidated into a \"models at a glance\" page ([Supported foundation models](https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html), undated).", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party platform with partner catalog, so no routed-provider count: AWS states \"100+ foundation models from industry-leading providers\" and names Amazon, Anthropic, DeepSeek, Moonshot AI, MiniMax, OpenAI, xAI ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html), undated). The FAQ lists 13 providers — AI21 Labs, Amazon, Anthropic, Cohere, DeepSeek, Luma AI, Meta, Mistral AI, OpenAI, poolside \"(coming soon)\", Stability AI, TwelveLabs, Writer — without a total ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20).", "first_party_vs_partner": "Documented mix. Amazon Titan models are \"created by AWS,\" \"exclusive to Amazon Bedrock,\" and served through \"a fully managed API\"; third-party models are supplied by partner providers and some Bedrock models are sold by third-party providers as \"Third-Party Content\"; Bedrock Marketplace models are \"offered by various providers\" and deployed to fully managed SageMaker endpoints ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). Inference is supported \"in all Regions and with all models supported by Amazon Bedrock\" ([Supported Regions and models for inference](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-supported.html)).", "custom_endpoint_support": "Partly: customers can bring their own weights via Custom Model Import (Llama 2/3, Mistral, Flan architectures) and deploy Marketplace models to their own managed SageMaker endpoints; Guardrails can also be applied to \"self-hosted models outside Amazon Bedrock\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)). Registering an arbitrary external private URL as a Bedrock model: n.a.", "byok_mode": "not_applicable", "merchant_of_record": "AWS: usage is billed on the AWS account (\"You are charged…\") per the Bedrock pricing page ([Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/)), including per-use charges for Guardrails, while access through SageMaker Unified Studio \"comes at no extra cost,\" with users paying for \"the usage of the underlying resources\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). An explicit statement naming the invoicing entity for third-party models: n.a. (the FAQ only labels some models \"Third-Party Content\").", "key_management_note": "Authentication is AWS-native: Amazon Bedrock API keys authenticate with a bearer token \"instead of AWS credentials\", where short-term keys inherit permissions from the IAM principal that generated them and long-term keys create an IAM user with attached policies; SigV4 AWS credentials also work ([Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html); [Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). There are no upstream provider secrets to store. Requests are processed inside the customer's AWS account boundary and can be kept off the public internet with PrivateLink ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20); batch inputs/outputs live in the customer's own S3 buckets ([Batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)).", "api_openai_chat_completions_note": "Yes: `https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions` and `https://bedrock-mantle.{region}.api.aws/v1/chat/completions` ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). Native alternatives are `InvokeModel` and the unified `Converse` API ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20).", "api_anthropic_messages_note": "n.a. as a native Anthropic Messages endpoint; Anthropic models are reached through Bedrock's own `InvokeModel`/`Converse` APIs ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).", "api_openai_responses_note": "Yes: OpenAI GPT-5.6 Sol, Terra and Luna are available \"through the Responses API on Amazon Bedrock,\" and server-side Web Search is invoked through \"a single tool-use parameter in the Responses API\" ([Custom model import overview / Bedrock what's-new content](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html); [What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html)). Exact path: n.a.", "api_embeddings_note": "n.a. as a dedicated endpoint path; embeddings run as model invocations on the `bedrock-runtime` endpoint (`InvokeModel`) ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).", "api_images_note": "n.a. as a dedicated endpoint path; image/video generation providers (Stability AI, Luma AI, TwelveLabs) are invoked through the same runtime APIs, including `StartAsyncInvoke` for asynchronous generations ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).", "api_audio_note": "n.a. on the pages fetched (no STT/TTS endpoint documented in Bedrock's inference API pages), though `InvokeModelWithBidirectionalStream` exists for streaming multimodal interaction ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).", "api_batch_note": "Yes: \"With batch inference, you can submit multiple prompts and generate responses asynchronously,\" formatted with either `InvokeModel` or `Converse` request format, with input/output JSONL files in S3; it does not support tool calling or structured output (`response_format`), each record is processed independently without multi-turn, and \"batch inference isn't supported for provisioned models\" ([Batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)).", "deployment_modes_note": "hosted cloud service (fully managed, serverless) ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20); customer-VPC private connectivity via AWS PrivateLink — \"establish private connectivity between your FMs and your Amazon Virtual Private Cloud (Amazon VPC) without exposing your traffic to the Internet\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)); dedicated managed endpoints for Marketplace models, deployed \"onto fully managed endpoints,\" including \"a fully managed SageMaker endpoint\" with a chosen instance count/type ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)). Self-host, on-prem, air-gapped: n.a.", "byok_mode_note": "not_applicable: Bedrock is a first-party inference platform; there are no upstream provider keys to bring, and access is authenticated with AWS credentials (SigV4) or Amazon Bedrock API keys ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html); [Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html)).", "access_meta": { "qualified": [ "api_anthropic_messages", "api_embeddings", "api_images", "api_audio" ], "model_count_kind": "vendor_total", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "No timeout header, URI parameter or body field. Timeout surfaces only as an error: `ModelTimeoutException`, HTTP `408`, \"Processing time exceeded the model timeout length\" ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented` as a service-side setting. The API reference states that for `ModelNotReadyException` (HTTP `429`) \"The AWS SDK will automatically retry the operation up to 5 times\"; no retry key or backoff strategy is published, so default backoff is `n.a.` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "`not_documented` for model/provider fallback. The only documented fallback-like behavior is latency-mode degradation: when the latency-optimization quota is reached \"we will attempt to serve the request with Standard latency\", and Llama 3.1 405B requests above ~11K total tokens \"fall back to the standard mode\" — controlled by `performanceConfig.latency: standard | optimized`, not by a fallback list ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "No health check, circuit breaker or ejection control; only error types (`throttlingException`, `serviceUnavailableException`, `modelStreamErrorException`) ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "rel_multi_region_config": "not_configurable", "rel_multi_region_note": "`not_configurable` in the sense asked: AWS runs the regions and picks for you. Cross-Region inference uses inference profiles that \"define a foundation model (FM) and the AWS Regions to which requests can be routed\"; for geographic or global profiles \"Amazon Bedrock automatically selects a commercial AWS Region to process the request\", and manual Region enablement is not required. You choose a profile, not a failover policy ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html), [Bedrock inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html)).", "rel_load_balancing_config": "not_configurable", "rel_load_balancing_note": "An inference profile \"distributes model invocation requests across these Regions\", with no weights or policy keys exposed ([Bedrock inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html)).", "architecture_class": "hyperscaler_api", "architecture_note": "A managed AWS regional service invoked through AWS SDKs; no runtime disclosed ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "self_host_data_plane": "no", "self_host_artifact_note": "`no` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported via `InvokeModelWithResponseStream` and `ConverseStream`. Caveats: check `responseStreamingSupported` from `GetFoundationModel` before assuming a model streams; \"The AWS CLI doesn't support streaming operations in Amazon Bedrock, including `ConverseStream`\"; mid-stream failures arrive as `modelStreamErrorException` (HTTP `424`, \"A streaming error occurred. Retry your request.\"); requires `bedrock:InvokeModelWithResponseStream` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "guard_pii": "sync_block", "guard_pii_note": "Sensitive information filters detect PII entity types and custom regex entities, with per-entity behaviour of `BLOCK` or `MASK` on both input and output ([Sensitive information filters](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html), [Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))", "guard_injection": "sync_block", "guard_injection_note": "\"Prompt attack\" is one of the content-filter categories with configurable strength thresholds; the detection mechanism is an AWS-managed classifier and is not described in detail ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "guard_content": "sync_block", "guard_content_note": "Content filters cover hate, insults, sexual, violence, misconduct, and prompt attacks, with configurable thresholds applied to prompts and responses ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))", "guard_custom": "sync_block", "guard_custom_note": "Customer policy forms are denied topics (natural-language topic definitions), word filters including custom word/phrase lists (profanity, competitor names), custom regex entities, and Automated Reasoning checks for policy validation ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "guard_model_policy": "yes", "guard_model_policy_note": "IAM identity-based policies gate `bedrock:InvokeModel` on specific model resources, and IAM condition keys can additionally require that a guardrail be attached to the call ([IAM identity-based policy examples](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "No documented timeout/fail-open behaviour for the guardrail evaluation itself ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))", "log_content": "nothing", "log_content_note": "Model invocation logging is disabled until the customer enables it; once enabled it captures full input and output JSON (payloads above 100 KB are stored in S3) to CloudWatch Logs and/or S3, and can include guardrail trace detail ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "log_opt_out": "yes", "log_opt_out_note": "Leaving model invocation logging off keeps CloudWatch operational metrics without any prompt/completion storage; the feature is opt-in per account/region ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))", "tracing_support": "not_documented", "tracing_note": "`n.a.` from the fetched pages — guardrail traces are logged as part of invocation logs, but OpenTelemetry support is not documented on the pages fetched ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))", "export_destinations": [ "CloudWatch Logs", "S3", "CloudTrail" ], "export_destinations_note": "Amazon CloudWatch Logs and Amazon S3 for invocation logs ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html)); AWS CloudTrail records guardrail configuration and enforcement actions ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` on the fetched pages; the closest documented mechanism is `ApplyGuardrail`, which evaluates text without invoking a model and can therefore be run over stored traffic ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))", "observability_requires_saas": null, "log_retention_note": "Determined by the destination the customer chooses (CloudWatch Logs group retention or S3 lifecycle policy); no Bedrock-side default window is stated ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` for numeric figures. Latency-optimized inference is described only qualitatively (\"faster response times\", \"significantly reduced latency without compromising accuracy\") and per-request latency is reported back as `metadata.metrics.latencyMs` ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html), [Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).", "multi_surface": {}, "throughput_prose": "`n.a.` Throughput is a purchasing construct: \"To increase throughput for a model, you can purchase Provisioned Throughput\", and \"Inference profiles currently don't support Provisioned Throughput\" ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html)).", "benchmark_independence": "No figures published, vendor or otherwise ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Cross-Region inference adds no routing cost, is billed at the source Region's price, stays on the AWS network, is encrypted in transit, and is auditable via CloudTrail `additionalEventData.inferenceRegion` ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html)). Latency-optimized inference was in preview on the fetched page and is limited to specific models/regions ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html)).", "guard_notes": "Two features stand out for auditors: `ApplyGuardrail` lets the same policy be enforced for models running outside Bedrock, and IAM condition-based enforcement can make an un-guardrailed `InvokeModel` call impossible rather than merely discouraged ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/)).", "fit_agent_features_note": "Amazon Bedrock Mantle supports stateful conversation management, `previous_response_id`, streaming, multi-turn interactions, client-side tool use, background processing, and asynchronous inference on `bedrock-mantle`; server-side tools are unavailable on `bedrock-runtime`. AgentCore additionally documents stateful MCP sessions, elicitation, sampling, progress notifications, managed memory and session state, and multi-step agent workflows. ([Amazon Bedrock Mantle agent capabilities](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))", "fit_byok_required": "no", "fit_byok_required_note": "No. Authentication uses a Bedrock short-term API key generated in the console ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))", "fit_client_sdk_langs": [ "C++", "Go", "Java", "JavaScript", "Kotlin", ".NET", "PHP", "PowerShell", "Python", "Ruby", "Rust", "SAP ABAP", "Swift" ], "fit_client_sdks_note": "The OpenAI Python SDK is documented with `from openai import OpenAI`; the examples require Python 3.7 or later and use `pip install openai boto3`. AWS SDKs are available for C++, Go, Java, JavaScript, Kotlin, .NET, PHP, PowerShell, Python/Boto3, Ruby, Rust, SAP ABAP, and Swift; the JavaScript v3 Bedrock Runtime package is `@aws-sdk/client-bedrock-runtime`, with classes including `BedrockRuntimeClient`, `ConverseCommand`, `ConverseStreamCommand`, and `InvokeModelCommand`. ([Amazon Bedrock SDK general information](https://docs.aws.amazon.com/bedrock/latest/userguide/sdk-general-information-section.html))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Amazon Bedrock supports AWS credentials for HTTP requests, while OpenAI SDK usage requires an Amazon Bedrock API key. For AgentCore Gateway, IAM SigV4 is the default inbound authorization mechanism and IAM-based authentication is supported. ([Amazon Bedrock authentication and endpoints](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))", "fit_cloudflare_workers": "third_party_documented", "fit_cloudflare_workers_note": "Documented by Cloudflare: Bedrock can be called from a Cloudflare Worker through Cloudflare AI Gateway using the `aws4fetch` library for client-side AWS SigV4 signing. ([Cloudflare AI Gateway - Amazon Bedrock provider](https://developers.cloudflare.com/ai-gateway/usage/providers/bedrock/))", "fit_credit_card": "required_for_model_access", "fit_credit_card_note": "Not stated in the quickstart; the pricing page says \"Get started for free.\" However model access requires \"Valid AWS Marketplace payment method.\" ([Amazon Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))", "fit_ecosystem_notes": "Amazon Bedrock provides managed, serverless foundation models through unified `bedrock-runtime` and `bedrock-mantle` endpoints; `bedrock-runtime` is recommended for new applications, while Projects are created only on `bedrock-mantle`. The OpenAI-compatible endpoints are powered by Mantle, have endpoint-specific quotas, and are available in the documented AWS Regions; Amazon Bedrock Knowledge Bases can use Amazon S3 and Amazon OpenSearch Serverless. ([Amazon Bedrock Mantle overview](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))", "fit_eval_prereqs": "An AWS account (step 1) and Python already installed; an SDK for the chosen API; environment variables for the API key. Marketplace-related IAM permissions matter for model access ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "Partially: the page instructs you to save Python code as `bedrock-first-request.py` and run `python3 bedrock-first-request.py`, and recommends the `bedrock-runtime` endpoint, but no request snippet was rendered in the fetched content ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))", "fit_gateway_platform": "platform_gateway", "fit_gateway_platform_note": "Amazon Bedrock AgentCore Gateway is the documented gateway platform. The evidence does not document Kong, APISIX, Envoy, Istio, or nginx gateway plugins for Amazon Bedrock. ([AgentCore Gateway MCP server](https://docs.aws.amazon.com/prescriptive-guidance/latest/semantic-layer-agentic-ai-ontology-reasoning-virtual-knowledge-graph/mcp-server.html))", "fit_helm_chart": "EKS Helm documentation only; no Bedrock chart or operator", "fit_kubernetes": "generic_platform_only", "fit_kubernetes_note": "Amazon EKS documentation describes Helm for installing and managing charts, including charts from the `eks-charts` GitHub repository or ArtifactHub. No official Amazon Bedrock gateway Helm chart or operator is documented. ([Helm on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/helm.html))", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Amazon Bedrock AgentCore supports MCP Streamable HTTP transport, normally with `stateless_http=True`; the `/mcp` endpoint supports MCP RPC messages, tool invocation and management, capability discovery, resource access, and multi-step workflows. Stateful sessions use the `Mcp-Session-Id` header, and the required container port is `8000`. ([AgentCore Runtime MCP protocol contract](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-mcp-protocol-contract.html))", "fit_meta": { "not_documented_fields": [] }, "fit_model_gate": "approval_required", "fit_model_gate_note": "Yes, partly automated: \"Access to all Amazon Bedrock foundation models is enabled by default with the correct AWS Marketplace permissions\" and \"When you invoke a third-party model for the first time in your account, Amazon Bedrock automatically initiates the subscription process in the background\"; Anthropic models require a First Time Use (FTU) form (companyName, companyWebsite, intendedUsers, industryOption, useCases) after which \"Access to the model is granted immediately after use case details are successfully submitted\"; subscription setup \"may take up to 15 minutes.\" GovCloud console access is a 10-step procedure ending \"the Access status changes to Access granted or Available to request,\" and a 5-step programmatic procedure exists ([Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html)). Non-GA models are gated: \"Access to Claude Mythos 5 and Claude Mythos Preview is gated and requires approval. Contact your Anthropic account team to request access on Bedrock.\" ([Amazon Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/))", "fit_onboarding_notes": "Multiple API surfaces are available (Anthropic-native Messages, OpenAI-compatible Responses, Chat Completions, Converse, Invoke), so SDK choice depends on the API. After permissions are granted, completion \"may take up to 2 minutes.\" Availability responses expose `agreementAvailability: AVAILABLE`, `authorizationStatus: AUTHORIZED`, `entitlementAvailability: AVAILABLE`, `regionAvailability: AVAILABLE` ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Existing OpenAI SDKs can be used by changing the base URL and API key. Documented base URLs are `https://bedrock-runtime..amazonaws.com/openai/v1` (recommended for new applications) and `https://bedrock-mantle..api.aws/v1`; the OpenAI SDK requires an Amazon Bedrock API key. ([Amazon Bedrock OpenAI-compatible APIs](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))", "fit_prod_prereqs": "IAM roles / temporary credentials instead of the short-term API key; for managing model access via SDK/CLI: attach `AmazonBedrockFullAccess`, configure the AWS SDK, AWS CLI version 2.27.42 or later, and know the model ID; Marketplace permissions `aws-marketplace:Subscribe`, `Unsubscribe`, `ViewSubscriptions`; a valid AWS Marketplace payment method; GovCloud third-party models require access enabled in both the linked commercial account and the GovCloud account ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [ "LlamaIndex", "Strands" ], "fit_python_frameworks_note": "LlamaIndex integrations are documented for Amazon Bedrock, including `Bedrock`, `BedrockEmbedding`, `AmazonKnowledgeBasesRetriever`, `RouterQueryEngine`, `SubQuestionQueryEngine`, and `ReActAgent`. AWS Strands is also explicitly documented as a supported agent framework for AgentCore Runtime. ([LlamaIndex and Amazon Bedrock RAG](https://aws.amazon.com/blogs/machine-learning/build-powerful-rag-pipelines-with-llamaindex-and-amazon-bedrock/))", "fit_quickstart_steps": 5, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "platform_provider_resources", "fit_terraform_note": "The HashiCorp AWS provider documents the Terraform resource `aws_bedrockagentcore_registry`. The resource is deprecated; the evidence states that functionality will move from the `bedrock-agentcore` namespace to `agent-registry` on August 6, 2026, while the resource continues to work until September 17, 2026. ([Terraform AWS AgentCore Registry resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/bedrockagentcore_registry))", "fit_terraform_ref": "hashicorp/aws resource aws_bedrockagentcore_registry — deprecated, moving to agent-registry on 2026-08-06 and working until 2026-09-17", "fit_third_party_documented": { "cloudflare_workers": "Cloudflare" }, "fit_vendor_time_claim": "Vendor claim (page description), verbatim: \"Get started with Amazon Bedrock in minutes.\" ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Per an AWS Vercel case study, Vercel AI SDK integrates with Amazon Bedrock through the dedicated provider package `@ai-sdk/amazon-bedrock`. ([AWS Vercel case study](https://aws.amazon.com/solutions/case-studies/vercel-case-study/))", "fit_vercel_ai_sdk_package": "@ai-sdk/amazon-bedrock" }, { "slug": "azure-ai-foundry", "name": "Azure AI Foundry", "company": "Microsoft", "tagline": "Microsoft's Azure platform for deploying models from its own and partner catalogs, now branded Microsoft Foundry.", "category": "cloud_platform", "deployment": "both", "license": "Proprietary", "founded_year": null, "hq_country": "United States", "url_home": "https://azure.microsoft.com/en-us/products/ai-foundry", "url_docs": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "url_pricing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "url_status": null, "url_github": null, "url_changelog": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry", "url_models_api": null, "github_stars": null, "model_count": 11000, "model_count_note": "Vendor states \"11K+ Foundry Models\" / \"over 11,000+ models\", spanning Azure OpenAI, Microsoft, and partner catalogs.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": 0, "free_tier": "Azure free account gives $200 credit valid 30 days; the Foundry platform itself is free to use and you pay only for the models and compute you deploy.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Region-pinned Managed Compute GPU deployments", "amount": "Billed per GPU-hour (A100, H100, H200, MI300 SKUs)" }, { "label": "Provisioned throughput (PTU)", "amount": "Reserved capacity billed per PTU rather than per token" } ], "self_host_cost_note": "Mostly managed, with Foundry Local for on-device execution. Microsoft sets its own per-token prices rather than marking up a third party; Global Batch and Data Zone Batch deployments are 50% below the standard rate with a 24-hour completion target, and DeveloperTier deployments are cheap but capped at 24 hours with no SLA and no data-residency guarantee.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "Azure OpenAI (AzureOpenAI client, /openai/deployments/{deployment})", "Azure AI Model Inference", "Foundry Agent Service" ], "sdk_note": "Azure clients are built on the official OpenAI packages via the AzureOpenAI class; endpoints take the form https://RESOURCE-NAME.openai.azure.com with a deployment name in the path rather than a model name.", "cap_failover": null, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": true, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": null, "cap_video": null, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": null, "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": null, "eu_residency": true, "eu_residency_note": "DataZoneStandard, DataZoneProvisionedManaged and DataZoneBatch deployments keep processing inside the EU data zone, which follows the Azure EU Data Boundary and may include EFTA countries (Norway, Switzerland); single-region Standard deployments process in the customer-specified geography, while Global deployments may process anywhere. Abuse-monitoring data for EEA deployments is reviewed only by Microsoft employees located in the EEA.", "zdr": true, "zdr_note": "Managed customers may apply for modified abuse monitoring; once approved, the storage and human-review step is not performed. It is an application, not a switch.", "air_gapped": null, "best_for": "Microsoft-centric enterprises that want first-party OpenAI models plus a very large partner catalog under Azure governance.", "strengths": [ "Largest published catalog of the three hyperscalers at 11,000+ models, with a built-in Model Router", "Fine-grained residency control through Global, Data Zone, single-region and Azure Government deployment types", "Modified abuse monitoring gives approved customers an effective zero-retention path", "Azure OpenAI clients are thin wrappers on the official OpenAI SDKs, so migration is mostly a base-URL change", "Platform surface is free; you pay only for deployed models and compute" ], "weaknesses": [ "No cross-cloud routing: you can only reach models Microsoft hosts, so multi-cloud failover needs a separate gateway", "Heavy Azure coupling — resources, quotas, deployments and PTU commitments are all Azure-specific", "Constant renaming (Azure AI Studio to Azure AI Foundry to Microsoft Foundry) makes docs and portal naming inconsistent", "Compliance attestations are documented at the Azure service level, not specifically under the Foundry brand, so scope needs verification per service" ], "avoid_if": "You need vendor-neutral routing across clouds, or you want a single flat token price without capacity planning around PTUs and deployment types.", "reviews": [ { "url": "https://blog.chiffers.com/new-vs-old-azure-ai-foundry-how-it-affects-your-code/", "why": "Practitioner maps the breaking changes between old and new Foundry projects at the code level (azure-ai-projects 2.x, AIProjectClient, Responses vs Assistants API).", "date": "2026-06-09", "type": "written", "title": "New vs Old Azure AI Foundry: How it affects your code", "publisher": "blog.chiffers.com" }, { "url": "https://www.infoq.com/news/2025/05/azure-ai-foundry-agents-ga/", "why": "InfoQ's Steef-Jan Wiggers details what GA changed in Foundry's agent service - connected agents, SDK surface, A2A support - rather than restating Microsoft's messaging.", "date": "2025-05-26", "type": "written", "title": "Azure AI Foundry Agent Service GA Introduces Multi-Agent Workflows", "publisher": "InfoQ" }, { "url": "https://www.youtube.com/watch?v=Sq8Cq7RZM2o", "why": "Independent Azure trainer walks the Foundry portal, model catalog, playground, evaluations and agents, flagging where the abstraction leaks.", "date": "2025-03-05", "type": "video", "title": "Azure AI Foundry Overview", "publisher": "YouTube - John Savill's Technical Training" }, { "url": "https://www.reddit.com/r/AZURE/comments/1vea8u8/azure_ai_foundry_gpt4o_to_gpt51_migration_changed/", "why": "13-comment migration thread with concrete constraints: datazone-standard deployments lacking GPT-5.1, and quota/capacity negotiations through account managers.", "date": "2026-08-03", "type": "community", "title": "Azure AI Foundry: GPT-4o to GPT-5.1 migration changed our RAG agent's response style", "publisher": "Reddit - r/AZURE" }, { "url": "https://www.reddit.com/r/ArtificialInteligence/comments/1nhic70/azure_ai_foundry_is_awful/", "why": "12-comment complaint thread with specifics: an agent silently losing its AI Search tool connection and 800k-token context configuration problems.", "date": "2025-09-15", "type": "community", "title": "Azure AI Foundry is awful", "publisher": "Reddit - r/ArtificialInteligence" } ], "sources": { "zdr": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy", "name": "https://azure.microsoft.com/en-us/products/ai-foundry", "soc2": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-soc-2", "company": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "tagline": "https://azure.microsoft.com/en-us/products/ai-foundry", "sdk_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages", "url_docs": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "url_home": "https://azure.microsoft.com/en-us/products/ai-foundry", "zdr_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy", "cap_batch": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types", "cap_evals": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "free_tier": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "hipaa_baa": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-hipaa-us", "deployment": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "extra_fees": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "hq_country": "https://en.wikipedia.org/wiki/Microsoft", "zdr_status": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy", "model_count": "https://azure.microsoft.com/en-us/products/ai-foundry", "url_pricing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "api_surfaces": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages", "eu_residency": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types", "product_fate": "https://learn.microsoft.com/en-us/azure/foundry/how-to/high-availability-resiliency", "regions_note": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy", "cap_image_gen": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "url_changelog": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry", "vendor_status": "https://learn.microsoft.com/en-us/azure/foundry/whats-new-foundry", "cap_embeddings": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/azure-government", "cap_guardrails": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy", "certifications": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-fedramp", "config_as_code": "https://learn.microsoft.com/en-us/azure/foundry/how-to/create-resource-terraform", "retention_days": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy", "sla_uptime_pct": "https://azure.microsoft.com/en-us/products/ai-foundry/models/openai", "logs_by_default": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/abuse-monitoring", "model_count_note": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "cap_observability": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry", "cap_pii_redaction": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry", "eu_residency_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy", "last_release_date": "https://learn.microsoft.com/en-us/azure/foundry/whats-new-foundry", "openai_compatible": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages", "subprocessors_url": "https://www.microsoft.com/en-us/trust-center/privacy/data-access", "cap_load_balancing": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types", "seat_fee_usd_month": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry", "guardrails_location": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/content-safety/data-privacy", "requires_vendor_sdk": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints", "self_host_cost_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types", "azure_prompt_caching": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/prompt-caching", "azure_foundry_pricing": "https://azure.microsoft.com/en-us/pricing/details/microsoft-foundry/", "data_export_available": "https://learn.microsoft.com/en-us/azure/foundry/how-to/diagnostic-logging", "prompt_transits_vendor": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy", "cap_conditional_routing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/", "trains_on_customer_data": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy", "azure_foundry_models_pricing": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/" }, "verified_at": { "zdr": "2026-08-29", "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-08-29", "cap_batch": "2026-08-29", "cap_evals": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "eu_residency": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "model_count_note": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_load_balancing": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-08-31T04:37:34.758692+00:00", "vendor_status": "hyperscaler", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "A first-party Microsoft Azure product line with monthly feature roll-ups. Worth noting that the SLA does not cover everything: Microsoft states the Agent Service carries no availability or state-durability SLA.", "last_release_date": "2026-08-13", "sla_uptime_pct": 99.9, "sla_url": "https://azure.microsoft.com/en-us/products/ai-foundry/models/openai", "claim_basis": {}, "last_release_note": "Microsoft publishes no per-release dates, so this is when the “What's new in Microsoft Foundry” page was last updated.", "prompt_transits_vendor": "yes", "prompt_path_note": "Microsoft hosts the models it sells inside its own Azure environment, and those models do not interact with services operated by the model's original provider.", "logs_by_default": "none", "logs_note": "Models are stateless and prompts are not stored by the system for automated review. Abuse monitoring is the exception: flagged content goes to a store logically separated per customer resource, reachable only by authorised Microsoft staff through secure workstations with just-in-time approval.", "retention_days": null, "retention_note": "No day count is published on the current page. It states where flagged content is stored and who may see it, but not for how long — do not assume the 30-day figure that appeared in older versions.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Prompts, completions, embeddings and training data are not used to train generative foundation models, or to improve Microsoft or third-party products, without your explicit permission or instruction.", "subprocessors_url": "https://www.microsoft.com/en-us/trust-center/privacy/data-access", "regions_note": "Processing stays in your specified geography — with a significant exception. Any deployment type labelled Global may process prompts and responses in any geography, and DataZone widens the boundary too.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "the Azure SOC offering page did not load during research", "status": "not_published" }, "fedramp": { "detail": "High P-ATO issued by the JAB for Azure and Azure Government", "status": "certified" }, "gdpr_dpa": { "detail": "Microsoft Data Protection Addendum", "status": "certified" }, "iso27001": { "detail": "audited certification", "status": "certified" }, "hipaa_baa": { "detail": "the HIPAA page carried no BAA statement", "status": "not_published" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Content Safety models run inside Microsoft's Azure infrastructure. Data stays in Azure and in the region you chose, and is not stored or human-reviewed — but the filtering happens on Microsoft's side.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference with an explicitly free platform layer. 'The Foundry platform is free to use and explore'; individual features billed at their normal rates. The gateway/routing layer is $0; models are billed separately.", "min_commitment": "None required. Microsoft Agent pre-purchase plan is a 1-year metered plan buying Agent Commit Units at tiered discounts — 20,000 ACUs = 5%, 100,000 = 10%, 500,000 = 15% (dollar prices shown as '$-').", "overage_note": "None stated on either pricing page.", "cache_support": "prefix", "cache_read_discount_pct": null, "cache_write_premium_pct": 0, "cache_pricing_note": "Exact-prefix prompt caching, enabled by default, no opt-out. The first 1,024 tokens must be identical; a single character difference in the first 1,024 tokens = miss. Hits extend every 128 identical tokens. Caches clear within 5–10 minutes of inactivity and always within an hour, and are not shared across subscriptions. Discount: Standard cached-token discount is not published as a number; Provisioned deployments get up to 100% discount on input tokens. No cache-write charge documented. On Provisioned capacity, cached input can be effectively free.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null }, "rate_limiting": { "note": "Not stated as a cost control.", "tier": null, "supported": null }, "spend_webhooks": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated on the pricing pages fetched.", "tier": null, "supported": null } }, "cost_attribution": "Not stated on the pricing pages.", "cost_export": "Not stated on the pricing pages.", "product_kind_note": "Microsoft calls it \"Microsoft Foundry Models\", \"your one-stop destination for discovering, evaluating, and deploying powerful AI models\" and \"the hub for discovering and using a wide range of models to build generative AI applications\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). It is a cloud model platform with a catalog split between models Azure sells and hosts and partner/community models; a `model-router` model exists inside it but the product is not a third-party routing proxy ([Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26).", "deployment_modes": [ "saas" ], "deployment_note": "Serverless deployment types are Global/Data Zone/Regional Standard, Provisioned and Batch variants plus Developer ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Gating is by subscription, quota and region rather than SKU tier: pay-per-token requires the billing account's country to be one where the provider makes the offer available and a project resource in the model's region; some models need quota requests (Tier 5/6 have quota by default) or registration (`computer-use-preview`, `grok-code-fast-1`, `grok-4`), and partner/community models \"require Azure Marketplace\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview); [Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)). Managed compute deploys to \"Azure Machine Learning managed compute for real-time inference\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Two coexisting surfaces: a single Azure-schema models inference endpoint (\"usually with the form `https://.services.ai.azure.com/models`\") that \"allows customers to use a single endpoint with the same authentication and schema\" across deployed models, and the Azure OpenAI API which \"exposes the full capabilities of OpenAI models\" while \"non-OpenAI models can also be used for compatible functionalities\" ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26). It is not a plain OpenAI base-URL swap: routing is by deployment name (case-insensitive) matched to the request's `model`/`name` field, each Azure OpenAI deployment has its own URL (`/deployments/`), and requests carry an `api-version` query parameter ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference)). SDKs: `azure-ai-inference` (Python/JS/C#/Java) for the models endpoint and `openai` / `Azure.AI.OpenAI` for the Azure OpenAI surface ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference)).", "model_count_low": 10000, "model_count_high": 10000, "model_count_source_note": "\"over 10,000 models\" in the catalog, with roughly \"50 new models\" published per month ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28); the partner page describes itself as listing only \"a selection of Foundry Models from partners and community\" ([Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party platform plus partner marketplace, so no routed-provider total is published; named providers include Azure OpenAI, Microsoft, Anthropic, Mistral, Meta, Cohere, NVIDIA, DeepSeek and Hugging Face ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).", "first_party_vs_partner": "Explicit two-way split. \"Foundry Models sold by Azure\" are hosted and sold by Microsoft under Microsoft Product Terms and billed as \"First Party Consumption Services\" via Azure meters — \"Models sold by Azure are also hosted by Azure and operated by Azure as part of the Foundry Models service\", covering all Azure OpenAI models plus selected models from top providers. \"Foundry Models from partners and community\" come from third-party organizations, partners, research labs and community contributors (including Anthropic's Claude family and Hugging Face open models), \"require Azure Marketplace\", and their providers \"define the license terms and set the price for use of their models using Azure Marketplace\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).", "custom_endpoint_support": "Partly: customers can deploy model weights themselves onto managed compute VMs inside their Foundry resource, which then exposes a REST inference API ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Registering an external customer-hosted endpoint (vLLM, Ollama, private URL) with the Foundry inference endpoint: n.a. on the pages fetched.", "byok_mode": "not_applicable", "merchant_of_record": "Microsoft/Azure for models sold by Azure, billed as \"First Party Consumption Services\" via Azure meters; for partner and community models the provider \"set[s] the price for use of their models using Azure Marketplace\", so charges flow through Azure Marketplace (and Student, Visual Studio Enterprise and Free-credit subscriptions cannot purchase Marketplace SaaS offers) ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)). Serverless usage is billed on inputs/outputs (typically tokens) and managed compute on VM core hours ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)).", "key_management_note": "Two authentication methods: API keys sent in the `api-key` HTTP header (\"all API requests must include the API Key in the `api-key` HTTP header\") and Microsoft Entra ID bearer tokens in the `Authorization` header ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24); the models inference SDKs use `AzureKeyCredential` / `AZURE_INFERENCE_CREDENTIAL` ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26). Keys are scoped to the Foundry/Azure OpenAI resource, and there are no upstream third-party secrets to vault since Microsoft hosts the models ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Request traffic can be confined to a private endpoint with public network access disabled on the hub (propagation up to five minutes), with the caveat that Azure OpenAI On Your Data is unavailable for serverless deployments in private hubs ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)). Customer-managed KMS specifics: n.a.", "api_openai_chat_completions_note": "Yes, two paths: the unified models inference endpoint `POST https://.services.ai.azure.com/models/chat/completions?api-version=...` and the Azure OpenAI form `POST https://.services.ai.azure.com/openai/deployments//chat/completions?api-version=2024-10-21` ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).", "api_anthropic_messages_note": "n.a. Claude models are offered as partner models but no Anthropic Messages endpoint is documented on the pages fetched ([Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners); [How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).", "api_openai_responses_note": "n.a.: the `/v1/responses` path is not mentioned on the pages fetched, though the Azure OpenAI v1 APIs are referenced generally ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), last updated 2026-06-24; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).", "api_embeddings_note": "Yes: the models inference endpoint \"supports the following modalities: Text embeddings… Image embeddings… Chat completions\" ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26); embeddings are also used for RAG over custom data ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).", "api_images_note": "Yes: `.../openai/deployments/{deployment-id}/images/generations` and `.../images/edits` (documented for FLUX models), plus DALL·E deployments ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24).", "api_audio_note": "Yes, STT documented: audio transcription (\"Transcribes audio into the input language\") and translation (\"Transcribes and translates input audio into English text\") as `multipart/form-data` requests with `file`, `prompt`, `response_format`, `temperature`, `language`, returning `json`, `text`, `srt`, `verbose_json` or `vtt` ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24). TTS specifically: n.a. on the pages fetched (audio generation appears only as a catalog filter) ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).", "api_batch_note": "Yes at deployment level: Global Batch and Data Zone Batch deployment types exist and the Azure OpenAI API \"supports additional features like assistants, threads, files, and batch inference\"; no separate batch endpoint path is documented ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).", "deployment_modes_note": "hosted cloud service in three shapes: serverless deployments where \"Microsoft hosts the models in a managed Azure infrastructure\"; managed compute, where weights are deployed to \"dedicated virtual machines with managed compute\" billed by \"virtual machine core hours\"; and \"instant access (preview)\" with \"no deployment required\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Private networking is supported (disable public network access on the hub and use \"a private endpoint for the hub\"), but the page \"does not describe this as hybrid, VPC, on-premises, or air-gapped deployment\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)). Local execution exists for at least one model via Foundry Local (`gpt-oss-20b` \"available via managed compute and Foundry Local\") ([Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26).", "byok_mode_note": "not_applicable: Foundry is the model host, authenticated with Azure resource credentials rather than upstream provider keys ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26; [Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24).", "access_meta": { "qualified": [ "api_openai_chat_completions", "api_anthropic_messages", "api_audio", "api_batch" ], "model_count_kind": "vendor_total", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` on the fetched Foundry pages ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented` as a platform setting; default count and backoff: `n.a.` Microsoft's own gateway guidance tells you to build it yourself and to honor the `Retry-After` header from Azure OpenAI responses ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).", "rel_retry_default_note": null, "rel_fallback_config": "per_request", "rel_fallback_note": "`per_request` and `dashboard_only`, as a SINGLE-ALTERNATIVE spillover: deployment property `spilloverDeploymentName`, or per-request header `x-ms-spillover-deployment`, targeting one corresponding standard deployment in the same Azure OpenAI resource. If both are set, the deployment property wins. Triggers: `429` (PTUs exhausted), `400` (long-context request), `500`, `503`. If the spillover request also fails, the original PTU response is returned and `x-ms-spillover-error` carries the spillover status code. Not available for the Responses API ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management), [Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)).", "rel_fallback_shape": "single", "rel_health_check_config": "not_configurable", "rel_health_check_note": "`not_configurable` / not provided by the platform. Microsoft states plainly that Azure OpenAI \"doesn't provide any sort of health check endpoint to preemptively know whether it's available to handle requests\", and recommends that your gateway circuit-break on `429`/`500`/`503`, honor `Retry-After`, and remove faulted backends from the pool — with Azure API Management offering \"built-in circuit breaker functionality\" ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).", "rel_multi_region_config": "not_configurable", "rel_multi_region_note": "`not_configurable` at the deployment level: the vendor runs the regions. Global Provisioned traffic is \"Routed across Azure regions globally\", Data Zone Provisioned \"Stays within a geographic zone (US or EU)\", Regional Provisioned \"Stays in the deployment's specific Azure region\" ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)). Spillover itself cannot cross resources — both deployments must be in the same Azure OpenAI resource with matching data-processing level ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)). Cross-region active-passive failover is described only as a gateway pattern you implement ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).", "rel_load_balancing_config": "not_configurable", "rel_load_balancing_note": "`not_configurable` natively; no weights. Global Provisioned routes across regions automatically with no exposed policy ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)); spillover only prioritizes the provisioned deployment before overflowing to standard ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)). Multi-backend round-robin is a gateway pattern you build, and Microsoft warns endpoints in a pool must serve \"the same model at the same version\" ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).", "architecture_class": "hyperscaler_api", "architecture_note": "Managed Azure service; deployment types are selected via `sku-name` (`GlobalProvisionedManaged`, `ProvisionedManaged`), and no runtime is disclosed ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)).", "self_host_data_plane": "no", "self_host_artifact_note": "`no` for the Foundry inference plane. Managed compute and serverless API endpoints are documented as Azure-hosted deployment options ([Azure AI Foundry deployments overview](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/deployments-overview)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported via `stream: true`; tokens return \"as soon as they're available\", which \"doesn't change the time to get all the tokens\" but \"reduces the time for first response\". Documented caveat: \"Many clients and intermediary layers have timeouts on individual calls\" and \"Long generation calls might be canceled due to client-side time outs\" — streaming is recommended partly to keep those connections alive ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)). Model router forwards a standard chat-completions response but drops several parameters for o-series reasoning models (`Temperature`, `Top_P`, `stop`, `presence_penalty`, `frequency_penalty`, `logit_bias`, `logprobs`) and does not support `reasoning_effort` ([Azure model router](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/model-router)).", "guard_pii": "sync_observe", "guard_pii_note": "The content-filtering system includes a PII filter applied to model output; the docs describe detection and annotation and do not state that Foundry redacts the text ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))", "guard_injection": "sync_block", "guard_injection_note": "Prompt Shields \"detects and blocks\" user prompt attacks and indirect (cross-domain) prompt injections; optional binary classifier models flag jailbreak risk, and indirect-attack detection requires document delimiters in the prompt ([Prompt Shields / jailbreak detection](https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/jailbreak-detection), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))", "guard_content": "sync_block", "guard_content_note": "Azure AI Content Safety runs an ensemble of classification models over prompt and completion \"synchronously as the service processes prompts to generate content\", filtering hate, sexual, violence, and self-harm at configured severity thresholds; audio models such as Whisper are excluded ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))", "guard_custom": "sync_block", "guard_custom_note": "Customer-defined policy takes the form of configurable severity thresholds per category and custom blocklists; separate protected-material and code classifiers can be enabled ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))", "guard_model_policy": "yes", "guard_model_policy_note": "`yes`, structurally — models are consumed as named deployments on a Foundry resource, and access is governed by Azure RBAC on those resources with diagnostic/policy control via Azure Policy; an explicit \"model allowlist\" UI is not documented on the fetched pages ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` — no fail-open/fail-closed statement for the filtering service ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering))", "log_content": "metadata_only", "log_content_note": "Platform metrics and activity logs are collected automatically, but resource logs \"aren't collected and stored until you create a diagnostic setting\"; prompts and completions are not stored by the content classifiers (\"No prompts or generated content are stored in the content classifier models\"). Abuse monitoring may retain a sample of prompts and completions for human review in a per-resource, per-geography store unless modified abuse monitoring is approved ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy))", "log_opt_out": "yes", "log_opt_out_note": "Do not create (or scope down) diagnostic settings and platform metrics still flow; abuse-monitoring storage and human review can be removed via approved modified abuse monitoring ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy))", "tracing_support": "otel", "tracing_note": "OpenTelemetry natively, using the OTel GenAI semantic conventions, with multi-agent spans surfaced in Azure Monitor Application Insights ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))", "export_destinations": [ "Azure Monitor", "Log Analytics", "Event Hubs", "Azure Storage" ], "export_destinations_note": "Via diagnostic settings: Azure Monitor Logs / Log Analytics, Azure Storage, Azure Event Hubs (the route to non-Azure partner systems), and certain Microsoft monitoring partners; all metrics are exportable with diagnostic settings, plus Log Analytics workspace data export ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai)); Application Insights for traces ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — no feedback/score endpoint documented on the fetched pages ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` from the fetched pages ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))", "observability_requires_saas": null, "log_retention_note": "Determined by the destination (Log Analytics workspace, Storage, Event Hubs) rather than by Foundry; no service-side default stated ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` for numeric figures. Azure publishes latency definitions and SLA posture instead: **End-to-end Request Time** (non-streaming, measured by the API gateway), **Time to Response** (first-token, PTU/PTU-managed), **Average Token Generation Rate** (first-to-last token divided by generated tokens); Standard and Batch have \"None\" for a latency SLA while Provisioned and Priority processing have a \"Defined latency target per model\" ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency), [Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)). Spillover \"might incur additional latency\" ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)).", "multi_surface": { "rel_fallback": [ "per_request", "dashboard_only" ] }, "throughput_prose": "Capacity-planning tables rather than benchmarks. GPT-4o mini worked examples: 800/150 tokens at 30 RPM = 28,500 total TPM = 15 PTUs; 5,000/50 at 1,000 RPM = 5,050,000 TPM = 140 PTUs; 1,000/300 at 500 RPM = 650,000 TPM = 30 PTUs. \"The number of PTUs scales roughly linearly with call rate when the workload distribution remains constant\", and estimates exclude prompt-caching benefits ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)).", "benchmark_independence": "All figures are vendor (Microsoft) capacity-planning examples, not measured benchmarks and not third-party ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency))." }, "guard_pii_default": "sync_observe", "guard_injection_default": "sync_block", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Foundry's built-in reliability is deployment-type selection plus single-target spillover; anything resembling a gateway (retry policy, circuit breaking, multi-backend balancing, active-passive regional failover) is documented as something you build in front of it, typically with API Management ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).", "guard_notes": "Anything typed into the Completions or Chat playground \"generates metrics and log data for your Azure OpenAI resource\", which surprises teams who assume the playground is ephemeral ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai)). Human reviewers only reach flagged data, via point-wise request-ID queries from Secure Access Workstations under JIT approval ([Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy)).", "fit_agent_features_note": "The documented agent features include stateful multi-turn sessions with `previous_response_id`, manual chaining, parallel multi-agent orchestration, function/tool calling, remote MCP tools, Code Interpreter, image generation, background tasks, streaming, response compaction, hosted agents with session-level state, and Foundry Agent Service threads, messages, and runs. A2A is not documented in the supplied evidence. ([Microsoft Foundry Responses API](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses))", "fit_byok_required": "no", "fit_byok_required_note": "No. Authentication uses `DefaultAzureCredential` ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))", "fit_client_sdk_langs": [ "Python", "JavaScript", ".NET", "Java" ], "fit_client_sdks_note": "Official client libraries include Python `openai`, `azure-ai-projects`, and `azure-identity`; JavaScript `openai`, `@azure/ai-projects`, and `@azure/identity`; .NET `OpenAI`, `Azure.AI.Projects`, and `Azure.Identity`; and Java OpenAI and Azure Identity packages including `com.openai:openai-java` and `com.azure:azure-identity`. The Foundry SDK documents classes such as `AIProjectClient`, `ProjectsClient`, `ProjectsClientBuilder`, and `ChatCompletionsClient`; Foundry Local additionally documents C#, JavaScript, Python, and Rust SDKs. ([Microsoft Foundry SDK overview](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/sdk-overview))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Microsoft Entra ID authentication is supported through `DefaultAzureCredential`, bearer tokens, managed identities, and Azure RBAC/resource role assignments; the documented token scope is `https://ai.azure.com/.default`. Foundry Local also documents Entra authentication with an app registration using `entraAuth.tenantId` and `entraAuth.clientId`. ([Microsoft Foundry model endpoints](https://learn.microsoft.com/en-us/azure/foundry-models/concepts/endpoints))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated; only \"Create one for free.\" / \"create a free account if needed\" ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota), [Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))", "fit_ecosystem_notes": "Microsoft Foundry generally requires an Azure subscription, a Foundry resource or project, and deployed models or other Azure resources; deployments are Azure resources subject to Azure policies. Foundry projects expose `https://.services.ai.azure.com/api/projects/` and an OpenAI-compatible `/openai/v1` endpoint, while hosted agents run on Foundry-managed isolated Micro VMs with automatic scaling, a dedicated Microsoft Entra agent identity, session-level state, and observability. Foundry Local is a separate self-contained SDK deployment for Windows, macOS, and Linux that does not require the Foundry Local CLI on end-user machines. ([Microsoft Foundry SDK overview](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/sdk-overview))", "fit_eval_prereqs": "An Azure subscription (\"create a free account if needed\"); a hub-based project or a Foundry project; for a Foundry project, subscription Owner access. Credit card, billing enablement, cluster and provider key are not mentioned ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `response = chat.complete(model=\"gpt-4o-mini\", messages=[...])` (Python; C#/JavaScript variants shown) ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))", "fit_gateway_platform": "third_party_documented", "fit_gateway_platform_note": "Yes (documented by Kong): the Kong `ai-proxy-advanced` plugin has an Azure AI Foundry / Azure OpenAI chat-route example with global, service, route, consumer and consumer-group scoped configuration. ([Kong ai-proxy-advanced Azure chat route example](https://developer.konghq.com/plugins/ai-proxy-advanced/examples/azure-chat-route/))", "fit_helm_chart": "Foundry Local ships as an Azure Arc Kubernetes extension (Microsoft.Foundry / inference-operator) on Arc clusters 1.29+; no chart name or repo is stated", "fit_kubernetes": "helm_unnamed", "fit_kubernetes_note": "Foundry Local can be deployed as an Azure Arc Kubernetes extension with extension type `Microsoft.Foundry`, extension name `inference-operator`, and release namespace `foundry-local-operator`; Helm is also supported, but no chart name or repository is stated. The deployment requires an Azure Arc-enabled Kubernetes cluster running version 1.29 or later. ([Deploy Foundry Local with the Azure Arc extension](https://learn.microsoft.com/en-us/azure/azure-sovereign-clouds/private/foundry-local/deploy-foundry-local-arc-extension))", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Microsoft Foundry supports remote MCP servers through the Responses API using an `mcp` tool with fields including `server_label`, `server_url`, optional `require_approval`, and custom `headers`; approval uses `mcp_approval_request` and `mcp_approval_response`. Foundry Agent Service also supports MCP tool invocation and approval workflows, while Foundry Toolbox can expose managed tools and skills through a single MCP endpoint. ([Use the Responses API with Microsoft Foundry](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers" ] }, "fit_model_gate": "quota_gated", "fit_model_gate_note": "Yes, in the form of deployment plus quota: you must deploy `gpt-4o-mini` before calling it, and rate limits/quota apply (\"Rate limit is exceeded… you want to get more quota\") ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code)). Quota mechanics: \"If instant model requests are throttled, you can request a global quota increase or create a deployment with reserved capacity\"; \"Use the shared quota only for creating temporary test endpoints, not production endpoints\"; \"Some model and region combinations might not support quota increases\"; requesting an increase is a 7-step procedure ending \"Select the Request quota button in the upper-right corner\" ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))", "fit_onboarding_notes": "Installs differ by language: `pip install azure-ai-projects azure-ai-inference azure-identity`, `pip install openai azure-ai-projects azure-identity`, `dotnet add package Azure.Identity|Azure.AI.Projects|Azure.AI.Agents.Persistent|Azure.AI.Inference`, or `npm install`. REST access tokens expire in 60–90 minutes. Quota changes \"allow up to 15 minutes for changes to propagate,\" and \"If the issue persists after 24 hours, contact Azure support.\" ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code), [Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Microsoft Foundry documents OpenAI SDK compatibility using `base_url`/`baseURL`, with the exact base URL `https://.openai.azure.com/openai/v1/`; the deployment name is passed in the `model` field and authentication can use `api_key`. Foundry project endpoints also expose an OpenAI-compatible `/openai/v1` endpoint. ([Microsoft Foundry model endpoints](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints))", "fit_prod_prereqs": "RBAC and quota administration: Cognitive Services Usages Reader at subscription level to view quota; Owner or Contributor on the subscription to request increases; Cognitive Services Contributor + Usages Reader to edit quota allocations; an Azure subscription and a Foundry project ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [ "LangChain", "LangGraph", "LlamaIndex" ], "fit_python_frameworks_note": "Microsoft Foundry documents LangChain and LangGraph through `langchain-azure-ai`, including `AzureAIOpenAIApiChatModel` and `AzureAIDocumentIntelligenceTool`; vector-store packages include `langchain-azure-cosmosdb`. Foundry classic documents LlamaIndex through `llama-index-llms-azure-inference` and `llama-index-embeddings-azure-inference`, with `AzureAICompletionsModel` and `AzureAIEmbeddingsModel`. ([Microsoft Foundry LangChain integration](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/langchain))", "fit_quickstart_steps": 54, "fit_quickstart_steps_caveat": "54 numbered items interleave several language tracks and portal tracks rather than forming one linear procedure, and the final item concerns managing files on a vector store rather than making a first call. Treat this as \"not one procedure\", not as 54 steps of work.", "fit_quickstart_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "platform_provider_resources", "fit_terraform_note": "Terraform support is documented through the Terraform AzAPI Provider and AzureRM Provider for managing Microsoft Foundry resources, deployments, projects, connections, and capability hosts. The Terraform Registry documents provider `hashicorp/azurerm` and resource `azurerm_ai_foundry`; the newer Foundry guidance recommends provisioning the new resource with `azurerm_cognitive_account` rather than legacy hub-based projects. ([Create a Microsoft Foundry resource with Terraform](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-resource-terraform))", "fit_terraform_ref": "AzAPI and AzureRM; azurerm_ai_foundry exists, though newer guidance points to azurerm_cognitive_account", "fit_third_party_documented": { "vercel_ai_sdk": "a community publisher (@quail-ai)", "gateway_platform": "Kong" }, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "community_provider", "fit_vercel_ai_sdk_note": "A community Vercel AI SDK provider is documented as `@quail-ai/azure-ai-provider`, created with `createAzure`. This is an unofficial/community integration, not Microsoft vendor documentation. ([Community Azure AI provider for Vercel AI SDK](https://ai-sdk.dev/providers/community-providers/azure-ai))", "fit_vercel_ai_sdk_package": "@quail-ai/azure-ai-provider" }, { "slug": "cloudflare-ai-gateway", "name": "Cloudflare AI Gateway", "company": "Cloudflare", "tagline": "Edge proxy in front of a curated set of AI providers, with caching, rate limiting, DLP and analytics.", "category": "cloud_platform", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2009, "hq_country": "United States", "url_home": "https://developers.cloudflare.com/ai-gateway/", "url_docs": "https://developers.cloudflare.com/ai-gateway/features/", "url_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "url_status": "https://www.cloudflarestatus.com", "url_github": null, "url_changelog": "https://developers.cloudflare.com/ai-gateway/changelog/", "url_models_api": null, "github_stars": null, "model_count": null, "model_count_note": "Cloudflare publishes no gateway model count. Its upstream list is narrower than the marketplaces: docs say all features work across 20+ supported AI providers, and the provider-native page enumerates 23 (Workers AI, Bedrock, Anthropic, Azure OpenAI, Baseten, Cartesia, Cerebras, Cohere, Deepgram, DeepSeek, ElevenLabs, Fal AI, Google AI Studio, Vertex AI, Groq, HuggingFace, Ideogram, Mistral, OpenAI, OpenRouter, Parallel, Perplexity, Replicate, xAI).", "provider_count": 23, "token_markup_pct": 0, "credit_fee_pct": 5, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "BYOK is a built-in feature at no stated extra cost: provider keys are stored encrypted in Cloudflare and rotated without code changes across 20+ providers.", "seat_fee_usd_month": 0, "free_tier": "AI Gateway core features (dashboard analytics, caching, rate limiting) are free on all plans; DLP scanning is free on all plans; Workers Free stores 100,000 logs total across gateways, Workers Paid 10,000,000 logs per gateway.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Unified Billing credit purchases", "amount": "5% fee on all credits purchased ($100 of credit costs $105)" }, { "label": "Logpush", "amount": "Workers Paid only: 10 million requests/month included, then $0.05 per million" }, { "label": "Guardrails inference", "amount": "Billed as Workers AI token usage for @cf/meta/llama-guard-3-8b evaluations" } ], "self_host_cost_note": null, "openai_compatible": null, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "Provider-native APIs (OpenAI, Anthropic, Google, Workers AI and others)" ], "sdk_note": "One line of code: change the provider base URL to the gateway endpoint; provider-native request formats are preserved.", "cap_failover": true, "cap_load_balancing": null, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": true, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": true, "cap_audio": true, "cap_video": null, "cap_batch": null, "latency_overhead_ms": null, "latency_source_note": "No vendor figure. A third-party hands-on (Kalvium Labs, 2026-04-24) reports 20-60 ms added per request measured across several production systems on US and European cloud regions, i.e. under 5% of a typical 0.8-4 s LLM call; Cloudflare states caching can cut latency up to 90% on cache hits.", "throughput_rps": null, "soc2": null, "hipaa_baa": null, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "No ZDR commitment is published. The equivalent is disabling payload collection per request.", "air_gapped": false, "best_for": "Teams already on Cloudflare Workers who want free caching, analytics, spend limits, DLP and guardrails at the edge.", "strengths": [ "Core gateway features are free on every Cloudflare plan, with no markup on provider tokens", "Free DLP scanning for PII, financial and healthcare data, plus llama-guard-based guardrails", "Dynamic Routing gives visual, code-free routing by user segment, geography, content or A/B split", "Spend limits scoped by model, provider or custom metadata, with automatic blocking", "Edge caching claimed to cut latency up to 90% and analytics via GraphQL API" ], "weaknesses": [ "Upstream provider list (~23) is far narrower than OpenRouter or Requesty, and no model count or public catalog endpoint is published", "Unified Billing adds a 5% fee on credit purchases", "No published SOC 2 / HIPAA / ZDR posture specific to AI Gateway", "Log storage caps (100k on Workers Free) and Logpush restricted to Workers Paid" ], "avoid_if": "You need a broad model marketplace, a public pricing catalog endpoint, or documented gateway-level compliance commitments.", "reviews": [ { "url": "https://dzone.com/articles/conversational-ai-cloudflare-workers-ai-gateway", "why": "Mayur Vekariya's build article puts AI Gateway in front of Workers AI with KV, Durable Objects and Vectorize, showing configuration and caching behaviour in a real app.", "date": "2025-10-07", "type": "written", "title": "Building a Production-Ready Conversational AI Agent With Cloudflare Workers and AI Gateway", "publisher": "DZone" }, { "url": "https://www.infoq.com/news/2024/06/cloudflare-ai-gateway/", "why": "Renato Losio's independent GA writeup explains what the gateway proxies (Workers AI, OpenAI, Bedrock, Vertex, Azure OpenAI, HuggingFace) and the caching/rate-limiting story.", "date": "2024-06-02", "type": "written", "title": "Cloudflare AI Gateway Now Generally Available", "publisher": "InfoQ" }, { "url": "https://www.youtube.com/watch?v=_CXZ1uo8ZQA", "why": "Mehul Mohan dissects AI-gateway architecture (normalization, fallback, analytics) and where Cloudflare's offering sits against OpenRouter and Vercel.", "date": "2025-11-21", "type": "video", "title": "How to build your own AI gateway (and do a generational ragebait on twitter)", "publisher": "YouTube - Mehul Mohan" }, { "url": "https://www.reddit.com/r/CloudFlare/comments/1uwe2kn/ai_gateway_is_not_production_ready/", "why": "21-comment thread with hard failure reports: random 500s from the gateway, an essential model down for over a week, and users planning a move to Vercel AI Gateway.", "date": "2026-07-14", "type": "community", "title": "AI Gateway is not production ready", "publisher": "Reddit - r/CloudFlare" }, { "url": "https://www.reddit.com/r/LLMDevs/comments/1lycx6i/either_i_dont_get_cloudflares_ai_gateway_or_it/", "why": "6-comment thread on the gateway's actual security model, including per-IP rate limits and the risk of credit drain when clients call it directly.", "date": "2025-07-12", "type": "community", "title": "Either I don't get Cloudflare's AI gateway, or it does not do what I expected it to", "publisher": "Reddit - r/LLMDevs" } ], "sources": { "license": "https://developers.cloudflare.com/ai-gateway/", "sdk_note": "https://developers.cloudflare.com/ai-gateway/", "url_docs": "https://developers.cloudflare.com/ai-gateway/features/", "cap_audio": "https://developers.cloudflare.com/ai-gateway/usage/providers/", "free_tier": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "air_gapped": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "deployment": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "extra_fees": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "hq_country": "https://en.wikipedia.org/wiki/Cloudflare", "url_status": "https://www.cloudflarestatus.com", "zdr_status": "https://developers.cloudflare.com/ai-gateway/observability/logging/", "cap_budgets": "https://developers.cloudflare.com/ai-gateway/features/", "url_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "api_surfaces": "https://developers.cloudflare.com/ai-gateway/usage/providers/", "cap_failover": "https://developers.cloudflare.com/ai-gateway/", "founded_year": "https://en.wikipedia.org/wiki/Cloudflare", "product_fate": "https://developers.cloudflare.com/ai-gateway/changelog/", "regions_note": "https://www.cloudflare.com/trust-hub/responsible-ai/", "tracing_note": "https://developers.cloudflare.com/ai-gateway/observability/otel-integration/", "byok_fee_note": "https://developers.cloudflare.com/ai-gateway/features/", "cap_image_gen": "https://developers.cloudflare.com/ai-gateway/usage/providers/", "url_changelog": "https://developers.cloudflare.com/ai-gateway/changelog/", "vendor_status": "https://developers.cloudflare.com/ai-gateway/changelog/", "byok_supported": "https://developers.cloudflare.com/ai-gateway/features/", "cap_guardrails": "https://developers.cloudflare.com/ai-gateway/features/", "certifications": "https://www.cloudflare.com/trust-hub/compliance-resources/soc-2/", "config_as_code": "https://developers.cloudflare.com/api/terraform/resources/ai_gateway/", "credit_fee_pct": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "provider_count": "https://developers.cloudflare.com/ai-gateway/usage/providers/", "retention_days": "https://developers.cloudflare.com/ai-gateway/reference/limits/", "cap_rate_limits": "https://developers.cloudflare.com/ai-gateway/features/", "logs_by_default": "https://developers.cloudflare.com/ai-gateway/observability/logging/", "tracing_support": "https://developers.cloudflare.com/ai-gateway/observability/otel-integration/", "cap_cache_simple": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "model_count_note": "https://developers.cloudflare.com/ai-gateway/usage/providers/", "token_markup_pct": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "cap_observability": "https://developers.cloudflare.com/ai-gateway/features/", "cap_pii_redaction": "https://developers.cloudflare.com/ai-gateway/features/", "last_release_date": "https://developers.cloudflare.com/ai-gateway/changelog/", "subprocessors_url": "https://www.cloudflare.com/gdpr/subprocessors/", "seat_fee_usd_month": "https://developers.cloudflare.com/ai-gateway/reference/pricing/", "guardrails_location": "https://developers.cloudflare.com/ai-gateway/features/guardrails/", "latency_source_note": "https://www.kalviumlabs.ai/blog/cloudflare-ai-gateway-vs-direct-openai-api/", "requires_vendor_sdk": "https://developers.cloudflare.com/ai-gateway/usage/chat-completion/", "data_export_available": "https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/", "prompt_transits_vendor": "https://developers.cloudflare.com/ai-gateway/features/guardrails/", "cap_conditional_routing": "https://developers.cloudflare.com/ai-gateway/features/", "trains_on_customer_data": "https://www.cloudflare.com/trust-hub/responsible-ai/", "cloudflare_ai_gateway_caching": "https://developers.cloudflare.com/ai-gateway/features/caching/", "cloudflare_ai_gateway_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/" }, "verified_at": { "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "cap_audio": "2026-08-29", "free_tier": "2026-08-29", "air_gapped": "2026-08-29", "deployment": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "cap_budgets": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "tracing_note": "2026-09-16", "byok_fee_note": "2026-08-29", "cap_image_gen": "2026-08-29", "url_changelog": "2026-08-29", "byok_supported": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "credit_fee_pct": "2026-08-29", "provider_count": "2026-08-29", "cap_rate_limits": "2026-08-29", "tracing_support": "2026-09-16", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "token_markup_pct": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "seat_fee_usd_month": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-16T13:41:52.154147+00:00", "vendor_status": "hyperscaler", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "A first-party Cloudflare product shipping steadily, including unified Workers AI billing and anomalous-spend insights in August 2026.", "last_release_date": "2026-08-19", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "A proxy that sits between your application and the model, so every request passes through Cloudflare.", "logs_by_default": "full_content", "logs_note": "The most permissive default in this catalogue. Logs are enabled by default for each gateway and include the user prompt and the model response alongside metadata. Opting out is per request, via the cf-aig-collect-log-payload header.", "retention_days": null, "retention_note": "Retention is capped by count, not time: 100,000 logs per account on the free plan and 10 million per gateway on paid. Logs persist until the cap is reached, at which point new logs stop being saved.", "zdr_status": "not_published", "trains_on_customer_data": "no", "training_note": "Cloudflare does not use customer content to train any large language models, and does not train its own machine-learning models on customer content without consent.", "subprocessors_url": "https://www.cloudflare.com/gdpr/subprocessors/", "regions_note": "A global edge with platform-level data-residency support, but no AI-Gateway-specific region pinning and no customer-VPC deployment.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II covering security, confidentiality and availability", "status": "certified" }, "fedramp": { "detail": "High authorisation for Cloudflare for Government", "status": "certified" }, "gdpr_dpa": { "detail": "subprocessor list and GDPR programme published", "status": "certified" }, "iso27001": { "detail": "also 27018 and 27701", "status": "certified" }, "hipaa_baa": { "detail": "services described as aligning to HIPAA needs; no BAA offer found", "status": "alignment_only" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Guardrails inspect interactions in real time using a Llama Guard model on Workers AI — Cloudflare's infrastructure, not yours.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "markup_only", "pricing_model_note": "Free gateway core + credit top-up fee (Unified Billing, 5% on credit purchases) + plan-tied log/Logpush limits. Inference passed through at cost — 0% markup.", "min_commitment": "None stated. Enterprise customers reach out to their account team.", "overage_note": "The bite is on logs, not requests. Persistent log storage capped at 100,000 logs total across all gateways on Workers Free and 10,000,000 logs per gateway on Workers Paid, with automatic log deletion. Logpush is Workers Paid only, includes 10M requests/month then charges +$0.05/million. Guardrails usage is billed as Workers AI token-based inference.", "cache_support": "exact", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact-match caching: based on exact match of the entire request. Disabled by default. Serves cached provider responses from Cloudflare's cache. No differential cached-token pricing published. A cache hit skips the provider call entirely — customer captures the full saving and Cloudflare charges nothing for the cache itself. The only related cost is log storage of the cached requests.", "cost_controls": { "enforcement": "pre_request", "spend_alerts": { "note": "Not stated; only stated monitoring is 'You can monitor your usage in the AI Gateway dashboard' (after-the-fact).", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Supported, free on all plans, enforced pre-request.", "tier": "free", "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Not stated.", "cost_export": "Logpush on Workers Paid (10M requests/month included, then $0.05/million). CSV, cost API, webhook and warehouse export not stated.", "product_kind_note": "Cloudflare positions AI Gateway as the way to \"gain visibility and control over your AI apps\", gathering \"insights on how people are using your application with analytics and logging\" and controlling scale with \"caching, rate limiting, as well as request retries, model fallback, and more\" ([AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20). Functionally it is an observability plane that also proxies, exposing \"an OpenAI-compatible `/chat/completions` endpoint\" for \"integration with multiple AI providers using a single URL\" ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07).", "deployment_modes": [ "saas" ], "deployment_note": "No installation: \"The `default` gateway is created automatically on your first request — no setup needed\", and a specific gateway ID can replace `default`; the endpoint embeds the Cloudflare account ID (\"Replace `{account_id}` with your Cloudflare account ID\") ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07). Tier gating is on ancillary features, not deployment: core features (dashboard analytics, caching, rate limiting) and DLP scanning are free on all plans; persistent-log storage is capped at 100,000 logs total on Workers Free vs 10,000,000 logs per gateway on Workers Paid; Logpush is \"only available on the Workers Paid plan\" (10 million requests/month included, then $0.05/million) ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "not_documented", "api_images": "not_documented", "api_audio": "not_documented", "api_batch": "not_documented", "api_surface_note": "Two modes: a unified OpenAI-compatible endpoint where you switch providers by changing the `model` and `apiKey` parameters (examples use the OpenAI JS SDK), and provider-native pass-through where you keep the provider's own request format and paths ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07; [Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20). Gateway-level auth adds a `cf-aig-authorization` header, and when using stored keys you \"remove provider authorization headers from your requests\" while still passing `cf-aig-authorization`; a non-default stored key is selected with `cf-aig-byok-alias` ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31). Streaming caveats: n.a.", "model_count_low": null, "model_count_high": null, "model_count_source_note": "n.a.: no model total is published; the overview only invites users to \"explore all AI models available through AI Gateway\" ([AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20).", "provider_count_low": 24, "provider_count_high": 24, "provider_count_source_note": "24 providers are listed by name — Workers AI, Amazon Bedrock, Anthropic, Azure OpenAI, Baseten, Cartesia, Cerebras, Cohere, Deepgram, DeepSeek, ElevenLabs, Fal AI, Google AI Studio, Google Vertex AI, Groq, HuggingFace, Ideogram, Mistral AI, OpenAI, OpenRouter, Parallel, Perplexity, Replicate, xAI — with no stated total ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).", "first_party_vs_partner": "Documented mix: mostly third-party routed providers, plus Cloudflare's own Workers AI as a supported provider, and Guardrails inference runs on Workers AI using `@cf/meta/llama-guard-3-8b` ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20; [AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19).", "custom_endpoint_support": "Yes: register a Custom Provider via the API or \"Compute & AI > AI Gateway > Custom Providers\" with a unique `slug` and an HTTPS `base_url`, and self-hosted models are explicitly supported; management calls need a Cloudflare API token with `AI Gateway - Edit` ([Custom providers](https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/)). vLLM/Ollama/SageMaker by name: n.a.", "byok_mode": "both", "merchant_of_record": "Split by mode: with BYOK the upstream provider bills the customer's own account; with Unified Billing Cloudflare invoices, applying \"a 5% fee… to all credits purchased through Unified Billing\" (a $100 credit purchase results in a $105 charge) while \"inference pricing from providers is passed through with no markup — you pay the same per-token rates as you would directly with the provider\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/)). Guardrails usage \"is billed as Workers AI token-based inference\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/)). An explicit statement of the invoicing entity for provider tokens: n.a.", "key_management_note": "Stored provider keys live in Cloudflare Secrets Store, which gives \"secure storage and limit exposure\", \"easier key rotation\" and pairing with \"rate limit, budget limit and other restrictions with Dynamic Routes\"; dashboard-created secrets are named automatically, API-created secrets must follow the required naming convention because \"the `secret_id` returned by Secrets Store is not used by AI Gateway for runtime lookup\" ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31). Multiple keys per provider are supported, each with an alias (`default` when unspecified), selectable per request via `cf-aig-byok-alias`; the dashboard shows keys by provider with last-used time and `active`/`expired`/`invalid` status, rotation takes effect immediately \"without any code changes or downtime\", and keys can be deleted ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/)). Because it is hosted-only, requests transit Cloudflare and can be persisted as logs (with plan-based storage caps and free DLP scanning); the overview notes logging gives \"insight on requests and errors\" but does not state whether Cloudflare stores request content by default ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20).", "api_openai_chat_completions_note": "Yes: an OpenAI-compatible unified endpoint at `/compat/chat/completions` (provider-native routes also accept paths such as `/v1/chat/completions`), with models addressed in `{provider}/{model}` form, e.g. `openai/gpt-5-mini`, `google-ai-studio/gemini-2.5-flash`, `anthropic/claude-sonnet-4-5` ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07; [Custom providers](https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/)).", "api_anthropic_messages_note": "Yes via provider-native passthrough: Anthropic is a supported provider whose native endpoint is proxied ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20). The specific `/v1/messages` path is not shown on the pages fetched: n.a.", "api_openai_responses_note": "n.a. (not documented on the pages fetched).", "api_embeddings_note": "n.a. as a documented gateway path; embedding-capable providers (Cohere, Workers AI, Google) are proxied provider-natively ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).", "api_images_note": "n.a. as a documented gateway path, though image providers (Fal AI, Ideogram, Replicate) are supported provider-natively ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).", "api_audio_note": "n.a. as a documented gateway path; audio providers are supported provider-natively — Deepgram and ElevenLabs (STT/TTS) and Cartesia ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).", "api_batch_note": "n.a.", "deployment_modes_note": "hosted SaaS on Cloudflare's network only, \"available to use on all plans\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20). Self-host, hybrid/VPC, on-prem, air-gapped: n.a.", "byok_mode_note": "both: BYOK lets you \"securely store your AI provider API keys directly in the Cloudflare dashboard\" ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31), while Unified Billing runs usage against Cloudflare account credits ([Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/); [AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19). Provider credentials take precedence over Unified Billing credits when both exist ([Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/)).", "access_meta": { "qualified": [ "api_anthropic_messages", "api_embeddings", "api_images", "api_audio" ], "model_count_kind": null, "provider_count_kind": "enumerated" }, "rel_timeout_config": "per_request", "rel_timeout_note": "Header `cf-aig-request-timeout`, set on a provider-specific endpoint, in milliseconds. Behavior is first-response based: as long as the first part of the response returns in time, \"such as when streaming a response\", the gateway waits. Default value not stated ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).", "rel_retry_config": "per_request", "rel_retry_note": "`per_request` plus `dashboard_only` defaults. Headers `cf-aig-max-attempts`, `cf-aig-retry-delay` (ms, max 5 seconds) and `cf-aig-backoff` (`constant` | `linear` | `exponential`), with \"a maximum of five retry attempts\". \"On the final retry attempt, your gateway will wait until the request completes, regardless of how long it takes.\" Gateway-level defaults (\"up to 5 attempts\", constant/linear/exponential, 100ms–5s delay) apply to all requests and can be overridden per request. The default attempt count itself is not published: `n.a.` ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/), [Cloudflare auto-retry changelog](https://developers.cloudflare.com/changelog/post/2026-04-02-auto-retry-upstream-failures/)).", "rel_retry_default_note": "`per_request` plus `dashboard_only` defaults. \"On the final retry attempt, your gateway will wait until the request completes, regardless of how long it takes.\" Gateway-level defaults (\"up to 5 attempts\", constant/linear/exponential, 100ms–5s delay) apply to all requests and can be overridden per request.", "rel_fallback_config": "per_request", "rel_fallback_note": "ORDERED, expressed as an array of request objects to the Universal Endpoint: \"You can add as many fallbacks as you need, just by adding another object in the array\", and the response header `cf-aig-step` reports which step served (`0` primary, `1` second model, `2` third, incrementing per fallback). No weighted split at this layer, and the fallback key itself is not named ([Cloudflare AI Gateway fallbacks](https://developers.cloudflare.com/ai-gateway/configuration/fallbacks/)).", "rel_fallback_shape": "weighted", "rel_health_check_config": "not_documented", "rel_health_check_note": "There is no upstream health check, circuit breaker or provider-ejection control; the documented mechanism is retry-then-fallback on error ([Cloudflare auto-retry changelog](https://developers.cloudflare.com/changelog/post/2026-04-02-auto-retry-upstream-failures/)).", "rel_multi_region_config": "not_configurable", "rel_multi_region_note": "Cloudflare runs the network: the Gateway Worker \"operates near the user's location\" on \"Cloudflare's vast network of data centers\", but no user-configurable cross-region failover exists ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).", "rel_load_balancing_config": "dashboard_only", "rel_load_balancing_note": "`dashboard_only` (plus JSON config), via Dynamic Routing: the **Percentage** node \"Routes requests probabilistically across multiple outputs\", i.e. weighted-by-percentage splitting; Rate Limit and Budget Limit nodes \"switch to fallback when exceeded\". Routes are built in a visual interface or JSON-based configuration ([Cloudflare Dynamic Routing](https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/)). The product page adds \"Automatically route requests based on latency, cost, or availability\" ([Cloudflare AI Gateway product page](https://www.cloudflare.com/products/ai-gateway/)).", "architecture_class": "edge_worker", "architecture_note": "Explicitly built on Cloudflare Workers (\"a serverless platform that runs on the Cloudflare network\"), with a Gateway Worker and Durable Objects (each Account Manager with its own SQLite database) for logging ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).", "self_host_data_plane": "no", "self_host_artifact_note": "No self-host artifact; usage is via the gateway URL or Workers bindings such as `env.AI.gateway(\"my-gateway\").getUrl(\"openai\")` ([Cloudflare AI Gateway product page](https://www.cloudflare.com/products/ai-gateway/)).", "latency_claims": [], "throughput_claims": [ { "url": "https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/", "unit": "requests/year", "scope": "marketing", "value": "over 2B", "metric": "fleet total", "publisher": "vendor", "conditions": "Aggregate proxied volume in one year, not a capacity benchmark." } ], "streaming_support": "yes", "streaming_note": "Supported. Caveat tied to the timeout semantics: `cf-aig-request-timeout` is evaluated on the first part of the response, so once streaming starts the gateway will wait for the rest ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).", "guard_pii": "sync_block", "guard_pii_note": "AI Gateway DLP scans prompts and responses in real time using Cloudflare's DLP detection engines and account-level profiles. On **Block**, \"the provider response is discarded and replaced with a DLP error response (status 400)\"; on **Flag**, findings are attached to the `cf-aig-dlp` response header and logged while the original response is still returned. The page does not state that DLP redacts ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))", "guard_injection": "sync_block", "guard_injection_note": "Prompt-injection detection uses the Workers AI model `@cf/meta/prompt-guard-2-86m`, exposed as category P1 with per-category Flag / Ignore / Block actions ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/), [Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/))", "guard_content": "sync_block", "guard_content_note": "Llama Guard 3 8B on Workers AI evaluates hazard categories S1–S13 inline, again with Flag / Ignore / Block per category ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/))", "guard_custom": "sync_block", "guard_custom_note": "Custom policy takes the form of Cloudflare One DLP detection profiles (predefined or custom) selected at the gateway level; \"There is no per-request header to select specific DLP profiles or to bypass DLP scanning\", and different policies require separate gateways ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages — gateway-level configuration governs guardrails and DLP, but an administrative model/provider allow-deny list is not documented in what was fetched ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/), [DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "configurable", "guard_failure_mode_note": "Documented per action rather than per outage: with a category set to Block, \"the request will be blocked\" (fail closed for that category), while Flag returns the response and records the finding (fail open) ([Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/), [DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/)). Behaviour when the guardrail model itself is unavailable is `not_documented`", "log_content": "full_request_and_response", "log_content_note": "Logging is enabled by default and includes prompt and response payloads, along with DLP FLAG/BLOCK fields ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))", "log_opt_out": "yes", "log_opt_out_note": "`cf-aig-collect-log-payload: false` keeps the log entry as metadata only, and `cf-aig-collect-log: false` disables the log entry for that request ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))", "tracing_support": "otel", "tracing_note": "Configurable native OTLP export in JSON or Protobuf. Documented attributes include model, provider, usage, cost, custom metadata and prompt/completion payloads. Review content handling and collector access before enabling exports; validate parent-context propagation in your application.", "export_destinations": [ "Logpush", "R2", "Webhook" ], "export_destinations_note": "Workers Logpush to R2 and other object-storage/HTTP destinations, limited to 10 million logs per gateway across up to 4 Logpush jobs ([Logpush](https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/))", "feedback_capture": "yes", "feedback_capture_note": "Yes — the response header `cf-aig-log-id` identifies the log entry, and a `PATCH` to the AI Gateway logs API records feedback (for example `-1` for negative), requiring an API token with `AI Gateway - Edit`/Write ([Add human feedback using API](https://developers.cloudflare.com/ai-gateway/evaluations/add-human-feedback-api/))", "eval_hooks": "yes", "eval_hooks_note": "Yes, on Cloudflare's platform — Evaluations run against datasets built from filtered logs, reporting performance, speed, and cost; human feedback is the first evaluator and is in open beta ([Evaluations](https://developers.cloudflare.com/ai-gateway/evaluations/))", "observability_requires_saas": null, "log_retention_note": "Storage-limit driven and configurable: when the gateway's log limit is reached, the oldest logs are automatically deleted ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` for gateway overhead. The only timing published is operational: \"New requests show up within 15 seconds\" for log visibility; positioning the Durable Object near the Gateway Worker \"significantly enhances the speed of log insertion and query operations\" with no number ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).", "multi_surface": { "rel_retry": [ "per_request", "dashboard_only" ] }, "throughput_prose": "No RPS or tokens/sec benchmark. Scale figures only: AI Gateway proxied \"over 2 billion requests in just one year\"; storage limits of 10 million logs per Durable Object / per gateway and 100 million per account at the default 10 gateways ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).", "benchmark_independence": "Vendor-published operational statistics only; no benchmark and no claims about competitors ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "choose_at_setup", "guard_custom_default": "choose_at_setup", "rel_notes": "Two different retry/fallback surfaces coexist: header-driven retries with `cf-aig-*` on provider endpoints, and node-based Dynamic Routing where a fallback is a graph edge. The scaling blog also shows an older-style `\"retry\": 2` / `\"delay\": 200` / `\"onComplete\"` JSON shape, which does not match the header names in the current docs ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/), [Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).", "guard_notes": "Two enforcement caveats matter. Guardrails do not enforce on streaming responses — streaming traffic is evaluated/logged through the REST path but not blocked ([Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/)). DLP on streaming SSE responses \"buffers the full streamed response before scanning\", which removes the latency benefit of streaming, while request-only scanning avoids that cost; base64 images, external URLs, and binary multipart data are not inspected ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/)).", "fit_agent_features_note": "The provided Cloudflare documentation describes function calling and tool use through supported models and provider tools, Agents SDK sessions, durable state, WebSockets, scheduling, recoverable execution, and multi-step agent harnesses controlling planning, tool use, tool results, streaming, and continuation. A2A is not documented. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_byok_required": "no", "fit_byok_required_note": "No for the documented Workers AI call; provider authentication options are \"Unified Billing, BYOK, or request headers\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))", "fit_client_sdk_langs": [ "JavaScript", "Python" ], "fit_client_sdks_note": "Documented clients include the OpenAI SDK for JavaScript and Python; Anthropic SDK for JavaScript and Python; `@google/genai`; `@mistralai/mistralai`; the Cohere Python SDK; Groq SDK for JavaScript; HuggingFace `@huggingface/inference`; Fal AI `@fal-ai/client`; `ai-gateway-provider` for the Vercel AI SDK; and Cloudflare Workers `env.AI` bindings. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_cloud_iam": "upstream_credentials_only", "fit_cloud_iam_note": "Amazon Bedrock support is documented with AWS SigV4 credentials, including `accessKeyId`, `secretAccessKey`, `region`, and an optional AWS STS `sessionToken`. Google Vertex AI supports service-account JSON credentials or direct GCP access tokens, including GCP service-account authentication; Azure OpenAI documentation specifies an Azure OpenAI API key rather than Entra or managed identity. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_cloudflare_workers": "user_facing", "fit_cloudflare_workers_note": "Cloudflare Workers support is documented through the `env.AI` binding, including `env.AI.run()` and `env.AI.gateway()`, third-party models, Workers AI models, logging, feedback, metadata, and gateway configuration. Workers AI can also integrate with AI Gateway by adding a gateway object to a Workers AI request; the gateway must be in the same account as the Worker. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated as required: \"AI Gateway is available to use on all plans.\" and \"AI Gateway's core features available today are offered for free.\" ([Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))", "fit_ecosystem_notes": "AI Gateway is operated through a Cloudflare account and dashboard/control plane, with Cloudflare API tokens, gateway configuration, billing, logging, analytics, and provider-key management. It supports Workers AI, Anthropic, Google Gemini, OpenAI, Replicate, custom providers with HTTPS endpoints and custom domains, caching, rate limiting, retries, DLP, OpenTelemetry, and Stripe-related configuration; self-hosting and on-premises deployment are not documented. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_eval_prereqs": "A Cloudflare account, your Account ID, and a Cloudflare API token with \"AI Gateway - Read\", \"AI Gateway - Edit\" and \"Workers AI - Read\" permissions. No credit card, cloud account, cluster or provider key stated; pricing page says \"all it takes is a Cloudflare account and one line of code to get started.\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "no", "fit_eval_snippet_note": "No copy-pasteable completion request was rendered on the fetched pages; the documented request shape is a Workers AI model with the `@cf/` prefix plus header `cf-aig-gateway-id: default`, and the unified API exposes an OpenAI-compatible `/chat/completions` endpoint where you \"replace `{account_id}` with your Cloudflare account ID\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Unified API (OpenAI compat) · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. (not documented)", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Cloudflare Agents explicitly support MCP tools, but an AI Gateway MCP server, MCP gateway or registry, and dedicated MCP tool proxy are not documented. Separately, Cloudflare Zero Trust documents MCP server portals that proxy upstream MCP tools and prompts behind one HTTP endpoint at `/mcp`. ([Cloudflare MCP server portals](https://developers.cloudflare.com/cloudflare-one/access-controls/ai-controls/mcp-portals/))", "fit_meta": { "not_documented_fields": [ "fit_gateway_platform", "fit_kubernetes" ] }, "fit_model_gate": "none", "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or waitlist gate is stated ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))", "fit_onboarding_notes": "Gateway creation can be skipped: the `cf-aig-gateway-id: default` header auto-creates the default gateway on the first authenticated request. Workers AI requests use Standard billing by default and can be switched to Unified billing ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "OpenAI-compatible SDK usage is documented. The exact documented base URL is `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/openai`; the OpenAI JavaScript and Python SDKs are shown, and Grok documentation also says to replace the xAI endpoint with `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/grok`. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_prod_prereqs": "n.a. (not documented): \"Production infrastructure requirements: None stated.\" Prepaid AI Gateway credits are mentioned for unified billing ([Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/), [Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [ "LlamaIndex" ], "fit_python_frameworks_note": "LlamaIndex integration is documented through the `llama-index-llms-cloudflare-ai-gateway` package and the `CloudflareAIGateway` class (`Bases: LLM`). LangChain support for Cloudflare AI Gateway is not documented in the provided evidence. ([LlamaIndex Cloudflare AI Gateway integration](https://developers.llamaindex.ai/python/framework-api-reference/llms/cloudflare_ai_gateway/))", "fit_quickstart_steps": 13, "fit_quickstart_steps_caveat": "13 entries spread across three separate numbered lists (2 + 7 + 4), so no single steps-to-first-call figure exists.", "fit_quickstart_url": "https://developers.cloudflare.com/ai-gateway/get-started/", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "official_provider", "fit_terraform_note": "Official Terraform support is documented through the `cloudflare/cloudflare` provider. Documented resources include `cloudflare_ai_gateway` and `cloudflare_ai_gateway_dynamic_routing`, with the latter documented at `registry.terraform.io/providers/cloudflare/cloudflare/latest/docs/resources/ai_gateway_dynamic_routing`. ([Terraform Cloudflare AI Gateway resource](https://registry.terraform.io/providers/cloudflare/cloudflare/latest/docs/resources/ai_gateway))", "fit_terraform_ref": "cloudflare/cloudflare with cloudflare_ai_gateway and cloudflare_ai_gateway_dynamic_routing", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Vercel AI SDK integration is documented through the `ai-gateway-provider` npm package. The package provides `createAiGateway`, `createUnified`, `createOpenAI`, `createAnthropic`, `createGoogle`, and `createXai`; Cloudflare Workers usage can also use `workers-ai-provider`. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))", "fit_vercel_ai_sdk_package": "ai-gateway-provider" }, { "slug": "google-vertex-ai", "name": "Google Vertex AI", "company": "Google Cloud", "tagline": "Google Cloud's model platform for Gemini plus 200+ Model Garden models, now branded Gemini Enterprise Agent Platform.", "category": "cloud_platform", "deployment": "managed_only", "license": "Proprietary", "founded_year": null, "hq_country": "United States", "url_home": "https://cloud.google.com/vertex-ai", "url_docs": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models", "url_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "url_status": null, "url_github": null, "url_changelog": null, "url_models_api": null, "github_stars": null, "model_count": 200, "model_count_note": "Vendor states Model Garden offers \"200+ Google and third-party AI models and tools\".", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": null, "free_tier": "$300 in free credits for new Google Cloud customers; Grounding with Google Search includes 5,000 free queries per month.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Provisioned Throughput (GSU)", "amount": "$7.14 per GSU 1-week, $3.6986 1-month, $3.2877 3-month, $2.7397 1-year (global endpoint; non-global higher)" }, { "label": "Grounding with Google Search", "amount": "5,000 free queries/month, then $14 per 1,000 (Gemini 3) or $35 per 1,000 (Gemini 2.5)" }, { "label": "Web Grounding for Enterprise", "amount": "$45 per 1,000 requests" }, { "label": "Grounding with your own data", "amount": "$2.50 per 1,000 requests" }, { "label": "Tuned model endpoints", "amount": "1.5x base token price for Gemini 3 and newer" } ], "self_host_cost_note": "Managed only. Google sets its own per-token and per-modality prices rather than marking up a third party; Batch mode is 50% below interactive pricing, and Provisioned Throughput is sold in Generative AI Scale Units (GSUs) on 1-week to 1-year terms. Gemini 3.7/3.6 Flash carries introductory pricing of $0.75/1M input and $3.75/1M output through Dec 31 2026, rising to $1.50/$7.50 afterwards.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": null, "api_surfaces": [ "Vertex AI Gemini API", "OpenAI Chat Completions (OpenAI-compatible endpoint)", "Interactions API", "Live API", "Online and batch prediction" ], "sdk_note": "Gemini models can be called with the OpenAI Python/TypeScript libraries against Vertex AI's OpenAI-compatible Chat Completions endpoint.", "cap_failover": null, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": true, "cap_cache_semantic": null, "cap_observability": null, "cap_prompt_management": true, "cap_evals": true, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": null, "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": null, "eu_residency": true, "eu_residency_note": "The EU multi-region endpoint https://aiplatform.eu.rep.googleapis.com keeps ML processing inside the EU jurisdictional boundary, and ten European regions are available (europe-west1/2/3/4/6/8/9, europe-north1, europe-central2, europe-southwest1). The global endpoint does not satisfy ML-processing residency requirements, and global endpoints for partner and open models do not support data residency.", "zdr": true, "zdr_note": "Documented as a configuration discipline rather than a switch: you must act in several areas, request an exception if you are in scope for abuse-monitoring prompt logging, and explicitly set store to false. Google warns it may not be achievable with some advanced AI features.", "air_gapped": null, "best_for": "Google Cloud customers who want Gemini alongside third-party models with strong, explicitly documented EU residency and ZDR controls.", "strengths": [ "Most explicit data-governance documentation of the three: per-feature ZDR conditions and a dedicated EU multi-region endpoint", "Generative AI on Vertex AI / Gemini Enterprise Agent Platform is named in scope for SOC 2 Type II, audited quarterly", "Broad modality coverage in one platform: Gemini text, Imagen images, Veo video, Chirp audio, Lyria music, embeddings", "Grounding options span Google Search, Maps and your own data, with a ZDR-friendly enterprise variant", "Batch at 50% off plus Provisioned Throughput GSU terms from one week to one year" ], "weaknesses": [ "No cross-cloud routing — only models Google hosts are reachable, so failover to Azure or AWS needs an external gateway", "Residency and ZDR guarantees do not extend to global endpoints for partner and open-weight models", "Pricing spans tokens, modality units, GSUs, grounding calls and cache storage, so effective cost is hard to model", "Value drops sharply if your data is not already in Google Cloud, given egress and integration cost" ], "avoid_if": "You are not on Google Cloud, or you need one gateway that can fail over to models hosted by other vendors.", "reviews": [ { "url": "https://www.brahimbouine.com/blog/gcp-vertex-ai-model-serving-benchmarks/", "why": "Named engineer publishes his own Vertex AI endpoint benchmarks across T4/L4/A100/TPU and CPU machine types under production-shaped traffic.", "date": "2026-01-12", "type": "written", "title": "GCP Vertex AI Model Serving Benchmarks: Endpoint Performance Under Production Traffic", "publisher": "brahimbouine.com" }, { "url": "https://www.youtube.com/watch?v=gR4mXir0I-g", "why": "Tech Field Day session where independent delegates question a Vertex AI PM while multimodal and long-video demos are run live.", "date": "2024-06-17", "type": "video", "title": "Google Cloud Vertex AI Platform", "publisher": "YouTube - Tech Field Day" }, { "url": "https://www.reddit.com/r/googlecloud/comments/1r30jrr/why_vertex_is_so_slower_then_ai_studio/", "why": "Practitioners compare Vertex latency and 'overloaded' errors against AI Studio and the direct OpenAI API when moving to production.", "date": "2026-02-12", "type": "community", "title": "Why Vertex is so slower then AI Studio?", "publisher": "Reddit - r/googlecloud" }, { "url": "https://www.reddit.com/r/googlecloud/comments/16oha9y/do_people_actually_use_vertex_ai/", "why": "26-comment thread with concrete outcomes, including a team moving ML workloads to AWS after a year and another exporting models to edge to avoid $1k/month endpoint costs.", "date": "2023-09-21", "type": "community", "title": "Do people actually use Vertex AI?", "publisher": "Reddit - r/googlecloud" } ], "sources": { "zdr": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "name": "https://cloud.google.com/vertex-ai", "soc2": "https://cloud.google.com/security/compliance/soc-2", "company": "https://cloud.google.com/vertex-ai", "tagline": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models", "sdk_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library", "url_docs": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models", "url_home": "https://cloud.google.com/vertex-ai", "zdr_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "cap_audio": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "cap_batch": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "cap_evals": "https://cloud.google.com/vertex-ai", "cap_video": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "free_tier": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "hipaa_baa": "https://cloud.google.com/security/compliance/hipaa-compliance", "extra_fees": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "hq_country": "https://en.wikipedia.org/wiki/Google", "zdr_status": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/vertex-ai-zero-data-retention", "model_count": "https://cloud.google.com/vertex-ai", "url_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "api_surfaces": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "eu_residency": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations", "product_fate": "https://docs.cloud.google.com/vertex-ai/docs/release-notes", "regions_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations", "cap_image_gen": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "vendor_status": "https://cloud.google.com/vertex-ai/sla", "cap_embeddings": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "certifications": "https://cloud.google.com/security/compliance/soc-2", "config_as_code": "https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/vertex_ai_endpoint", "retention_days": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "sla_uptime_pct": "https://cloud.google.com/vertex-ai/sla", "logs_by_default": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "tracing_support": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging", "cap_cache_simple": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "model_count_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models", "eu_residency_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations", "last_release_date": "https://docs.cloud.google.com/vertex-ai/docs/release-notes", "openai_compatible": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library", "subprocessors_url": "https://cloud.google.com/terms/subprocessors", "vertex_ai_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "guardrails_location": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance", "requires_vendor_sdk": "https://cloud.google.com/vertex-ai/generative-ai/docs/migrate/openai/overview", "self_host_cost_note": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "cap_prompt_management": "https://cloud.google.com/vertex-ai", "data_export_available": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging", "prompt_transits_vendor": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations", "trains_on_customer_data": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/vertex-ai-zero-data-retention" }, "verified_at": { "zdr": "2026-08-29", "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "cap_batch": "2026-08-29", "cap_evals": "2026-08-29", "cap_video": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "extra_fees": "2026-08-29", "model_count": "2026-08-29", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "eu_residency": "2026-08-29", "cap_image_gen": "2026-08-29", "cap_embeddings": "2026-08-29", "config_as_code": "2026-08-29", "tracing_support": "2026-09-05", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-09-05T20:16:27.577312+00:00", "vendor_status": "hyperscaler", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "A first-party Google Cloud service with roughly weekly release notes. The SLA is tiered rather than single-figure: 99.9% for training, deployment and batch prediction, but 99.5% for custom-model online prediction and pipelines.", "last_release_date": "2026-08-23", "sla_uptime_pct": 99.9, "sla_url": "https://cloud.google.com/vertex-ai/sla", "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Models and generative features are exposed as regional endpoints plus a global endpoint, all operated by Google.", "logs_by_default": "none", "logs_note": "Request-response logging is disabled by default and can be enabled per model and project, writing to a BigQuery table you designate. Two exceptions: abuse-monitoring logging, and Gemini's in-memory caching of customer data with a 24-hour time to live.", "retention_days": 0, "retention_note": "Zero by default, with named exceptions: 30 days for Grounding with Google Search or Maps, three days for certain grounding-derived queries, and 24 hours for in-memory cache and session resumption.", "zdr_status": "yes", "trains_on_customer_data": "no", "training_note": "Under the training restriction in the service terms, Google will not use your data to train or fine-tune any AI or ML model without your prior permission, across all managed models on Vertex AI.", "subprocessors_url": "https://cloud.google.com/terms/subprocessors", "regions_note": "Around 29 regional endpoints across the Americas, Europe, Asia-Pacific and the Middle East, plus jurisdictional multi-region endpoints that keep processing inside a boundary. Google explicitly warns against the global endpoint if you have processing requirements, because you cannot know which region handles the request.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II, issued quarterly", "status": "certified" }, "fedramp": { "detail": "High P-ATO", "status": "certified" }, "gdpr_dpa": { "detail": "Cloud Terms DPA and subprocessor list", "status": "certified" }, "iso27001": { "detail": "accredited certification for the Cloud ISMS", "status": "certified" }, "hipaa_baa": { "detail": "BAAs entered as necessary, covering the whole infrastructure", "status": "available" } }, "guardrails_location": "vendor_saas", "guardrails_note": "Google may log prompts to detect abuse and policy violations as part of delivering the service, which happens on Google's side.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference, bundled routing; per-token model pricing plus separately metered grounding, caching storage and provisioned throughput. No platform fee, subscription or per-seat charge — routing layer effectively $0.", "min_commitment": "None for pay-as-you-go. Provisioned Throughput is sold in GSUs with 1-week, 1-month, 3-month and 1-year commitments. Flexible Savings Plans are spend-based committed use discounts.", "overage_note": "No log/trace retention or request-volume overage. Feature meters: grounding with Google Search $35 per 1,000 requests above 1,500/day (Gemini 2.0 Flash), Web Grounding for enterprise $45 per 1,000 requests, grounding with your data $2.50 per 1,000 prompts. Explicit context cache storage billed per token-hour ($0.0000045/token-hour for Gemini 3.1 Pro / 3 Pro / 2.5 Pro; $0.000001 for Flash/Flash-Lite).", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Implicit cached input plus explicit Context Cache (prefix-based, not semantic). Prices cached input tokens, cache hits, explicit context-cache storage, and Anthropic-style cache writes. Examples: Gemini 3.1 Pro cached input $0.20/1M vs full input; Gemini 2.5 Pro $0.125 (≤200K) / $0.25 (>200K); Claude Opus 5 cache hit $0.50/1M vs 5-min cache write $6.25/1M; Claude Sonnet 5 cache hit $0.20 promotional / $0.30 from Sep 1 2026; DeepSeek-V3.1 cache hit $0.06; GLM-5 cache hit $0.10. Gemini implicit caching has no cache-write charge beyond storage; Anthropic models on Vertex charge explicit 5-min and 1-hour cache write rates above the input rate. A single universal cached-token discount or premium percentage is not stated. Explicit caching adds a storage meter (per token-hour) and Anthropic-style writes cost more than plain input — short-lived or low-reuse caches can be a net loss.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "rate_limiting": { "note": "Not stated as a cost control on the pricing page.", "tier": null, "supported": null }, "spend_webhooks": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated on the pricing page.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated on the pricing page.", "tier": null, "supported": null } }, "cost_attribution": "Billing tags are the only stated mechanism — 'to split out Computer Use costs, apply billing tags'. Per key/user/team/customer not stated. Published cost levers: Flexible Savings Plans, Provisioned Throughput GSUs, Model Optimizer configured for cost/quality/balance.", "cost_export": "Not stated on the pricing page.", "product_kind_note": "Google describes Model Garden as \"a single place to discover, customize, and deploy a wide variety of models from Google and Google partners\" on the Gemini Enterprise Agent Platform ([Model Garden](https://cloud.google.com/model-garden)), with inference served by the Gemini API in Vertex AI (\"Use `generateContent` or `streamGenerateContent` to generate content with Gemini\") ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). It is a cloud model platform, first-party plus partner catalog.", "deployment_modes": [ "saas", "hybrid_vpc" ], "deployment_note": "Self-deployable partner models must first be purchased through Google Cloud Marketplace (\"you must first purchase them through Google Cloud Marketplace\"), via a \"Contact sales\" form and a sales representative, then deployed with \"one-click deployment\" to a Vertex AI Endpoint or with the Vertex AI SDK for Python (`model_garden.OpenModel(...).deploy(machine_type=..., accelerator_type=..., serving_container_image_uri=..., use_dedicated_endpoint=True, accept_eula=True)`); \"self-deployed models aren't serverless\" and \"only the shared public endpoint type is supported for these deployments\" ([Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09; [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11). Customers can apply existing Compute Engine reservations and committed-use discounts, but \"you cannot export the weights of self-deployed partner models\" ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "not_documented", "api_surface_note": "Native surface is proprietary: `generateContent` / `streamGenerateContent` over REST or the Google Gen AI SDK (Python `client.models.generate_content(...)`, Go `client.Models.GenerateContent(...)`), with streaming via `generate_content_stream` / `GenerateContentStream` ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). An OpenAI-library base-URL swap is documented against the `.../endpoints/openapi` path, but with gaps: unsupported parameters are silently dropped (\"If you pass any unsupported parameter, it is ignored\"), `candidateCount` (multiple candidates) is preview and does not work with `streamGenerateContent`, and for self-deployed Model Garden containers \"not every model deployed to these containers supports the Chat Completions API\" ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).", "model_count_low": 200, "model_count_high": 200, "model_count_source_note": "\"200+ available models\" ([Model Garden](https://cloud.google.com/model-garden), undated).", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party platform plus partner catalog; no provider total is published. Sources are \"models from Google and Google partners\", with named third parties including Anthropic, Meta (Llama), Mistral AI, AI21 and TII ([Model Garden](https://cloud.google.com/model-garden)).", "first_party_vs_partner": "Documented mix: Google first-party foundation models (Gemini, Imagen, Veo, Chirp), open models (Gemma, CodeGemma, PaliGemma, Llama, Mistral, Falcon, BERT, T5-FLAN, ViT, EfficientNet), and third-party partner models such as \"Anthropic's Claude Model Family\"; partner models can be managed (MaaS) or self-deployed after a Marketplace license purchase ([Model Garden](https://cloud.google.com/model-garden); [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09).", "custom_endpoint_support": "Yes within Google Cloud: Model Garden lets customers \"self-deploy and serve open, partner, and custom models\", including deploying \"models with custom weights\" onto Vertex AI Endpoints using prebuilt vLLM or HF TGI containers; registering an arbitrary external private URL as a model is not documented ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).", "byok_mode": "not_applicable", "merchant_of_record": "Google Cloud: usage is billed to the Google Cloud account, charged \"only for requests that return a 200 response code\" and metered in tokens, cached tokens, images, video, audio, characters, requests, embeddings, node-hours and similar units; for self-deployed partner models \"you are billed for both the model usage and the underlying Gemini Enterprise Agent Platform infrastructure consumed\", with partner licenses obtained through Cloud Marketplace ([Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing); [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11). New customers get \"up to $300 in free credits\" ([Model Garden](https://cloud.google.com/model-garden)).", "key_management_note": "Authentication is Google Cloud-native (project/location initialization, e.g. `vertexai.init(project=..., location=...)`, and the OpenAI-compat page defers to \"authentication and credentialing with the OpenAI-compatible syntax\"), so no third-party provider secrets are stored by the platform ([Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09; [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29). Self-deployed models run inside the customer's own project and VPC network, keeping serving traffic within customer-controlled infrastructure ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09). CMEK/KMS specifics and vendor visibility of request content: n.a. on the pages fetched.", "api_openai_chat_completions_note": "Yes: an OpenAI-compatible surface at base URL `https://{location}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi`, called with `client.chat.completions.create(...)` and models such as `google/gemini-2.0-flash-001` ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), last updated 2026-06-29).", "api_anthropic_messages_note": "n.a. on the pages fetched (Claude is offered as a partner model, but no Anthropic Messages endpoint is documented here) ([Model Garden](https://cloud.google.com/model-garden)).", "api_openai_responses_note": "n.a.", "api_embeddings_note": "Yes as a billed capability/unit (\"embedding\" units are listed among the charged units) ([Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing)); a specific embeddings endpoint path is not on the pages fetched: n.a.", "api_images_note": "Yes: first-party text-to-image with \"Imagen and Gemini 3 Pro Image for text-to-image\" and \"Veo for text-to-video and image-to-video\", with image-generation and video-generation billing units ([Model Garden](https://cloud.google.com/model-garden); [Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing)). Endpoint paths: n.a.", "api_audio_note": "Yes, both directions as products: \"Chirp for speech-to-text\" and \"Text-to-speech\" are listed pre-trained APIs ([Model Garden](https://cloud.google.com/model-garden)); Gemini also accepts audio input MIME types (`audio/mpeg`, `audio/mp3`, `audio/wav`) with preview `audioTimestamp` support ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). Endpoint paths: n.a.", "api_batch_note": "n.a. on the pages fetched.", "deployment_modes_note": "hosted cloud service (managed/MaaS models that \"don't require manual deployment\") and customer-project self-deployment, where self-deployed models \"run securely within your Google Cloud project and VPC network\" on chosen on-demand hardware ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), last updated 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), last updated 2025-12-09). Self-host outside Google Cloud, on-prem, air-gapped: n.a.", "byok_mode_note": "not_applicable: Google hosts the models and access is authenticated with Google Cloud project credentials; there are no upstream provider keys to supply ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).", "access_meta": { "qualified": [ "api_anthropic_messages", "api_embeddings", "api_audio", "api_batch" ], "model_count_kind": "vendor_total", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` on the fetched Vertex pages ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented` as a service setting. Google's guidance is client-side: \"Implement a retry strategy by using truncated exponential backoff.\" Default count: `n.a.` ([Vertex AI error code 429](https://cloud.google.com/vertex-ai/generative-ai/docs/error-code-429)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "No model/provider fallback list ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_configurable", "rel_health_check_note": "`not_configurable`; the closest documented behavior is automatic capacity-aware rerouting on the global endpoint: \"it dynamically routes your request to a region with available capacity\" and if a cached region is at capacity \"the system will automatically try the next available region to serve the request\" ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).", "rel_multi_region_config": "not_configurable", "rel_multi_region_note": "`not_configurable` beyond endpoint choice, and Google runs the routing. Setting `location` to `GLOBAL` selects a global entry point that \"does not tie your request to a single region\" and is recommended when \"architecting for maximum resilience against regional disruptions\", with the caveat that it \"does not guarantee that requests will be processed in any specific location\". Multi-region endpoints keep processing inside a jurisdiction: `us` → `https://aiplatform.us.rep.googleapis.com`, `eu` → `https://aiplatform.eu.rep.googleapis.com` ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai), [Vertex AI deployments and endpoints](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)).", "rel_load_balancing_config": "not_configurable", "rel_load_balancing_note": "Region selection by available capacity is automatic, with \"a new, separate global quota\" for the global endpoint; no weights or policy keys ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).", "architecture_class": "hyperscaler_api", "architecture_note": "Managed Google Cloud service consumed via `google-genai` / `vertexai` SDKs (Python, Go) or REST; no runtime disclosed ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).", "self_host_data_plane": "no", "self_host_artifact_note": "`no` ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).", "latency_claims": [], "throughput_claims": [], "streaming_support": "yes", "streaming_note": "Supported: `streamGenerateContent`, `client.models.generate_content_stream` (Python), `client.Models.GenerateContentStream` (Go), and `stream=True` through the OpenAI-compatible surface. Caveat: multiple candidates (`candidateCount`) is a Preview feature that works with `generateContent` but \"streamGenerateContent is not supported\" ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).", "guard_pii": "sync_block", "guard_pii_note": "Two layers. Non-configurable Vertex safety filters block CSAM and personally identifiable information, surfacing an SPII finish reason (`sync_block`) ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)). Model Armor uses Sensitive Data Protection templates: with `INSPECT_ONLY` it flags (`sync_observe`), and with `INSPECT_AND_BLOCK` it \"issues a block verdict\" (`sync_block`). Notably, \"Model Armor doesn't pass the de-identified data—such as masked, redacted, or hashed content—back\" to the platform, so de-identification is not a pass-through redaction ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))", "guard_injection": "sync_block", "guard_injection_note": "Vertex's JAILBREAK classifier is available but **off by default** ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)); Model Armor's `pi_and_jailbreak_filter_settings` with a confidence level \"blocks the prompt or response\" when enforcement is `INSPECT_AND_BLOCK`, and the blocked response carries `blockReason: MODEL_ARMOR` ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))", "guard_content": "sync_block", "guard_content_note": "Configurable harm categories block at chosen thresholds ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)); Model Armor Responsible AI filters cover `HATE_SPEECH`, `HARASSMENT`, `DANGEROUS`, `SEXUALLY_EXPLICIT` with confidence levels ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))", "guard_custom": "sync_block", "guard_custom_note": "Policy takes the form of Model Armor templates (filter settings and confidence levels) plus project- or org-level **floor settings** that impose a minimum; document and image screening and multi-language detection are template options ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages — the Model Garden availability page states nothing about access restriction ([Available models](https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models))", "guard_execution_location": "vendor_saas", "guard_third_party": [ "Google Model Armor", "Sensitive Data Protection" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` — the docs specify enforcement modes (`INSPECT_ONLY` vs `INSPECT_AND_BLOCK`, floor settings defaulting to `INSPECT_ONLY` on the gcloud path) but not what happens if Model Armor is unreachable ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))", "log_content": "nothing", "log_content_note": "Request-response logging must be enabled explicitly; when on, `full_request` and `full_response` (i.e. prompt and completion text) are written to BigQuery, with an `enableOtelLogging` option ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "log_opt_out": "yes", "log_opt_out_note": "`set_request_response_logging_config(enabled=False)` disables it; Cloud Monitoring metrics are unaffected ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "tracing_support": "otel_partial", "tracing_note": "Partially — `enableOtelLogging` is documented as an option on request-response logging; a full OTel agent-trace model is not described on the fetched pages ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "export_destinations": [ "Cloud Logging", "Cloud Monitoring", "BigQuery" ], "export_destinations_note": "BigQuery tables for request-response logs ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging)); Cloud Logging for Model Armor verdicts via `enableCloudLogging` ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` on the fetched pages; logged traffic lands in BigQuery, where the customer can run their own scoring ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "observability_requires_saas": null, "log_retention_note": "Governed by the destination BigQuery dataset's own table/partition expiration; no Vertex-side default stated ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))", "production_meta": { "sources": {}, "qualified": [ "guard_injection" ], "latency_prose": "`n.a.` Only qualitative statements (\"fast responses worldwide\"; prompt caching gives \"lower latency and lower costs\"; cached requests routed \"for the lowest latency\") ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).", "multi_surface": {}, "throughput_prose": "`n.a.` Throughput is a reservation product: Provisioned Throughput reserves capacity for specific models ([Vertex AI error code 429](https://cloud.google.com/vertex-ai/generative-ai/docs/error-code-429)).", "benchmark_independence": "No figures published ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai))." }, "guard_pii_default": "sync_block", "guard_injection_default": "off_by_default", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Resilience and data residency pull in opposite directions here: the global endpoint is the resilience play, regional/multi-region endpoints are the residency play, and Google's own recommendation is global for primary traffic with regional as a secondary path. Constraints to plan around: the Claude global endpoint supports pay-as-you-go only, Provisioned Throughput is regional-endpoint only, you should not send the same request to both endpoints simultaneously, and Private Google Access is not supported for multi-region endpoints (use Private Service Connect) ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai), [Vertex AI deployments and endpoints](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)).", "guard_notes": "Floor settings are the strongest governance primitive here: an organisation can force a minimum Model Armor enforcement for every project so a team cannot dial its own template down ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration)). Sanitising prompts or responses containing documents is not supported in the Vertex integration ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration)).", "fit_agent_features_note": "Function and tool calling are documented for Vertex integrations, including OpenAI `tools` with `tool_choice=\"auto\"`, Vercel AI SDK tools such as `googleVertex.tools.codeExecution`, and Agent Platform classes including `FunctionDeclaration`, `Tool`, and `AutomaticFunctionCallingResponder`. Google documentation also mentions ADK agents, multi-agent systems, A2A communication, and Agent Engine deployment. ([Agent Platform Python SDK reference](https://docs.cloud.google.com/python/docs/reference/agentplatform/latest))", "fit_byok_required": "no", "fit_byok_required_note": "No. Authentication is a Google express-mode API key or ADC ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))", "fit_client_sdk_langs": [ "Python", "Go", "Java", "Node.js" ], "fit_client_sdks_note": "Documented client libraries include the OpenAI library for Python and TypeScript/JavaScript, REST API, Vercel’s `@ai-sdk/google-vertex`, LangChain’s `langchain-google-vertexai`, Google’s `google-cloud-aiplatform` and `google-genai` packages, and LlamaIndex’s Vertex integration. The Vertex AI SDK is documented for Python, Go, Java, and Node.js. ([Vertex AI SDK language support](https://docs.cloud.google.com/gemini-enterprise-agent-platform/machine-learning/python-sdk/use-vertex-ai-sdk))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Google Cloud authentication is documented through Application Default Credentials and service accounts. The OpenAI integration uses `google.auth.default(scopes=[\"https://www.googleapis.com/auth/cloud-platform\"])`, refreshes credentials with `google.auth.transport.requests.Request()`, and passes `credentials.token` as `api_key`; the Vercel provider also documents `google-auth-library`, `GOOGLE_APPLICATION_CREDENTIALS`, and Edge credentials such as `GOOGLE_CLIENT_EMAIL` and `GOOGLE_PRIVATE_KEY`. ([Vertex AI OpenAI compatibility guide](https://cloud.google.com/vertex-ai/generative-ai/docs/start/openai?hl=es))", "fit_cloudflare_workers": "user_facing", "fit_cloudflare_workers_note": "Cloudflare Workers are explicitly cited as a supported Edge runtime for the Vercel AI SDK Google Vertex provider through `@ai-sdk/google-vertex/edge`. ([Vercel AI SDK Google Vertex provider](https://ai-sdk.dev/providers/ai-sdk-providers/google-vertex))", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented) — no credit-card, free-tier or trial-credit wording; billing enablement is required for existing (non-express) users ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start), [Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))", "fit_ecosystem_notes": "Vertex AI documentation covers Gemini models, Anthropic Claude partner models, xAI Grok partner models, Vertex MaaS open models, Model Garden, Vector Search, Agent Engine, and Google Cloud MCP services. The OpenAI-library integration is in preview and uses one-hour access tokens, while newer documentation directs users toward the Google Gen AI SDK; several older Vertex AI SDK modules are deprecated as of June 24, 2025 and scheduled for removal on June 24, 2026. ([Agent Platform Python SDK reference](https://docs.cloud.google.com/python/docs/reference/agentplatform/latest))", "fit_eval_prereqs": "A valid `@gmail.com` Google Account; new users sign up for express mode, obtain an express-mode API key and enable the Vertex AI API; existing users need a Google Cloud project, billing enabled, the Vertex AI API enabled, and ADC or an API key bound to a service account, plus the installed gcloud CLI; roles: `roles/aiplatform.user`, and possibly `roles/resourcemanager.projectCreator` and `roles/serviceusage.serviceUsageAdmin` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `client = genai.Client(http_options=HttpOptions(api_version=\"v1\"))` then `client.models.generate_content(model=\"gemini-2.5-flash\", contents=\"How does AI work?\")`; a REST `curl -X POST ...:generateContent` form is also shown ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start)). The Studio quickstart shows no request snippet ([Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))", "fit_gateway_platform": "platform_gateway", "fit_gateway_platform_note": "Google Cloud documents the GKE Inference Gateway API for ingress traffic, fine-grained routing, advanced load balancing, and centralized policy attachment. ([GKE inference workload deployment](https://docs.cloud.google.com/application-design-center/docs/ai-pretrained-inference-gke-cluster-workload))", "fit_helm_chart": "GKE documentation only; no Vertex AI chart", "fit_kubernetes": "generic_platform_only", "fit_kubernetes_note": "No Vertex-AI-specific Helm chart or operator is documented; Google documents generic Helm delivery to GKE (Artifact Registry Helm repositories, sample `hello-chart`, cluster `chart-cluster`) and a GKE inference example serving `google/gemma-7b-it` with the `vllm/vllm-openai` image. ([Google Artifact Registry Helm charts docs](https://docs.cloud.google.com/artifact-registry/docs/helm/store-helm-charts))", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Google Cloud documents remote MCP servers for using Google products and services in AI agents and applications. Google’s MCP Toolbox for Databases is documented as an open-source MCP server, including deployment alongside a Vertex AI Agent Engine workflow; Vertex-specific MCP gateway, registry, or tool-proxy functionality is not stated. ([Google Cloud MCP documentation](https://docs.cloud.google.com/mcp))", "fit_meta": { "not_documented_fields": [ "fit_credit_card" ] }, "fit_model_gate": "enablement_step", "fit_model_gate_note": "Yes, an API enablement step: \"Enable the Vertex AI API in the console.\" ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))", "fit_onboarding_notes": "Environment setup requires `GOOGLE_CLOUD_PROJECT`, `GOOGLE_CLOUD_LOCATION=global`, `GOOGLE_GENAI_USE_VERTEXAI=True`, plus `gcloud init` and `gcloud auth application-default login`. Express mode is the lower-effort branch for new users. The console-only path (Prompt Gallery → Submit) produces output without any code, and generated Python can be opened in a Colab Enterprise notebook; the default model there is `gemini-3.1-pro-preview` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start), [Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Vertex AI documents using the OpenAI Python client with `api_key=credentials.token` and `base_url`. The documented base URLs are `https://{location}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi` and, in the Chinese page, `https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi`. ([Vertex AI OpenAI compatibility guide](https://cloud.google.com/vertex-ai/generative-ai/docs/start/openai?hl=es))", "fit_prod_prereqs": "A Google Cloud project with billing enabled, the Vertex AI API enabled, ADC or a service-account-bound API key, and the Vertex AI User role `roles/aiplatform.user` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "cloud_console", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "LangChain documents Vertex AI integrations through `langchain-google-vertexai`, including `VertexAI`, `ChatVertexAI`, `VertexAIModelGarden`, `GemmaVertexAIModelGarden`, and `GemmaChatVertexAIModelGarden`; the integration page is marked deprecated and directs users to `GoogleGenerativeAI`. LlamaIndex also documents a Vertex AI integration and says Google GenAI, installed with `google-genai`, provides the same Vertex functionality. ([LangChain Google Vertex AI integration](https://docs.langchain.com/oss/python/integrations/llms/google_vertex_ai))", "fit_quickstart_steps": 15, "fit_quickstart_steps_caveat": "15 on the API quickstart. A separate console-only Vertex AI Studio quickstart has 12 steps and needs no code.", "fit_quickstart_url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "platform_provider_resources", "fit_terraform_note": "The HashiCorp Google provider documents Vertex AI Terraform resources including `google_vertex_ai_index_endpoint`, `google_vertex_ai_index`, and `google_vertex_ai_dataset`; examples also use the `google-beta` provider. ([Terraform Google Vertex AI index endpoint](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/vertex_ai_index_endpoint))", "fit_terraform_ref": "hashicorp/google: google_vertex_ai_index_endpoint, google_vertex_ai_index, google_vertex_ai_dataset", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "The official Vercel AI SDK provider package is `@ai-sdk/google-vertex`. Documented submodules include `@ai-sdk/google-vertex/edge`, `/anthropic`, `/xai`, and `/maas`. ([Vercel AI SDK Google Vertex provider](https://ai-sdk.dev/providers/ai-sdk-providers/google-vertex))", "fit_vercel_ai_sdk_package": "@ai-sdk/google-vertex" }, { "slug": "truefoundry-ai-gateway", "name": "TrueFoundry AI Gateway", "company": "TrueFoundry", "tagline": "Closed-source enterprise AI gateway sold on request tiers, deployable as SaaS or inside the customer's own cloud.", "category": "cloud_platform", "deployment": "both", "license": "Proprietary", "founded_year": 2021, "hq_country": "India", "url_home": "https://www.truefoundry.com/ai-gateway", "url_docs": "https://docs.truefoundry.com/", "url_pricing": "https://www.truefoundry.com/pricing", "url_status": "https://status.truefoundry.com", "url_github": null, "url_changelog": null, "url_models_api": null, "github_stars": null, "model_count": 1600, "model_count_note": "Vendor-stated 1,600+ models (chat, completion, embedding, reranking) on the AI Gateway page; other vendor pages cite 250+ LLMs.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": true, "byok_fee_note": "You connect your own provider accounts and keys; TrueFoundry bills by request tier, not tokens.", "seat_fee_usd_month": null, "free_tier": "Developer plan $0/month: 50,000 requests/month, 3 users, 50,000 MCP tool calls, up to 10 saved prompts.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Pro plan", "amount": "$499/month (1M requests, 10 users)" }, { "label": "Pro Plus plan", "amount": "$2,999/month (1M requests, 25 users)" }, { "label": "Request/key overage", "amount": "$499/month per additional 2M requests and 5 API keys" } ], "self_host_cost_note": "No open-source edition; the data plane is proprietary. Vendor estimates roughly $600-$1,000/month of your own hosting cost to run the gateway (or control plus gateway) plane in your cloud, on top of the plan fee; fully managed SaaS carries no hosting cost. VPC, on-prem, air-gapped and multi-cloud installs are enterprise deployments, and a third-party review notes Helm-based ops effort is required.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": true, "portability_note": "Export is requested by email and returned as JSON, not self-serve.", "api_surfaces": [ "OpenAI Chat Completions", "OpenAI Embeddings", "Rerank", "MCP" ], "sdk_note": "Unified OpenAI-compatible endpoint plus a playground and virtual models; MCP servers are proxied through the same gateway.", "cap_failover": true, "cap_load_balancing": true, "cap_conditional_routing": true, "cap_cache_simple": true, "cap_cache_semantic": true, "cap_observability": true, "cap_prompt_management": true, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": true, "cap_budgets": true, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": true, "cap_embeddings": true, "cap_image_gen": null, "cap_audio": null, "cap_video": null, "cap_batch": null, "latency_overhead_ms": 3, "latency_source_note": "Vendor claim (TrueFoundry): \"sub-3 ms internal latency\" with 99.99% uptime and 10B+ requests/month. An independent review (llmtools.cc) records the same figure as vendor-claimed ~3-4 ms with 350+ RPS on a single vCPU and states it is not independently verified; no third-party benchmark was found.", "throughput_rps": null, "soc2": true, "hipaa_baa": null, "gdpr": true, "eu_residency": null, "eu_residency_note": "Deployment inside the customer's own cloud region is offered (\"no data leaves your domain\"), but no named EU residency commitment is published.", "zdr": null, "zdr_note": "No ZDR commitment. The equivalent is disabling request logging, or keeping the gateway in your own infrastructure.", "air_gapped": true, "best_for": "Enterprises that want a fully managed or in-VPC AI gateway with guardrails, MCP governance and SSO/RBAC, and are comfortable with closed source.", "strengths": [ "Broad routing feature set: weight, latency and priority routing, fallback chains, geo-aware routing and virtual models", "Both simple and semantic caching plus budget and rate-limit controls per team, model and key", "Deployable in the customer's VPC, on-prem, air-gapped or multi-cloud with SOC 2 and HIPAA/GDPR-ready configurations", "Detailed built-in observability (P50/P90/P99, TTFT, traces, alerts) and MCP gateway with RBAC" ], "weaknesses": [ "Not open source: the data plane is proprietary, so there is no free self-host path or community audit", "Steep jump from the $0 Developer tier to $499/month Pro, with overage billed in $499 blocks", "Performance figures are vendor claims only; a third-party review explicitly flags them as unverified", "Self-hosting adds an estimated $600-$1,000/month of infra plus Helm/Kubernetes operational burden" ], "avoid_if": "You require an open-source, auditable gateway or want to avoid per-request pricing tiers as volume grows.", "reviews": [ { "url": "https://www.reddit.com/r/LLMDevs/comments/1uz24rc/one_ai_gateway_for_a_handful_of_internal_teams/", "why": "15-comment selection thread weighing TrueFoundry against LiteLLM, Portkey, Kong and Databricks' gateway for multi-team internal use, including budgeting surprises.", "date": "2026-07-17", "type": "community", "title": "One AI gateway for a handful of internal teams, where did you land?", "publisher": "Reddit - r/LLMDevs" } ], "sources": { "gdpr": "https://www.truefoundry.com/ai-gateway", "soc2": "https://www.truefoundry.com/ai-gateway", "cap_mcp": "https://www.truefoundry.com/ai-gateway", "license": "https://llmtools.cc/tool/truefoundry/", "sdk_note": "https://www.truefoundry.com/ai-gateway", "url_docs": "https://docs.truefoundry.com/", "free_tier": "https://www.truefoundry.com/pricing", "air_gapped": "https://www.truefoundry.com/ai-gateway", "extra_fees": "https://www.truefoundry.com/pricing", "hq_country": "https://www.f6s.com/company/truefoundry", "url_status": "https://status.truefoundry.com", "weaknesses": "https://llmtools.cc/tool/truefoundry/", "cap_budgets": "https://www.truefoundry.com/ai-gateway", "model_count": "https://www.truefoundry.com/ai-gateway", "api_surfaces": "https://www.truefoundry.com/ai-gateway", "cap_failover": "https://www.truefoundry.com/ai-gateway", "founded_year": "https://www.f6s.com/company/truefoundry", "product_fate": "https://siliconangle.com/2026/06/25/truefoundry-acquires-mlops-pioneer-seldon-ai-accelerate-enterprise-agentic-ai/", "regions_note": "https://www.truefoundry.com/docs/platform/security-and-compliance", "vendor_status": "https://www.truefoundry.com/blog/announcing-our-19m-series-a-scaling-ai-deployment-with-autonomous-agents-on-autopilot", "byok_supported": "https://www.truefoundry.com/ai-gateway", "cap_embeddings": "https://www.truefoundry.com/ai-gateway", "cap_guardrails": "https://www.truefoundry.com/ai-gateway", "certifications": "https://www.truefoundry.com/docs/platform/security-and-compliance", "config_as_code": "https://www.truefoundry.com/docs/ai-gateway/virtual-model", "retention_days": "https://www.truefoundry.com/docs/platform/security-and-compliance", "sla_uptime_pct": "https://www.truefoundry.com/service-level-agreement", "cap_rate_limits": "https://www.truefoundry.com/ai-gateway", "logs_by_default": "https://www.truefoundry.com/docs/ai-gateway/request-logging", "cap_cache_simple": "https://www.truefoundry.com/ai-gateway", "model_count_note": "https://www.truefoundry.com/ai-gateway", "cap_observability": "https://www.truefoundry.com/ai-gateway", "cap_pii_redaction": "https://www.truefoundry.com/ai-gateway", "eu_residency_note": "https://www.truefoundry.com/ai-gateway", "last_release_date": "https://www.truefoundry.com/docs/changelog", "openai_compatible": "https://www.truefoundry.com/ai-gateway", "subprocessors_url": "https://www.truefoundry.com/docs/platform/security-and-compliance", "cap_cache_semantic": "https://www.truefoundry.com/ai-gateway", "cap_load_balancing": "https://www.truefoundry.com/ai-gateway", "guardrails_location": "https://www.truefoundry.com/docs/platform/security-and-compliance", "latency_overhead_ms": "https://www.truefoundry.com/ai-gateway", "latency_source_note": "https://llmtools.cc/tool/truefoundry/", "requires_vendor_sdk": "https://www.truefoundry.com/docs/ai-gateway/quick-start", "self_host_cost_note": "https://www.truefoundry.com/pricing", "truefoundry_pricing": "https://www.truefoundry.com/pricing", "cap_prompt_management": "https://www.truefoundry.com/ai-gateway", "data_export_available": "https://docs.truefoundry.com/ai-gateway/export-logstraces", "prompt_transits_vendor": "https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment", "cap_conditional_routing": "https://www.truefoundry.com/ai-gateway" }, "verified_at": { "gdpr": "2026-08-29", "soc2": "2026-08-29", "cap_mcp": "2026-08-29", "license": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "free_tier": "2026-08-29", "air_gapped": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "weaknesses": "2026-08-29", "cap_budgets": "2026-08-29", "model_count": "2026-08-29", "api_surfaces": "2026-08-29", "cap_failover": "2026-08-29", "founded_year": "2026-08-29", "byok_supported": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "cap_rate_limits": "2026-08-29", "cap_cache_simple": "2026-08-29", "model_count_note": "2026-08-29", "cap_observability": "2026-08-29", "cap_pii_redaction": "2026-08-29", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "cap_cache_semantic": "2026-08-29", "cap_load_balancing": "2026-08-29", "latency_overhead_ms": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "cap_prompt_management": "2026-08-29", "data_export_available": "2026-08-29", "cap_conditional_routing": "2026-08-29" }, "locked_fields": [], "published": true, "updated_at": "2026-08-31T04:37:47.93379+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed and expanding. Raised a $19M Series A led by Intel Capital in January 2026 and was itself the acquirer of MLOps vendor Seldon Technologies in June 2026. Note that the uptime commitment applies to Enterprise customers only; other plans get best-effort availability.", "last_release_date": "2026-08-21", "sla_uptime_pct": 99.9, "sla_url": "https://www.truefoundry.com/service-level-agreement", "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "depends", "prompt_path_note": "Three genuinely different data paths: managed SaaS where request and response data flows through TrueFoundry's control plane; gateway-in-your-infrastructure where all LLM traffic stays with you and TrueFoundry is not in the live request path; and fully self-hosted, where everything except the authentication and analytics servers runs on your infrastructure.", "logs_by_default": "full_content", "logs_note": "On hosted deployments logging happens unless you suppress it: absent or true in the logging-config header means logging occurs. You can control whether bodies are logged and redact patterns, and a deny rule always wins.", "retention_days": 90, "retention_note": "Ninety days for SaaS blob storage, configurable. In self-hosted modes the control plane does no long-term retention.", "zdr_status": "not_published", "trains_on_customer_data": "not_published", "training_note": "Neither the security page nor the deployment docs address training.", "subprocessors_url": "https://www.truefoundry.com/docs/platform/security-and-compliance", "regions_note": "You choose the region where the gateway is deployed and where logs and traces are stored, and you can bring your own S3, Azure Blob or GCS bucket. Gateway planes run active-active across multiple regions.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II programme for TrueFoundry SaaS", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "standard DPAs available", "status": "available" }, "iso27001": { "detail": "asserted in deployment docs but absent from the security page's programme list", "status": "claimed" }, "hipaa_baa": { "detail": "HIPAA programme maintained", "status": "certified" } }, "guardrails_location": "both", "guardrails_note": "The gateway plane evaluates PII, prompt-injection and content-filtering policy in memory on the request path — and in self-hosted modes that plane is yours.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "platform_fee", "pricing_model_note": "Free developer tier + flat monthly platform fee + per-unit usage overage + enterprise quote + optional self-host infra cost. Developer $0/mo, Pro $499/mo, Pro Plus $2,999/mo, Enterprise custom. TrueFoundry does not resell inference.", "min_commitment": "None stated. Enterprise implied custom.", "overage_note": "Request overage priced in blocks: 2M requests and 5 API keys for additional $499/month; Pro Plus 'contact sales'. Self-hosting adds ~$600–$1,000/month of infrastructure cost when you self-host the Gateway or both the Control and Gateway planes. Log/trace retention overage and export cost not stated.", "cache_support": "hybrid", "cache_read_discount_pct": null, "cache_write_premium_pct": null, "cache_pricing_note": "Exact ('Simple caching') and semantic caching both listed as features; provider passthrough not stated. No cached-token pricing published. TrueFoundry charges per request/platform, not per token, so caching cuts your provider bill and you keep the saving — but cached requests presumably still consume the tier's request allowance, and the feature list does not mark which tier includes semantic caching.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "'Alerts' listed, tier unstated.", "tier": null, "supported": true }, "virtual_keys": { "note": "'Virtual models' are listed; per-tier API key counts (5 extra keys bundled with the $499 overage pack). Not branded as virtual keys explicitly.", "tier": null, "supported": true }, "rate_limiting": { "note": "Listed, tier unstated.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "'RBAC on models' and 'RBAC on MCPs' are listed — allowlisting by role rather than a named allowlist.", "tier": null, "supported": true }, "budget_caps_per_key": { "note": "Not stated as per-key.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "'Budget limiting' listed without scope or tier.", "tier": null, "supported": true } }, "cost_attribution": "'Cost per team/user/model/application' plus custom metadata and metadata filtering. Per-key, per-tag and per-customer not stated.", "cost_export": "'Export to custom storage buckets', 'Data Lake Export', 'Connect multiple storage bucket' and 'Export to other monitoring platforms'. CSV, cost API and webhook export not stated; export pricing not stated.", "product_kind_note": "TrueFoundry calls it the \"TrueFoundry AI Gateway\", \"the proxy layer that sits between your applications and the LLM providers and MCP Servers\", and \"an enterprise-grade platform that enables users to access 1000+ LLMs using a unified interface while taking care of observability and governance\", offering \"one endpoint with an OpenAI-compatible schema for every provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). It is a routing proxy with an observability/governance plane.", "deployment_modes": [ "saas", "self_host", "hybrid_vpc", "on_prem" ], "deployment_note": "Four documented topologies with tier and cost: (1) \"AI Gateway SAAS only\" — Starter tier, $0 hosting, \"you don't need to deploy any infrastructure on your end\"; (2) \"SaaS AI Gateway with data storage on your own infrastructure\" — Starter, cost = \"storage cost in S3\"; (3) \"Gateway Plane only\" with TrueFoundry-hosted control plane — Enterprise, \"~$600/month\" hosting; (4) \"Control Plane + Gateway Plane\" on customer infrastructure — Enterprise, \"~$800-1000/month\" ([Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment)). Docker/Helm/Terraform specifics are not mentioned on the pages fetched: n.a. ([Self-hosted gateway](https://www.truefoundry.com/docs/ai-gateway/gateway-self-hosted)). Platform tiers are Developer $0, Pro $499/month, Pro Plus $2,999/month, Enterprise custom ([TrueFoundry pricing](https://www.truefoundry.com/pricing)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "yes", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap with the OpenAI SDK (`base_url=\"{GATEWAY_BASE_URL}\"`, or `OPENAI_BASE_URL`/`OPENAI_API_KEY` env vars), with ready snippets for OpenAI SDK, LangChain, LangGraph, LlamaIndex, Node.js, Google ADK and cURL; you need the Gateway Base URL, API key and Model ID ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)). The surface is broad: also `/rerank`, `/moderations`, `/files`, fine-tune, a Compaction API and `/proxy` for provider-native passthrough ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Streaming is marked supported for all listed chat providers (OpenAI, Azure OpenAI, Anthropic, Bedrock, Vertex, Cohere, Gemini, Groq, AI21, Cerebras, Wafer, SambaNova, Perplexity AI, Together AI, xAI, DeepInfra) with no documented caveats ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "model_count_low": 1000, "model_count_high": 1000, "model_count_source_note": "\"1000+ LLMs\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway), undated).", "provider_count_low": 27, "provider_count_high": 27, "provider_count_source_note": "n.a. as a stated total; the intro page enumerates 27 provider integrations (Gemini & Vertex AI, Google Gemini, AWS Bedrock, AWS SageMaker, Azure OpenAI, Azure AI Foundry, OpenAI, Cohere, Databricks, AI21, Anthropic, Together AI, xAI, DeepInfra, Perplexity AI, Mistral AI, Cloudera, Groq, ElevenLabs, Deepgram, Cartesia, Smallest AI, Snowflake Cortex, Self Hosted, OpenRouter, SambaNova, Cerebras) and adds \"if you don't see the provider you need, there is a high chance it will just work as self-hosted models or OpenAI provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway), undated).", "first_party_vs_partner": "All third-party or customer-run: TrueFoundry proxies provider APIs and customer-registered self-hosted models; the intro page does not claim any TrueFoundry-owned model hardware ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway); [Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)).", "custom_endpoint_support": "Yes: \"you can easily integrate any OpenAI-compatible self-hosted model\" by supplying \"the endpoint URL\" and optional \"Auth Data which will be used for Authentication of request to the model\"; documented targets include vLLM, Ollama, SGLang and Text Generation Inference running on \"your own infrastructure\", \"on-premises GPUs\", \"private cloud instances\", or \"any model server not deployed through TrueFoundry\", and registered models appear in the catalog next to cloud providers ([Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)).", "byok_mode": "byok_only", "merchant_of_record": "n.a. as an explicit statement about model usage; TrueFoundry invoices its platform tiers (Developer $0, Pro $499/month, Pro Plus $2,999/month, Enterprise custom, plus \"additional usage at transparent, per-unit rates\") while model calls run on the customer's own provider accounts ([TrueFoundry pricing](https://www.truefoundry.com/pricing); [Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)).", "key_management_note": "Upstream provider keys are held per Model Account, with separate accounts used to hold multiple keys for the same provider; self-hosted model registrations store optional \"Auth Data\" for upstream auth ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)). Caller auth uses a Personal Access Token (for development) or Virtual Account Token (for applications) created in the platform's Access section, and the gateway offers \"RBAC and scoped keys for users, teams, and applications\" plus spend limits and per-user/per-model/per-application throttles ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Data handling varies by topology: on SaaS \"data is encrypted at rest and in transit\" on infrastructure that is \"SOC2, ISO27001, GDPR, and HIPAA compliant\"; with customer-managed storage, request-response data still flows through the TrueFoundry control plane, which \"might cache some of the data for faster queries, but doesn't do any long term retention of the data\" and requires bucket access (\"you will not be able to use this feature if you don't give access to TrueFoundry control plane access to your bucket\"); fully self-hosted keeps both planes on customer infrastructure ([Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "api_openai_chat_completions_note": "Yes: OpenAI-schema chat completions through the gateway base URL (`https://gateway.truefoundry.ai` for SaaS; self-hosted URLs come from the Playground code snippet), used with `OpenAI(api_key=..., base_url=\"{GATEWAY_BASE_URL}\")` ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "api_anthropic_messages_note": "Yes as SDK compatibility: \"drop-in support for OpenAI, Anthropic, and other provider SDKs\", plus a `/proxy` API that can \"forward provider-native requests through the AI Gateway while keeping logging, rate limiting, and budget controls\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Explicit `/v1/messages` path: n.a.", "api_openai_responses_note": "n.a. (not documented on the pages fetched).", "api_embeddings_note": "Yes: \"Embedding (`/embeddings`)\", with a linked Embeddings API reference ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "api_images_note": "Yes: \"Image Generation (`/images/generations`)\", \"Image Edit (`/images/edits`)\", \"Image Variation (`/images/variations`)\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "api_audio_note": "Yes, both directions: Text To Speech and Speech to Text are documented, plus Audio Translation at `/audio/translations` and a Live/Realtime API ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "api_batch_note": "Yes: \"Batch (`/batches`)\" — \"run large workloads asynchronously at batch pricing\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).", "deployment_modes_note": "hosted SaaS (\"You can run the AI Gateway as fully managed SaaS… hosted in multiple regions across the world\"), self-host, hybrid/customer-cloud VPC, and on-prem: \"SaaS, hybrid, or fully self-hosted in your own VPC\" and \"Yes, the AI Gateway supports on-premise deployments on any infrastructure or cloud provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway); [Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment)). Air-gapped: n.a. (explicitly not mentioned on the deployment pages fetched).", "byok_mode_note": "byok_only: providers are added as Model Accounts — \"A Model Account represents one account of a model provider (e.g. OpenAI, Anthropic, AWS Bedrock)… You can add multiple accounts per provider, each with their own API keys\" — and models are added \"after providing API key\" ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)). No TrueFoundry-sold inference credits are documented; pricing is platform subscription plus usage ([TrueFoundry pricing](https://www.truefoundry.com/pricing)).", "access_meta": { "qualified": [ "api_anthropic_messages", "api_audio" ], "model_count_kind": "vendor_total", "provider_count_kind": "enumerated" }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` as a request-timeout key. What exists are SLA thresholds used for health, not for aborting a request: `sla_cutoff.time_to_first_token_ms` and `sla_cutoff.time_per_output_token_ms` ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).", "rel_retry_config": "config_file", "rel_retry_note": "`config_file` (also editable in the UI). `retry_config` with `attempts` (**default `0`**), `delay` (**default `100` ms**), and `on_status_codes` (**default `[\"429\",\"500\",\"502\",\"503\"]`**). Retries occur on the same target; backoff beyond the fixed `delay` is not documented ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).", "rel_retry_default_note": "`retry_config` with `attempts` (**default `0`**), `delay` (**default `100` ms**), and `on_status_codes` (**default `[\"429\",\"500\",\"502\",\"503\"]`**).", "rel_fallback_config": "config_file", "rel_fallback_note": "ORDERED for priority-based routing (`priority`, lower number = higher priority, `0` highest); fallback targets come from `load_balance_targets`, gated by `fallback_status_codes` (default `[\"401\",\"403\",\"404\",\"408\",\"429\",\"500\",\"502\",\"503\"]`) and `fallback_candidate` (default `true`). Ordering depends on strategy: latency-based puts lower-latency targets first; weight-based sticky routing tries remaining healthy targets in sequence and that order is explicitly \"not weight-based\" ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).", "rel_fallback_shape": "ordered", "rel_health_check_config": "config_file", "rel_health_check_note": "`config_file`, and it is passive health tracking plus cooldown ejection. The gateway \"continuously monitors every target\", marks failing targets unhealthy, moves them to the end of the list as last resort, and recovers automatically. Failure-based cooldown: 5xx/429/401/403, default threshold 2+ failures in a rolling 2-minute window. SLA-based cooldown (priority routing only) via `sla_cutoff.time_per_output_token_ms` / `time_to_first_token_ms`, averaged over a 3-minute rolling window using up to 10 samples with a minimum of 3; TTFT samples only come from streaming requests and TPOT samples require at least 30 output tokens ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). TrueFoundry's blog additionally describes classic circuit-breaker states (`closed`/`open`/`half_open`) as a pattern, but states no configuration keys for it ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).", "rel_multi_region_config": "config_file", "rel_multi_region_note": "SaaS requests are auto-tagged with `tfy_gateway_region` and `tfy_gateway_zone`, and you route on them with `metadata_match` (documented examples `US → azure-us/gpt-4o`, `EU → azure-eu/gpt-4o`, `IN → azure-in/gpt-4o`, and `metadata_match: {tfy_gateway_zone: SFO}`), with a catch-all `priority: 1` target as the cross-region fallback. Both are true here: TrueFoundry runs multiple SaaS gateway regions AND you configure the cross-region target chain yourself ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). Multiple gateways can also be deployed cross-region under one control plane ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).", "rel_load_balancing_config": "config_file", "rel_load_balancing_note": "Weights supported: `routing_config.type` of `weight-based-routing` (integer `weight` 0–100, must sum to 100), `priority-based-routing`, `latency-based-routing` (\"No weights needed\", selection by recent TPOT), and `complexity-based-routing` (`simple`/`medium`/`complex`). Targets are catalog identifiers (`azure/gpt-4o`, `openai/gpt-4o`, `bedrock/llama`, `onprem/llama`), with `metadata_match`, `headers_override` and `override_params` per target; rules are evaluated in order and the first match wins ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).", "architecture_class": "interpreted_proxy", "architecture_note": "`interpreted_proxy` (JS/edge runtime). \"The AI Gateway is written using the Hono framework\", described as \"ultra-fast, minimalistic, and is designed for the edge\" ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).", "self_host_data_plane": "yes", "self_host_artifact_note": "`yes`, but the artifact is not named on the fetched pages. Deployment is documented as SaaS, your VPC, on-prem or air-gapped, running as Kubernetes pods with readiness probes and optional `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` forward-proxy env vars; no Docker image name or Helm chart name is stated ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture), [TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).", "latency_claims": [ { "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture", "unit": "ms", "scope": "gateway_overhead", "value": "+7", "metric": "not stated", "publisher": "vendor", "conditions": "200-220 RPS on 1 vCPU / 1 GB RAM, tracing disabled. Percentile, payload and cache state not stated." }, { "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture", "unit": "ms", "scope": "gateway_overhead", "value": "+8", "metric": "not stated", "publisher": "vendor", "conditions": "Same hardware at 200-220 RPS with complete tracing enabled." }, { "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture", "unit": "ms", "scope": "gateway_overhead", "value": "+12", "metric": "not stated", "publisher": "vendor", "conditions": "350-370 RPS with complete tracing — the only figure here showing overhead rising with load." } ], "throughput_claims": [ { "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture", "unit": "RPS", "scope": "capacity", "value": "250", "metric": "sustained", "publisher": "vendor", "conditions": "Single pod, 1 CPU / 1 GB RAM, no degradation stated until about 350 RPS." } ], "streaming_support": "yes", "streaming_note": "Supported, with the most precise streaming-fallback semantics in this set: the gateway waits for the \"first non-empty stream chunk\", and if Anthropic returns `overloaded_error` before that chunk it falls back to the next eligible target. Separate caveat: `prompt_version_fqn` \"does not work with agents (when using MCP/tools)\". Note also that TTFT SLA samples are only recorded on streaming requests ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).", "guard_pii": "sync_block", "guard_pii_note": "Built-in PII Detection and Azure AI Language PII/PHI guardrails ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview)). Custom plugins distinguish mutation from validation: a Presidio `POST /pii-redaction` plugin returns `transformed: true` with a full replacement `requestBody` to redact, whereas a Guardrails-AI `POST /pii-detection` plugin \"only detects and reports PII without redacting it\" ([Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))", "guard_injection": "sync_block", "guard_injection_note": "Azure Prompt Shield is a built-in guardrail integration; external options include Model Armor, Prisma AIRS, CrowdStrike, and NeMo ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview))", "guard_content": "sync_block", "guard_content_note": "Azure Content Safety moderation and OpenAI Moderations are supported; blocked requests return a configurable `custom_error_message` ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview), [Guardrails configuration](https://www.truefoundry.com/docs/ai-gateway/guardrails-configuration))", "guard_custom": "sync_block", "guard_custom_note": "Policy forms: built-in Regex and Secrets Detection rules, plus custom HTTP plugins bound to lifecycle hooks `llm_input`, `llm_output`, `mcp_tool_pre_invoke`, `mcp_tool_post_invoke`, returning either a validate verdict or a mutated request body ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview), [Guardrails configuration](https://www.truefoundry.com/docs/ai-gateway/guardrails-configuration), [Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))", "guard_model_policy": "yes", "guard_model_policy_note": "Access control is applied at the model-account level, governing which users/teams and which PATs/VATs can reach which models ([Gateway access control](https://www.truefoundry.com/docs/ai-gateway/gateway-access-control))", "guard_execution_location": "both", "guard_third_party": [ "Azure AI Language", "Azure Content Safety", "OpenAI Moderation", "Microsoft Presidio" ], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` as a global fail-open/fail-closed rule. For custom plugins, HTTP 4xx/5xx from the plugin signals a \"processing or dependency failure only; not used for 'PII found' policy outcomes\", which separates infrastructure failure from a policy verdict but does not state the resulting request disposition ([Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))", "log_content": "configurable", "log_content_note": "Global logging mode is `HEADER_CONTROLLED` (default behaviour: log unless the header says otherwise), `ALWAYS`, or `NEVER`; logged requests, viewable under AI Gateway → Monitor → Requests, include request content, and sensitive values can be redacted per subject/model/metadata ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging))", "log_opt_out": "yes", "log_opt_out_note": "Set the global mode to `NEVER`, or send `X-TFY-LOGGING-CONFIG: {\"enabled\": false}` (stringified JSON) per request; metrics dashboards remain populated ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging), [Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))", "tracing_support": "otel", "tracing_note": "OpenTelemetry natively for both traces and metrics, with MCP tool and agent spans; trace storage is managed by TrueFoundry ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data))", "export_destinations": [ "OpenTelemetry", "Datadog", "Prometheus", "Grafana", "S3", "Webhook" ], "export_destinations_note": "OTLP exporters to Grafana Cloud, Datadog (OTLP), and AWS CloudWatch, plus any OTLP endpoint ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data)); aggregated metrics download as CSV and per-datasource metrics APIs for model, MCP, guardrail, cache, routing, and agent metrics ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` — no feedback/score endpoint documented ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` on the fetched pages; the closest is `audit_mode_blocked`, a guardrail outcome that records what *would* have been blocked without blocking it ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))", "observability_requires_saas": "Partially — the gateway is customer-deployed and telemetry can be exported to the customer's own stack, but \"Traces storage is managed by TrueFoundry\" ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data))", "log_retention_note": "`n.a.` — no retention window stated ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "Documented overhead table (metric: added latency in ms; hardware 1 vCPU / 1 GB RAM): at **200–220 RPS** `+7 ms` tracing disabled, `+7 ms` tracing enabled without request logs, `+8 ms` complete tracing; at **350–370 RPS** `+7 ms`, `+7 ms`, `+12 ms`. Percentile is not stated; payload and cache conditions are not stated ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)). The blog restates this as \"~7 ms overhead at 200 RPS even with tracing on\" and adds promotional figures **`~10ms`** (\"Built for Speed: ~10ms Latency, Even Under Load\") and **`~3–4 ms`**, neither with a stated percentile or conditions ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).", "multi_surface": {}, "throughput_prose": "**250 RPS** on a single pod with 1 CPU / 1 GB RAM, scaling \"without any degradation in performance until about 350 RPS\" ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)); the blog states **350+ RPS** on 1 vCPU ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).", "benchmark_independence": "All vendor self-published. The docs table (with RPS, hardware and tracing conditions) is the credible artifact; the blog's `~10ms` and `~3–4 ms` figures carry no methodology and the `~3–4 ms` figure appears in a comparative context, so treat it as marketing rather than a measured result ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture), [TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages))." }, "guard_pii_default": "choose_at_setup", "guard_injection_default": "choose_at_setup", "guard_content_default": "sync_block", "guard_custom_default": "choose_at_setup", "rel_notes": "Latency-based routing uses Time Per Output Token (total response time ÷ output tokens) over a 20-minute window with a 10-minute sticky epoch, chosen because TPOT folds TTFT and inter-token latency into one output-length-independent number ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). Control-plane/data-plane separation means gateway pods only take traffic after config sync (readiness probe) ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).", "guard_notes": "TrueFoundry's guardrail analytics are the most enforcement-aware of any product here: outcomes are split into `allowed`, `blocked`, `mutated`, and `audit_mode_blocked`, with per-guardrail block/mutate rates and P50–P99 guardrail latency, so you can measure whether a control actually fires ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics)). One documentation inconsistency: the analytics page labels \"Total Flagged Requests\" as \"requests that were blocked by a guardrail\" ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics)).", "fit_agent_features_note": "Through the OpenAI Agents SDK integration, TrueFoundry documents multi-agent workflows, handoffs, function tools, human-in-the-loop workflows, automatic tracing, session management, and conversation history across agent runs. The documentation does not state A2A support. ([TrueFoundry OpenAI Agents SDK integration](https://www.truefoundry.com/docs/ai-gateway/openai-agents-sdk))", "fit_byok_required": "yes", "fit_byok_required_note": "Effectively yes to have any model: \"Select the provider you want to add models from and add models after providing API Key,\" though the page does not state a first-call rule explicitly ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))", "fit_client_sdk_langs": [ "Python", "Node.js", "Go", "Rust", "Java" ], "fit_client_sdks_note": "Documented client and integration formats include OpenAI SDK, LangChain, LangGraph, Google ADK, LlamaIndex, Go-OpenAI, Rust-OpenAI, LangChain4j, Node.js, cURL, REST, and streaming APIs. Official OpenAI Agents SDK packages are `openai-agents` for Python and `@openai/agents` for JavaScript/TypeScript; documented classes include `Agent`, `Runner`, `OpenAIChatCompletionsModel`, and `AsyncOpenAI`. ([TrueFoundry OpenAI Agents SDK integration](https://www.truefoundry.com/docs/ai-gateway/openai-agents-sdk))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "Yes for platform-level cloud access: TrueFoundry documents AWS IAM Roles for Service Accounts (IRSA, `sts:AssumeRoleWithWebIdentity` with the `eks.amazonaws.com/role-arn` annotation) and GCP GKE Workload Identity; Azure is listed as \"Coming Soon\". ([TrueFoundry docs - access cloud services with IAM](https://www.truefoundry.com/docs/access-data-from-s3-or-other-clouds-services))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated; pricing lists a \"$ 0 / month\" \"Developer\" tier, with no trial-credit or credit-card wording ([Pricing - TrueFoundry](https://www.truefoundry.com/pricing))", "fit_ecosystem_notes": "TrueFoundry supports fully managed SaaS, hybrid, private-VPC, on-premises, and fully self-hosted deployments, including use on any infrastructure or cloud provider and as a standalone gateway. The self-hosted Gateway Plane is stateless and connects securely to the TrueFoundry control plane over WebSockets; the gateway supports providers including OpenAI, Anthropic, Azure, and 1000+ LLMs overall. ([TrueFoundry introduction to the LLM Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway))", "fit_eval_prereqs": "A TrueFoundry account (created in step 1) with a verified email, and a model-provider account plus API key to add models. Credit card, cloud account and cluster are not mentioned ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))", "fit_eval_shape": "cloud_console", "fit_eval_snippet": "partial", "fit_eval_snippet_note": "`client = OpenAI(api_key=\"your_truefoundry_api_key\", base_url=\"{GATEWAY_BASE_URL}\")` — but no completed request call is shown on the page; snippets live in the Playground's Code Snippets tab ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "tfy-llm-gateway for the Gateway Plane, truefoundry for the full control plane plus gateway plane", "fit_kubernetes": "official_helm", "fit_kubernetes_note": "The AI Gateway can be deployed on Kubernetes using the `tfy-llm-gateway` Helm chart at `https://github.com/truefoundry/infra-charts/tree/main/charts/tfy-llm-gateway`. A separate full TrueFoundry control-plane and gateway-plane installation uses the `truefoundry` chart from `https://github.com/truefoundry/infra-charts/tree/main/charts/truefoundry`. ([TrueFoundry deployment overview](https://www.truefoundry.com/docs/platform/deployment-overview))", "fit_mcp": "mcp_gateway", "fit_mcp_note": "TrueFoundry documents an MCP Gateway that centralizes access to MCP servers, with dynamic tool discovery and invocation, a curated registry, and support for remote, virtual, OpenAPI, and stdio MCP servers over streamable HTTP. It also documents centralized authentication and combining tools from multiple MCP servers through Virtual MCP Servers. ([TrueFoundry MCP Gateway overview](https://www.truefoundry.com/docs/ai-gateway/mcp/mcp-overview))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers", "fit_gateway_platform", "fit_model_gate", "fit_vercel_ai_sdk" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented) on the quickstart, self-hosting or pricing pages ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))", "fit_onboarding_notes": "No install commands are given in the quickstart at all; auth uses a PAT or VAT from the Access section, and multiple provider accounts/keys can be added. Self-hosted PostgreSQL 17+ requires disabling SSL (`force_ssl=0` on AWS, `require_secure_transport=false` on Azure) ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. TrueFoundry documents OpenAI-compatible drop-in usage by changing the client `base_url` and API key; the documented SaaS base URL is `https://gateway.truefoundry.ai`, while self-hosted examples use `{GATEWAY_BASE_URL}`. ([TrueFoundry AI Gateway quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start))", "fit_prod_prereqs": "An organization registered on TrueFoundry; Kubernetes 1.27+ with nodes having at least 100 GB ephemeral disk; dynamic PVC provisioning; an ingress controller or Istio service mesh; a domain and TLS certificate; outbound HTTPS access to TrueFoundry endpoints; PostgreSQL >= 13 (managed recommended); S3/GCS/Azure Blob or other S3-compatible blob storage; and TrueFoundry-provided Tenant Name, License Key (`TFY_API_KEY`) and Image Pull Secret. Sizing: medium tier ≥3 AI Gateway replicas (~500 req/s), large tier ≥10 replicas (~2,000 req/s) ([Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "infra_rollout", "fit_python_frameworks": [ "LangChain", "LangGraph", "LlamaIndex" ], "fit_python_frameworks_note": "TrueFoundry documents ready-to-use LangChain and LlamaIndex snippets. The LangChain integration uses `pip install langchain-openai` and the `ChatOpenAI` class; LangGraph examples use `StateGraph`, `MessagesState`, and related workflow APIs. ([TrueFoundry LangChain integration](https://www.truefoundry.com/docs/ai-gateway/langchain))", "fit_quickstart_steps": 4, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.truefoundry.com/gateway/quick-start", "fit_self_host_docs": "documented_no_command", "fit_self_host_install": "No literal install command is published on the self-hosting pages; they document topologies and prerequisites only ([Self-hosting of Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/ai-gateway/gateway-self-hosted), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))", "fit_terraform": "shipped_code", "fit_terraform_note": "TrueFoundry states that its software ships with OpenTofu/Terraform code, but no official Terraform provider, module, or Terraform Registry name is documented. ([TrueFoundry deployment overview](https://www.truefoundry.com/docs/platform/deployment-overview))", "fit_terraform_ref": "the software ships with OpenTofu/Terraform code; no provider, module or Registry name is documented", "fit_third_party_documented": {}, "fit_vendor_time_claim": null, "fit_vercel_ai_sdk": "not_documented", "fit_vercel_ai_sdk_note": "n.a. (not documented)", "fit_vercel_ai_sdk_package": null }, { "slug": "fireworks-ai", "name": "Fireworks AI", "company": "Fireworks AI", "tagline": "Inference provider serving open-weight models on its own stack, with fine-tuning and dedicated deployments.", "category": "inference_provider", "deployment": "managed_only", "license": "Apache-2.0", "founded_year": 2022, "hq_country": "United States", "url_home": "https://fireworks.ai/", "url_docs": "https://docs.fireworks.ai/getting-started/introduction", "url_pricing": "https://fireworks.ai/pricing", "url_status": "https://status.fireworks.ai/", "url_github": "https://github.com/fw-ai/cookbook", "url_changelog": null, "url_models_api": "https://api.fireworks.ai/v1/accounts/{account_id}/models", "github_stars": 211, "model_count": 27, "model_count_note": "Approved published total on 2026-09-26. See the linked source for scope.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": null, "free_tier": "$1 in free credits on signup, then postpaid billing.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Region-restricted deployments", "amount": "1.5x premium over standard deployment pricing" }, { "label": "On-demand GPUs", "amount": "H100/H200 $8.00/hour, B200 $13.00, B300 $15.00, GB300 $20.00" }, { "label": "Managed LoRA supervised fine-tuning", "amount": "$0.50-$10.00 per 1M tokens depending on model size" }, { "label": "Embeddings", "amount": "$0.008-$0.10 per 1M input tokens" } ], "self_host_cost_note": "No self-host option; Fireworks sets its own per-token serverless prices rather than marking up another vendor's list price, with DEFAULT, PRIORITY and FAST service tiers resolved live per model. Dedicated capacity is billed per GPU-hour, and batch inference plus supervised and reinforcement fine-tuning are priced separately.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": true, "config_as_code": null, "portability_note": "Export covers rated usage and cost via firectl, not request logs.", "api_surfaces": [ "OpenAI Chat Completions", "Batch inference API", "Fine-tuning API (SFT and RFT)", "Dedicated deployments" ], "sdk_note": "Documented as a drop-in replacement using the same API, so OpenAI SDKs work by changing the base URL; the service_tier parameter is Fireworks-specific and does not port to other providers.", "cap_failover": null, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": null, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": null, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": "Not a proxy, so there is no gateway overhead figure. Third-party measurements: a Telnyx-run streamed GLM-5.2 benchmark (240 successful requests, 10 runs per provider, June 25 2026) recorded a 1.28 s median time-to-first-token for Fireworks versus 0.79 s for Together, while Artificial Analysis' live GLM-5.1 and Kimi K3 trackers placed Fireworks near the top for output speed among ten tracked providers; a separate Python harness on Llama 3.1 8B reported about 320 tokens/sec median for Fireworks versus 580 for Groq and 250 for Together.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": null, "eu_residency": null, "eu_residency_note": "Region-restricted deployments are offered at a 1.5x price premium, but no specific EU region list or residency commitment was found in vendor documentation.", "zdr": true, "zdr_note": "Zero retention is the default for open-model inference without opt-in logging. Responses API storage is on by default for 30 days; set store=False to disable it.", "air_gapped": null, "best_for": "Teams wanting fast serving of open-weight models with tiered latency options and both supervised and reinforcement fine-tuning.", "strengths": [ "100+ open-weight models with a drop-in OpenAI-compatible API", "Selectable DEFAULT, PRIORITY and FAST service tiers plus batch inference for cost control", "Both supervised and reinforcement fine-tuning, with managed LoRA priced from $0.50 per 1M tokens", "States SOC 2 Type II and HIPAA compliance, with a public trust center and documented CMEK encryption states", "Region-restricted deployments available when workloads must stay in a specific geography" ], "weaknesses": [ "Single-source provider: no closed frontier models, no independent catalog, and no cross-provider routing or failover", "Only a one-time $1 evaluation credit, which is too small for realistic load testing", "GDPR, EU residency specifics and zero-data-retention are not documented on vendor pages found", "Serverless is multi-tenant and priority tiers cost more, so predictable latency effectively requires paying up or going dedicated" ], "avoid_if": "You need frontier closed models, multi-provider failover, or documented EU data residency and zero retention.", "reviews": [ { "url": "https://workos.com/blog/fireworks-ai-the-pytorch-teams-bet-on-inference-as-the-new-runtime", "why": "Zack Proser's deep dive into Fireworks' FireAttention stack, serverless vs on-demand deployment model and where its inference economics come from.", "date": "2026-01-08", "type": "written", "title": "Fireworks.ai: The PyTorch Team's Bet on Inference as the New Runtime", "publisher": "WorkOS" }, { "url": "https://aishwaryasrinivasan.substack.com/p/running-llama-4-maverick-full-1-m", "why": "Hands-on account of pushing Llama 4 Maverick's full 1M-token context through the Fireworks API, with the scripts and practical limits described.", "date": "2025-04-29", "type": "written", "title": "Running LLaMA 4 Maverick Full 1M-Token Context on Fireworks", "publisher": "AI with Aish (Substack)" }, { "url": "https://www.youtube.com/watch?v=1w8iEY3z4DE", "why": "theCUBE's hosts press Fireworks' CEO on inference economics and platform strategy in a third-party-produced conference interview.", "date": "2026-07-09", "type": "video", "title": "Lin Qiao, Fireworks AI | RAISE Summit 2026", "publisher": "YouTube - SiliconANGLE theCUBE" }, { "url": "https://www.youtube.com/watch?v=i__vPJ9bW6I", "why": "Research-podcast hosts interview a Fireworks co-founder on GPU sourcing, kernels, routing and cross-region training that underpin the service.", "date": "2026-06-24", "type": "video", "title": "Infrastructure for AI at Scale - With Benny Chen (Fireworks AI)", "publisher": "YouTube - The Information Bottleneck" }, { "url": "https://www.reddit.com/r/speechtech/comments/1pkq9pv/fireworksai_ast_critical_issues_stay_away_until/", "why": "Founder documents a 35-day Fireworks speech-to-text defect with dates, connection-count symptoms, cost impact and the workaround used, plus corroborating replies.", "date": "2025-12-12", "type": "community", "title": "Fireworks.ai AST critical issues (stay away until they fix them)", "publisher": "Reddit - r/speechtech" }, { "url": "https://www.reddit.com/r/LocalLLaMA/comments/1jti1j4/llama_4_behaving_differently_on_groq_vs_fireworks/", "why": "Head-to-head thread on identical-model output differences between the two providers, with notes on Fireworks' fine-tune hosting and tier/rate-limit behaviour.", "date": "2025-04-07", "type": "community", "title": "LLaMa 4 behaving differently on Groq vs Fireworks AI", "publisher": "Reddit - r/LocalLLaMA" } ], "sources": { "zdr": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "name": "https://fireworks.ai/pricing", "soc2": "https://docs.fireworks.ai/faq/enterprise/compliance/certifications", "company": "https://en.wikipedia.org/wiki/Fireworks_AI", "license": "https://github.com/fw-ai/cookbook", "tagline": "https://docs.fireworks.ai/getting-started/introduction", "sdk_note": "https://docs.fireworks.ai/getting-started/introduction", "url_docs": "https://docs.fireworks.ai/getting-started/introduction", "url_home": "https://fireworks.ai/pricing", "zdr_note": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "cap_audio": "https://docs.fireworks.ai/getting-started/introduction", "cap_batch": "https://docs.fireworks.ai/guides/batch-inference", "free_tier": "https://fireworks.ai/pricing", "hipaa_baa": "https://docs.fireworks.ai/faq/enterprise/compliance/certifications", "extra_fees": "https://fireworks.ai/pricing", "hq_country": "https://en.wikipedia.org/wiki/Fireworks_AI", "url_github": "https://docs.fireworks.ai/getting-started/introduction", "url_status": "https://docs.fireworks.ai/getting-started/introduction", "zdr_status": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "model_count": "https://api.fireworks.ai/inference/v1/models", "url_pricing": "https://fireworks.ai/pricing", "api_surfaces": "https://docs.fireworks.ai/guides/batch-inference", "founded_year": "https://en.wikipedia.org/wiki/Fireworks_AI", "github_stars": "https://github.com/fw-ai/cookbook", "product_fate": "https://docs.fireworks.ai/faq/deployment/serverless/service-levels", "regions_note": "https://docs.fireworks.ai/deployments/regions", "cap_image_gen": "https://docs.fireworks.ai/getting-started/introduction", "vendor_status": "https://www.reuters.com/technology/nvidia-backed-startup-fireworks-valued-175-billion-latest-funding-2026-07-16/", "cap_embeddings": "https://fireworks.ai/pricing", "certifications": "https://docs.fireworks.ai/guides/security_compliance/data_security", "retention_days": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "url_models_api": "https://docs.fireworks.ai/api-reference/list-models", "logs_by_default": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "model_count_low": "https://api.fireworks.ai/inference/v1/models", "model_count_high": "https://api.fireworks.ai/inference/v1/models", "model_count_note": "https://api.fireworks.ai/inference/v1/models", "eu_residency_note": "https://fireworks.ai/pricing", "fireworks_pricing": "https://fireworks.ai/pricing", "last_release_date": "https://docs.fireworks.ai/updates/changelog", "openai_compatible": "https://docs.fireworks.ai/tools-sdks/openai-compatibility", "guardrails_location": "https://docs.fireworks.ai/faq/security/infrastructure/model-security", "latency_source_note": "https://markaicode.com/vs/together-ai-vs-fireworks-ai/", "requires_vendor_sdk": "https://docs.fireworks.ai/tools-sdks/openai-compatibility", "self_host_cost_note": "https://docs.fireworks.ai/api-reference/list-models", "data_export_available": "https://docs.fireworks.ai/accounts/exporting-usage-costs", "prompt_transits_vendor": "https://docs.fireworks.ai/deployments/regions", "model_count_source_note": "https://api.fireworks.ai/inference/v1/models", "trains_on_customer_data": "https://docs.fireworks.ai/guides/security_compliance/data_handling", "fireworks_cost_structure_faq": "https://docs.fireworks.ai/faq/billing-pricing-usage/pricing/cost-structure", "fireworks_caching_billing_faq": "https://docs.fireworks.ai/faq-new/billing-pricing/is-prompt-caching-billed-differently" }, "verified_at": { "zdr": "2026-09-05", "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "license": "2026-09-17", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-09-05", "cap_audio": "2026-08-29", "cap_batch": "2026-08-29", "free_tier": "2026-08-29", "hipaa_baa": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_github": "2026-08-29", "url_status": "2026-08-29", "model_count": "2026-09-26", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "founded_year": "2026-08-29", "github_stars": "2026-09-17", "cap_image_gen": "2026-08-29", "cap_embeddings": "2026-08-29", "url_models_api": "2026-08-29", "model_count_low": "2026-09-26", "model_count_high": "2026-09-26", "model_count_note": "2026-09-26", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "data_export_available": "2026-08-29", "model_count_source_note": "2026-09-26" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:16:59.956263+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed with near-daily changelog activity. Raised a $1.51B Series D at a $17.5B valuation in July 2026. Note that the serverless tier carries no latency or availability guarantee at all.", "last_release_date": "2026-08-27", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Inference runs on Fireworks' own global hardware fleet.", "logs_by_default": "none", "logs_note": "No prompt or generation data is logged or stored for any open model without explicit opt-in — it exists only in volatile memory for the life of the request. Metadata such as token counts is logged as needed to run the service.", "retention_days": 0, "retention_note": "Zero by default. The Responses API is the exception: storage defaults to on, held 30 days, and you must pass store=False to opt out.", "zdr_status": "yes", "trains_on_customer_data": "opt_in", "training_note": "Nothing is stored without explicit opt-in, and opting in enables features such as FireOptimizer. Worth noting what is absent: no blanket “we never train on your data” statement was found.", "subprocessors_url": null, "regions_note": "Region groupings for global (the default), US, Europe and Asia-Pacific, plus around 19 pinnable single regions including Iowa, Frankfurt, Iceland and Tokyo.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "Type II", "status": "certified" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "controls mapped to GDPR and CCPA; no executable DPA page retrieved", "status": "claimed" }, "iso27001": { "detail": "achieved, alongside 27701 and 42001", "status": "certified" }, "hipaa_baa": { "detail": "described as HIPAA-certified and compliant; no BAA document retrieved", "status": "claimed" } }, "guardrails_location": "none", "guardrails_note": "No guardrails are applied to models by default; Fireworks expects customers to implement their own.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference (Fireworks is the provider). Per-token serverless, per-GPU-second on-demand, per-token/GPU-hour training, enterprise quote. No separate gateway or platform fee — routing layer is $0.", "min_commitment": "None stated for inference. Fine-tuning has no stated minimum on the pricing page.", "overage_note": "None stated. Serverless is postpaid, so cost scales with tokens. Note: 'Checkpoint storage for serverless models is included during private preview' implies a future charge.", "cache_support": "prefix", "cache_read_discount_pct": 50, "cache_write_premium_pct": 0, "cache_pricing_note": "Prompt caching with discounted cached tokens (prefix-style caching for serverless; mechanism not labelled exact-match or semantic). Default 50% discount, but exact discount varies by model. Pricing page separately lists 'Cached Prefill / 1M' rates for training that are 80% below prefill (e.g. Qwen 3.5 9B $0.132 vs $0.66; Kimi K3 $2.17 vs $10.87). No cache-write charge documented — close to a free win, but only serverless models are covered.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "Not stated.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "'High rate limits' on serverless; higher on Enterprise. Platform-set, not a configurable spend control.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Not stated.", "cost_export": "Not stated.", "product_kind_note": "\"Fireworks AI is the fastest platform for building with open source AI models\" ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)), exposing a \"Fireworks AI REST API\" for \"various language, image and embedding models using an API Key\" that \"also lets you automate management of models, deployments, datasets, and more\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). It is a first-party inference and training platform, not a router.", "deployment_modes": [ "saas" ], "deployment_note": "Serverless models are \"pre-deployed by the Fireworks team for the community to use\", shared, with \"no SLA guarantees for up-time or latency\", best-effort availability and at least two weeks' deprecation notice; \"neither custom base models nor LoRA addons are supported for serverless inference\" and \"all user-provided models, including trained models, require a dedicated deployment\" ([Models overview](https://docs.fireworks.ai/models/overview)). Dedicated deployments are \"private to you\", give access to \"a wide array of hardware\", support base models and LoRA addons, and are billed by GPU-second; deployments are addressed as `accounts//deployments/` or `accounts/fireworks/models/gpt-oss-120b#` ([Models overview](https://docs.fireworks.ai/models/overview); [On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). Region-restricted deployments carry a 1.5x premium ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap with the OpenAI SDK (`https://api.fireworks.ai/inference/v1`) or curl; a first-party Fireworks Python SDK also exists but is not required ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). All requests need `Authorization: Bearer ` and `Content-Type: application/json` ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). Operational caveat: a deployment scaled to zero returns \"a `503` error immediately\" while it scales up, so \"applications should implement retry logic\" ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). Models must be deployed before use — \"a model must be deployed before it can be used for inference\" ([Models overview](https://docs.fireworks.ai/models/overview)).", "model_count_low": 27, "model_count_high": 27, "model_count_source_note": "Approved published total on 2026-09-26. See the linked source for scope.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party only: Fireworks serves models on its own platform (\"the fastest platform for building with open source AI models\") and does not route to third-party provider APIs, so no provider count exists ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction); [Models overview](https://docs.fireworks.ai/models/overview)).", "first_party_vs_partner": "Vendor-operated hardware serving open-weight and customer models: deployments provide \"dedicated GPUs for your models\" and preemptible deployments borrow \"idle reserved GPUs\"; Fireworks maintains \"a library of common base models\" and customers can add their own ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments); [Models overview](https://docs.fireworks.ai/models/overview)). Whether that hardware is Fireworks-owned or leased from clouds is not stated: n.a. ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "custom_endpoint_support": "Bring-your-own-weights rather than bring-your-own-endpoint: \"upload your own models (for supported architectures) from Hugging Face or elsewhere\", plus LoRA addon upload/training, all requiring a dedicated deployment ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments); [Models overview](https://docs.fireworks.ai/models/overview)). Registering an external self-hosted endpoint (vLLM, Ollama, SageMaker, private URL): n.a.", "byok_mode": "platform_credits_only", "merchant_of_record": "Fireworks: it bills serverless usage per token (postpaid), dedicated/on-demand per GPU-second with \"no extra charges for start-up times\", fine-tuning per 1M training tokens and reinforcement fine-tuning per GPU hour billed per second, with preemptible capacity carrying \"no charge to hold dedicated capacity\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "key_management_note": "Single-vendor key model: API keys are created with `firectl api-key create` or in the dashboard at `https://app.fireworks.ai/settings/users/api-keys`, and every request carries `Authorization: Bearer `; account-scoped quota APIs (List/Get/Update Quota) exist ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). No third-party provider secrets are stored, and inference necessarily runs on Fireworks infrastructure, so Fireworks processes request content; the models overview asserts \"your data is yours\" ([Models overview](https://docs.fireworks.ai/models/overview)). Encryption/KMS specifics: n.a.", "api_openai_chat_completions_note": "Yes: `POST https://api.fireworks.ai/inference/v1/chat/completions`, configured as an OpenAI SDK `base_url`/`baseURL` of `https://api.fireworks.ai/inference/v1` ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "api_anthropic_messages_note": "n.a. (not documented on the pages fetched).", "api_openai_responses_note": "n.a.", "api_embeddings_note": "Yes: embedding models are part of the REST API and are priced per 1M input tokens by base-model size (up to 150M: $0.008; 150M–350M: $0.016; Qwen3 8B: $0.10), with \"embeddings & reranking\" positioned for \"search & context retrieval\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction); [Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)).", "api_images_note": "Yes as a supported model family: the REST API covers \"language, image and embedding models\" and the catalog spans \"text, vision, audio, image, and embeddings\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction); [Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)). Endpoint path: n.a.", "api_audio_note": "Yes as a supported modality (\"text, vision, audio, image, and embeddings\"), but no STT/TTS endpoint path is documented on the pages fetched ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)).", "api_batch_note": "Yes: \"Batch Inference — run async inference jobs at scale, faster and cheaper\" ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)). Endpoint path: n.a.", "deployment_modes_note": "hosted cloud service in two shapes: serverless inference (\"use popular models instantly with pay-per-token pricing\", pre-deployed by Fireworks and shared across users) and dedicated GPU deployments (\"deploy with high performance on dedicated GPUs with fast autoscaling and minimal cold starts\"), plus training ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction); [Models overview](https://docs.fireworks.ai/models/overview)). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. — the models overview page explicitly does not mention any of them ([Models overview](https://docs.fireworks.ai/models/overview)). Enterprise deployments are quote-based: \"contact us for enterprise deployments with faster speeds, lower costs, and higher rate limits\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22).", "byok_mode_note": "platform_credits_only in the sense that all usage is billed by Fireworks on its own API key: serverless is \"pay per token\" with postpaid billing and \"$1 in free credits\", on-demand is \"pay per GPU second\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [Models overview](https://docs.fireworks.ai/models/overview)). There are no upstream provider keys to bring; requests authenticate with `FIREWORKS_API_KEY` ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "access_meta": { "qualified": [ "api_images", "api_audio" ], "model_count_kind": "vendor_total", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` for inference requests. The only documented timeout is control-plane: `--wait-timeout` on `firectl deployment create` with `--wait`, **default 1 hour** ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented` as a platform feature; it is pushed to the client: when a deployment is scaled to zero, requests return `503` immediately and \"Your application should implement retry logic to handle this\". Default count and backoff: `n.a.` ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "No model or provider fallback list ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "`not_documented` as configurable. Deployment health is observable via `state` (`CREATING`, `READY`, `UPDATING`, `DELETING`, `DELETED`, `FAILED`) and replica counts (`min_replica_count`, `ready_replica_count`, `desired_replica_count`, `max_replica_count`), with guidance to \"Verify it's healthy\" before cutting traffic over — this is deployment status, not upstream ejection or circuit breaking ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "rel_multi_region_config": "config_file", "rel_multi_region_note": "`config_file` at creation time, and it is genuinely a capacity-failover control: `--region` accepts `GLOBAL`, `US`, `EUROPE`, `APAC`; `GLOBAL` is recommended for production workloads needing \"geographic availability or capacity failover\". Without `--region` a deployment is \"pinned to a single datacenter at creation time and will not be automatically migrated later\", and placement \"cannot be changed in place\" — you create a replacement deployment ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": "Deployment shapes can be optimized for speed, throughput or cost, but no load-balancing policy or weights ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "architecture_class": "vendor_saas", "architecture_note": "`vendor_saas` — an inference provider, not a gateway; the serving runtime is described by engine name (FireAttention V4) rather than a disclosed process runtime, and the control surface is the `firectl` CLI plus `https://api.fireworks.ai/inference/v1` ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200), [Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "self_host_data_plane": "no", "self_host_artifact_note": "Fireworks distributes `firectl` binaries (Homebrew, macOS arm64/x86_64, Linux x86_64, Windows 64-bit) and an open-source benchmarking tool, but not a self-hostable serving plane ([firectl CLI](https://docs.fireworks.ai/tools-sdks/firectl/firectl), [Fireworks performance benchmarking](https://docs.fireworks.ai/deployments/benchmarking)).", "latency_claims": [], "throughput_claims": [ { "url": "https://fireworks.ai/blog/fireattention-v4", "unit": "tokens/s", "scope": "generation_speed", "value": ">250", "metric": "output speed", "publisher": "vendor", "conditions": "DeepSeek V3 0324, FireAttention V4 at FP4, NVIDIA B200, 8 GPUs with NVLink, speculation disabled." } ], "streaming_support": "yes", "streaming_note": "Streaming is available through the OpenAI-compatible API surface; the fetched pages state no streaming-specific caveat, but they do state hard availability caveats: \"A preemptible deployment can be preempted mid-request and disappear with no warning\", \"Never point production or latency-sensitive traffic at a preemptible deployment\", and scaled-to-zero deployments return `503` immediately ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "guard_pii": "not_documented", "guard_pii_note": "No PII guardrail feature; the data-handling posture is zero retention rather than detection ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "guard_injection": "not_documented", "guard_injection_note": "`not_documented` as a platform feature ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "guard_content": "not_documented", "guard_content_note": "`not_documented` — no inline moderation filter is documented on the fetched pages; the data-handling and account-policy docs describe retention and model access only ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling), [Model access policy](https://docs.fireworks.ai/accounts/model-access-policy))", "guard_custom": "not_documented", "guard_custom_note": "`not_documented` ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "guard_model_policy": "yes", "guard_model_policy_note": "Enterprise account admins set a Model access policy controlling which models can be used for inference, deployments, and training; each model is allowed or denied per surface (serverless, serverless fast, dedicated, training), allowlist or denylist style, keyed on resource names like `accounts/fireworks/models/qwen3-235b-a22b`. It is account-wide: \"There is no per-user or per-group model access control today\", and updating requires the Admin role ([Model access policy](https://docs.fireworks.ai/accounts/model-access-policy))", "guard_execution_location": "not_documented", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "log_content": "metadata_only", "log_content_note": "Zero data retention applies by default and prompts/generations are not logged; the exception is the Responses API, where `store=True` is the default and stored responses are deleted after 30 days ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "log_opt_out": "yes", "log_opt_out_note": "`store=False` on Responses API calls prevents storage; ZDR is the default elsewhere ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "tracing_support": "proprietary", "tracing_note": "Via integrations rather than natively: Weights & Biases Weave and MLflow tracing are documented ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))", "export_destinations": [], "export_destinations_note": "A Prometheus-compatible metrics endpoint at `https://api.fireworks.ai/v1/accounts//metrics`, scrapeable by Prometheus, the OpenTelemetry Collector, the Datadog Agent, Vector, Grafana, and New Relic ([Exporting metrics](https://docs.fireworks.ai/deployments/exporting-metrics)); W&B and MLflow for traces ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` — evaluation happens in W&B/MLflow, not against Fireworks-stored traffic (there is none by default) ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability), [Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "observability_requires_saas": null, "log_retention_note": "No prompt/generation retention by default; Responses API stored objects are deleted after 30 days ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "**\">250 tokens/second\"** on DeepSeek V3 0324 with FireAttention V4 at FP4 (NVFP4) on NVIDIA B200, 8 GPUs with NVLink, speculation (MTP) disabled. This is a generation-speed figure, not gateway overhead; the post also names TTFT and end-to-end response time as metrics without giving numbers, and compares against H200 FP8 on SGLang and B200 FP4 on TRT-LLM. Related engine claims: NVFP4 has \"2x FLOPs throughput\" and needs \"~1.5x-2x less memory reads\" than MXFP6/MXFP8, and Hopper→Blackwell FP8 end-to-end improves \"~1.5-1.7x\" ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200)).", "multi_surface": {}, "throughput_prose": "Same \">250 tokens/second\" figure and hardware. Accelerator options are `NVIDIA_A100_80GB`, `NVIDIA_H100_80GB`, `NVIDIA_H200_141GB`, with the caveat that \"scaling is sub-linear (2x GPUs ≠ 2x performance)\". Fireworks publishes a self-service harness (`python benchmark.py --model ... --num-requests 1000 --concurrency 10`) rather than headline RPS numbers ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200), [Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments), [Fireworks performance benchmarking](https://docs.fireworks.ai/deployments/benchmarking)).", "benchmark_independence": "Vendor self-published, and **explicitly comparative against rival stacks** (SGLang on H200 FP8, TRT-LLM on B200 FP4). Fireworks itself flags the comparison as incomplete: \"We were not able to fully validate TRT-LLM FP4 performance on B200 due to setup complexity\", plus \"Speculation success being very prompt-dependent makes it hard to compare across implementations\" ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200)). Fireworks also appears as a compared provider in Groq's cited ArtificialAnalysis.ai benchmark — a third-party figure published by a competitor ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Reliability here is deployment topology, not routing config: choose `--region GLOBAL` at creation, avoid preemptible for production, keep `min_replica_count` above zero to avoid cold `503`s, and implement your own retries. Region placement being immutable is the biggest planning constraint ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).", "guard_notes": "Because nothing is retained, there is nothing to run guardrails or evals against after the fact — the trade-off is maximal privacy and minimal auditability ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling)). The user-management page documents no logging, tracing, or guardrail controls at all ([Managing users](https://docs.fireworks.ai/accounts/users)).", "fit_agent_features_note": "The Fireworks Responses API supports tool calls, `max_tool_calls`, parallel tool calls, `tool_choice`, streaming, and conversational continuation with `previous_response_id`. Fireworks also describes server-side handling of the agentic loop—reasoning, tool selection, and execution—and server-side state for long conversations. ([Fireworks Responses API reference](https://docs.fireworks.ai/api-reference/post-responses))", "fit_byok_required": "no", "fit_byok_required_note": "No. A Fireworks API key is used ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Documented clients include the Fireworks Python SDK, package `fireworks-ai` with class `Fireworks`; the OpenAI Python package `openai` with class `OpenAI`; the Anthropic Python package `anthropic` with class `anthropic.Anthropic`; the OpenAI JavaScript/TypeScript package `openai` with class `OpenAI`; and the Anthropic JavaScript/TypeScript package `@anthropic-ai/sdk` with class `Anthropic`. Fireworks also documents the `@ai-sdk/fireworks` JavaScript/TypeScript provider and `curl`. ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))", "fit_cloud_iam": "native_iam", "fit_cloud_iam_note": "For BYOC on AWS, Fireworks documents AWS IAM roles for the cluster and support for `AWS_PROFILE` when `firectl` uploads model files to S3. BYOC API examples use `api_key=\"unused\"` because the API key is not used, while airgapped deployment stores a metering key in AWS Secrets Manager. ([Fireworks EKS BYOC documentation](https://docs.fireworks.ai/ecosystem/integrations/eks-bring-your-own-cloud))", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_required", "fit_credit_card_note": "Not stated as required: \"Get started with $1 in free credits.\" ([Fireworks - Pricing](https://fireworks.ai/pricing))", "fit_ecosystem_notes": "Fireworks documents hosted serverless API endpoints and BYOC deployments, including private-preview deployments in customer-managed cloud, managed Kubernetes, GPU-cloud, or on-premises environments. Airgapped EKS deployments keep images, models, and dependencies in the organization’s infrastructure and cannot be managed through the Fireworks web app or `firectl`; the Fireworks Python SDK is documented as alpha and requires `--pre` for the latest version. ([Fireworks BYOC setup](https://docs.fireworks.ai/ecosystem/integrations/byoc/how-setup-works))", "fit_eval_prereqs": "A Fireworks API key created in the dashboard and exported as `FIREWORKS_API_KEY`. Account, credit card, cloud account, cluster and provider key are not mentioned as prerequisites ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))", "fit_eval_shape": "sdk_install", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `client.chat.completions.create(model=\"accounts/fireworks/models/deepseek-v3p1\", messages=[{\"role\":\"user\",\"content\":\"Say hello in Spanish\"}])`, with OpenAI, Anthropic, JavaScript and curl variants; example response `\"¡Hola!\"` ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))", "fit_gateway_platform": "third_party_documented", "fit_gateway_platform_note": "Not documented by Fireworks, but API7's APISIX-based AI gateway documents a `fireworks-ai` catalog provider that connects through the `openai` adapter with `api_base` = `https://api.fireworks.ai/inference/v1`. ([API7 AI Gateway - Fireworks AI provider](https://docs.api7.ai/ai-gateway/providers/fireworks-ai))", "fit_helm_chart": "Helm charts and templates for EKS BYOC clusters; no chart name or repo stated", "fit_kubernetes": "helm_unnamed", "fit_kubernetes_note": "Fireworks documents Kubernetes and Amazon EKS deployments, including deployment with Helm charts or Helm deployment templates. No official chart name, repository, or operator name is stated. ([Fireworks airgapped EKS deployment](https://docs.fireworks.ai/ecosystem/integrations/eks/airgapped))", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Fireworks supports MCP tools in its Responses API; the `tools` field accepts MCP tools and response tools support the `mcp` type. Fireworks also documents MCP server usage with an example SSE server URL such as `https://gitmcp.io/docs`. ([Fireworks Responses API reference](https://docs.fireworks.ai/api-reference/post-responses))", "fit_meta": { "not_documented_fields": [ "fit_cloudflare_workers" ] }, "fit_model_gate": "none", "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or region gate on either page ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart), [Fireworks - Pricing](https://fireworks.ai/pricing))", "fit_onboarding_notes": "The Python SDK is in alpha, so installs need `--pre` (`pip install --pre fireworks-ai`, `poetry add --pre fireworks-ai`, `uv add --pre fireworks-ai`); OpenAI/Anthropic-compatible paths avoid the alpha SDK (`pip install openai`, `pip install anthropic`, `npm install openai`, `npm install @anthropic-ai/sdk`). Priority tier is selected with `service_tier: \"priority\"` ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. For hosted inference, Fireworks documents the OpenAI SDK with `base_url=\"https://api.fireworks.ai/inference/v1\"`; BYOC examples use `base_url=\"http:///v1\"` and `api_key=\"unused\"`. ([Fireworks OpenAI compatibility docs](https://docs.fireworks.ai/tools-sdks/openai-compatibility))", "fit_prod_prereqs": "n.a. (not documented). No database, Redis, cluster, load balancer, control plane or licence requirement is stated ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart), [Fireworks - Pricing](https://fireworks.ai/pricing))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "sdk_install", "fit_python_frameworks": [ "LangChain", "LlamaIndex", "CrewAI", "PydanticAI", "Strands" ], "fit_python_frameworks_note": "Fireworks documents integrations with LangChain and LlamaIndex, and also lists CrewAI, PydanticAI, Strands Agents, and AWS AgentCore as supported agent frameworks. The LangChain JavaScript integration uses package `@langchain/fireworks` and class `Fireworks`; package and class names for the other documented integrations are not stated. ([Fireworks agent-framework integrations](https://docs.fireworks.ai/ecosystem/integrations/agent-frameworks))", "fit_quickstart_steps": 2, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.fireworks.ai/getting-started/quickstart", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "official_modules", "fit_terraform_note": "Fireworks documents the official Terraform module `fw-ai-external/aws-cluster/fireworksai` at `registry.terraform.io/modules/fw-ai-external/aws-cluster/fireworksai/latest`, with example version `0.1.2`. ([Fireworks EKS BYOC documentation](https://docs.fireworks.ai/ecosystem/integrations/eks-bring-your-own-cloud))", "fit_terraform_ref": "fw-ai-external/aws-cluster/fireworksai on the Registry, example version 0.1.2", "fit_third_party_documented": { "gateway_platform": "API7" }, "fit_vendor_time_claim": "Vendor claims, verbatim: \"Make your first Serverless API call in minutes\" and \"help you make your first API call in minutes\" ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart)); pricing page adds \"Get started in seconds with per token pricing, zero setup and no cold starts\" ([Fireworks - Pricing](https://fireworks.ai/pricing))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Fireworks has an official Vercel AI SDK provider package, `@ai-sdk/fireworks`, with provider instances `fireworks` and `createFireworks`. ([Vercel AI SDK Fireworks provider](https://ai-sdk.dev/providers/ai-sdk-providers/fireworks))", "fit_vercel_ai_sdk_package": "@ai-sdk/fireworks" }, { "slug": "groq", "name": "Groq", "company": "Groq", "tagline": "Inference provider running open-weight models on its own LPU hardware for very high output speed.", "category": "inference_provider", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2016, "hq_country": "United States", "url_home": "https://groq.com/", "url_docs": "https://console.groq.com/docs/models", "url_pricing": "https://groq.com/pricing", "url_status": "https://groqstatus.com/", "url_github": null, "url_changelog": null, "url_models_api": "https://api.groq.com/openai/v1/models", "github_stars": null, "model_count": 11, "model_count_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": null, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": null, "free_tier": "Free tier with per-model rate limits (for example gpt-oss-120b at 30 requests/min, 1,000 requests/day, 8,000 tokens/min); the paid Developer plan raises this to about 1,000 RPM and 250,000 TPM and adds Batch and Flex processing.", "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Speech-to-text (whisper-large-v3)", "amount": "$0.111 per hour of audio ($0.04/hour for v3-turbo)" }, { "label": "Text-to-speech (Orpheus)", "amount": "$22-$40 per 1M characters" } ], "self_host_cost_note": "No self-host option; Groq sets its own per-token prices on its own LPU/LPX hardware rather than marking up another vendor's list price. Representative rates: gpt-oss-120b $0.15 input / $0.60 output per 1M tokens, gpt-oss-20b $0.075/$0.30, Qwen3.6-27B $0.60/$3.00.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": null, "config_as_code": false, "portability_note": "Invoice downloads only; no documented usage or log export.", "api_surfaces": [ "OpenAI Chat Completions (https://api.groq.com/openai/v1)", "OpenAI Responses API", "Batch API (Developer plan)", "Speech-to-text and text-to-speech" ], "sdk_note": "Described as mostly compatible with OpenAI client libraries: base URL https://api.groq.com/openai/v1, with image input and function calling supported, but logprobs, logit_bias, top_logprobs, N>1 and vtt/srt outputs unsupported.", "cap_failover": null, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": null, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": null, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": true, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": null, "cap_image_gen": null, "cap_audio": true, "cap_video": null, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": "Groq is a direct provider, not a proxy, so the figure below is output speed rather than proxy overhead. Groq's own model docs publish per-model output speeds in tokens/sec: 1,000 t/s for GPT-OSS-20B, 500 t/s for GPT-OSS-120B and Qwen3.6-27B, 560 t/s for Llama 3.1 8B, 450 t/s for groq/compound, 280 t/s for Llama 3.3 70B. Independently, Artificial Analysis measured 3,400 tokens/sec on Gemma 4 31B at 100K-token context on Nvidia LPX racks built on Groq 3 LPUs, versus 882 t/s for Cerebras; The Register notes that test is a best case for the architecture. A third-party Python harness on Llama 3.1 8B measured about 580 tokens/sec median for Groq with sub-100 ms time-to-first-token.", "throughput_rps": 1000, "soc2": true, "hipaa_baa": null, "gdpr": null, "eu_residency": false, "eu_residency_note": "Groq states that all retained data is held in Google Cloud buckets in the United States, with Standard Contractual Clauses used for transfers; no EU residency option is documented.", "zdr": true, "zdr_note": "The most straightforward ZDR here: any customer can enable it in Data Controls, globally or per feature, with no request process.", "air_gapped": null, "best_for": "Latency-sensitive, user-facing applications on open-weight models where output speed matters more than catalog breadth.", "strengths": [ "Highest published output speeds of the group, up to 1,000 tokens/sec on GPT-OSS-20B on its own LPU hardware", "Zero Data Retention is self-serve for every customer, globally or per feature, with no retention of inference data by default", "Very low per-token prices, for example gpt-oss-120b at $0.15 input / $0.60 output per 1M tokens", "Mostly OpenAI-compatible API plus speech-to-text and text-to-speech endpoints", "SOC 2 Type II maintained with reports published for 2024 and 2025" ], "weaknesses": [ "Smallest catalog here: open-weight models only, with no frontier closed models and no published model count", "Single-source hardware provider — no cross-provider routing, load balancing or failover, so an outage has no in-platform fallback", "All retained data sits in US Google Cloud buckets, with no EU residency option and no documented HIPAA BAA", "Free-tier rate limits are tight (about 30 requests/min on gpt-oss-120b) and some OpenAI parameters are unsupported" ], "avoid_if": "You need a broad multi-vendor model catalog, EU data residency, or a signed HIPAA BAA.", "reviews": [ { "url": "https://www.tomshardware.com/tech-industry/semiconductors/nvidia-presents-groq-3-lpx-architecture-and-unveils-its-first-third-party-inference-benchmark", "why": "Luke James reports the first third-party benchmark numbers for the Groq 3 LPX rack and LP30 chip that power Groq's inference service, against Rubin and Cerebras.", "date": "2026-08-26", "type": "written", "title": "Hot Chips 2026: Nvidia presents Groq 3 LPX architecture and unveils its first third-party inference benchmark", "publisher": "Tom's Hardware" }, { "url": "https://newsletter.semianalysis.com/p/groq-inference-tokenomics-speed-but", "why": "SemiAnalysis models the per-token economics behind GroqCloud's headline latency, comparing its LPU deployment cost against GPU-based serving of Mixtral.", "date": "2024-02-21", "type": "written", "title": "Groq Inference Tokenomics: Speed, But At What Cost?", "publisher": "SemiAnalysis" }, { "url": "https://www.youtube.com/watch?v=S53BanCP14c", "why": "Hands-on Groq API tutorial on an established LLM channel: keys, Python/Colab usage, streaming, stop sequences and a Streamlit chatbot built on Mixtral and Llama 2 70B.", "date": "2024-02-29", "type": "video", "title": "Getting Started with Groq API | Making Near Real Time Chatting with LLMs Possible", "publisher": "YouTube - Prompt Engineering" }, { "url": "https://www.youtube.com/watch?v=WB1WLlKmlcQ", "why": "theCUBE interviews Groq's CEO at SC23 with a live latency demo and pointed questions about LPU economics versus GPUs and tokens-as-a-service.", "date": "2023-11-14", "type": "video", "title": "Jonathan Ross, Groq | SC23", "publisher": "YouTube - SiliconANGLE theCUBE" }, { "url": "https://www.reddit.com/r/LLMDevs/comments/1ufz1g0/groq_alternatives_for_production_apps_rate_limits/", "why": "7-comment thread on hitting Groq's rate limits and the closed developer plan when moving an app to production, with alternatives weighed.", "date": "2026-06-26", "type": "community", "title": "Groq alternatives for production apps? (Rate limits + Dev plan closed)", "publisher": "Reddit - r/LLMDevs" }, { "url": "https://news.ycombinator.com/item?id=46379183", "why": "Thread where practitioners discuss what the deal means for GroqCloud continuity, with concrete reliability comparisons against Cerebras and latency requirements for voice.", "date": "2025-12-24", "type": "community", "title": "Nvidia to buy assets from Groq for $20B cash", "publisher": "Hacker News" } ], "sources": { "zdr": "https://console.groq.com/docs/your-data", "name": "https://groq.com/company", "soc2": "https://trust.groq.com/", "company": "https://en.wikipedia.org/wiki/Groq", "tagline": "https://groq.com/pricing", "sdk_note": "https://console.groq.com/docs/openai", "url_docs": "https://console.groq.com/docs/models", "url_home": "https://groq.com/company", "zdr_note": "https://console.groq.com/docs/your-data", "cap_audio": "https://console.groq.com/docs/models", "cap_batch": "https://console.groq.com/docs/rate-limits", "free_tier": "https://console.groq.com/docs/rate-limits", "extra_fees": "https://console.groq.com/docs/models", "hq_country": "https://en.wikipedia.org/wiki/Groq", "url_status": "https://groqstatus.com/", "zdr_status": "https://console.groq.com/docs/your-data", "model_count": "https://api.groq.com/openai/v1/models", "url_pricing": "https://groq.com/pricing", "api_surfaces": "https://console.groq.com/docs/openai", "eu_residency": "https://console.groq.com/docs/your-data", "founded_year": "https://en.wikipedia.org/wiki/Groq", "product_fate": "https://groq.com/newsroom/groq-and-nvidia-enter-non-exclusive-inference-technology-licensing-agreement-to-accelerate-ai-inference-at-global-scale", "regions_note": "https://groq.com/platform", "vendor_status": "https://groq.com/newsroom/groq-closes-usd350-million-series-a-building-the-world-s-leading-ai-inference-cloud", "certifications": "https://console.groq.com/docs/legal/customer-data-processing-addendum", "config_as_code": "https://console.groq.com/docs/spend-limits", "retention_days": "https://console.groq.com/docs/your-data", "throughput_rps": "https://console.groq.com/docs/models", "url_models_api": "https://console.groq.com/docs/models", "cap_rate_limits": "https://console.groq.com/docs/rate-limits", "logs_by_default": "https://console.groq.com/docs/your-data", "model_count_low": "https://api.groq.com/openai/v1/models", "tracing_support": "https://console.groq.com/docs/arize", "model_count_high": "https://api.groq.com/openai/v1/models", "model_count_note": "https://api.groq.com/openai/v1/models", "eu_residency_note": "https://console.groq.com/docs/your-data", "groq_billing_faqs": "https://console.groq.com/docs/billing-faqs", "last_release_date": "https://console.groq.com/docs/changelog", "openai_compatible": "https://console.groq.com/docs/openai", "subprocessors_url": "https://console.groq.com/docs/legal/customer-data-processing-addendum", "groq_billing_plans": "https://console.groq.com/settings/billing/plans", "groq_prompt_caching": "https://console.groq.com/docs/prompt-caching", "guardrails_location": "https://console.groq.com/docs/your-data", "latency_source_note": "https://console.groq.com/docs/models", "requires_vendor_sdk": "https://console.groq.com/docs/openai", "self_host_cost_note": "https://console.groq.com/docs/models", "groq_pricing_marketing": "https://groq.com/pricing", "prompt_transits_vendor": "https://console.groq.com/docs/your-data", "model_count_source_note": "https://api.groq.com/openai/v1/models", "trains_on_customer_data": "https://groq.com/privacy-policy" }, "verified_at": { "zdr": "2026-08-29", "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-08-29", "cap_audio": "2026-08-29", "cap_batch": "2026-08-29", "free_tier": "2026-08-29", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "model_count": "2026-09-26", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "eu_residency": "2026-08-29", "founded_year": "2026-08-29", "config_as_code": "2026-08-29", "throughput_rps": "2026-08-29", "url_models_api": "2026-08-29", "cap_rate_limits": "2026-08-29", "model_count_low": "2026-09-26", "tracing_support": "2026-09-05", "model_count_high": "2026-09-26", "model_count_note": "2026-09-26", "eu_residency_note": "2026-08-29", "openai_compatible": "2026-08-29", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "model_count_source_note": "2026-09-26" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:17:00.111913+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Groq has not been acquired and says it will continue to operate as an independent company, having closed a $350M Series A in August 2026. But the shape of the company changed: in December 2025 Nvidia entered a non-exclusive inference-technology licensing agreement and hired founder Jonathan Ross, the president and other staff, with a new CEO taking over. Groq says GroqCloud will operate without interruption, though its public changelog has not been updated since April 2026.", "last_release_date": "2026-04-18", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": "Most recent GroqCloud changelog entry. Four months without a public update is an outlier among the actively developed products here.", "prompt_transits_vendor": "yes", "prompt_path_note": "Hosted inference. Where retention does occur, the data sits in Google Cloud buckets in the United States.", "logs_by_default": "none", "logs_note": "Groq does not retain customer data for inference requests by default. Usage metadata is collected for all users and contains no inputs or outputs. Temporary logging happens only to troubleshoot reliability-degrading errors or investigate suspected abuse.", "retention_days": 0, "retention_note": "Zero by default. Exceptions: up to 30 days for the reliability and abuse logs, 30 days for batch inputs and outputs, and fine-tuning artifacts held until you delete them.", "zdr_status": "yes", "trains_on_customer_data": "not_published", "training_note": "The data page is silent, and the privacy policy explicitly routes the question to the services agreement and data-processing addendum. Read that as silence, not a no.", "subprocessors_url": "https://trust.groq.com/subprocessors", "regions_note": "Retained data is US-only, in Google Cloud buckets. Inference itself spans 13 data centres on four continents including Finland, the UK, Saudi Arabia and Australia, but no customer region pinning is documented.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "the DPA commits to annual Type II audits; no report or portal retrieved", "status": "claimed" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "detail": "DPA published, with standard contractual clauses", "status": "certified" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "status": "not_published" } }, "guardrails_location": "none", "guardrails_note": "No Groq-operated guardrail or PII-redaction feature was found.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference on a free/Developer/Enterprise ladder; pay-as-you-go per token with progressive billing. No separate gateway or platform fee — routing/API layer is $0.", "min_commitment": "None stated. Bills in arrears; progressive billing thresholds at $1, $10, $100, $500 and $1,000 (India: $1, $10, then recurring $100). $0.50 minimum before billing.", "overage_note": "None stated. Downgrading stops usage charges immediately and returns the account to Free tier rate limits and restrictions.", "cache_support": "prefix", "cache_read_discount_pct": 50, "cache_write_premium_pct": 0, "cache_pricing_note": "Prefix prompt caching, free to use. 50% discount on cached input tokens; pricing discount applies only on successful cache hits. No cache-write charge. Clean win for the customer — no feature fee, no write premium — with one caveat: batch requests already receive a 50% discount on all tokens, no additional discount applied to cached tokens in batch requests. The two discounts do not stack.", "cost_controls": { "enforcement": "mixed", "spend_alerts": { "note": "Usage alerts against budget thresholds.", "tier": null, "supported": true }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Developer tier gives 'significantly increased rate limits for production workloads'. Platform-set; configurable limits not stated.", "tier": "developer", "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated as per-key.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Account-level Spend Limits supported: automated spending limits and proactive usage alerts as you approach budget thresholds. Listed among Developer-tier benefits.", "tier": null, "supported": true } }, "cost_attribution": "Not stated.", "cost_export": "Not stated.", "product_kind_note": "Groq presents GroqCloud as \"fast LLM inference, OpenAI-compatible\", where \"hosted models are directly accessible through the GroqCloud Models API endpoint\" ([Groq docs overview](https://console.groq.com/docs/overview); [Supported models](https://console.groq.com/docs/models)). It is a raw first-party inference API on Groq's own silicon — \"we pioneered the LPU\" ([Groq pricing](https://groq.com/pricing)).", "deployment_modes": [ "saas" ], "deployment_note": "No customer-installed component; usage is against `https://api.groq.com/openai/v1` with a Groq API key ([Groq docs overview](https://console.groq.com/docs/overview)). Capacity is Groq-operated: \"we're building hundreds of megawatts of capacity, with many more on the way\", and \"with LPX, it works alongside NVIDIA's next-generation GPUs\" ([Groq pricing](https://groq.com/pricing)). Tier gating shows up as rate limits under a \"DEVELOPER PLAN\" column, with some models' prices and limits marked \"Contact Sales\" ([Supported models](https://console.groq.com/docs/models)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "yes", "api_embeddings": "not_documented", "api_images": "not_documented", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap: \"We designed Groq API to be mostly compatible with OpenAI's client libraries\" — pass the Groq key as `api_key` and set `base_url` to `https://api.groq.com/openai/v1`; Groq also ships \"our own Groq Python and Groq TypeScript libraries that we encourage you to use\" ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)). Documented compatibility gaps: `logprobs`, `logit_bias`, `top_logprobs` and `messages[].name` are unsupported and \"will result in a 400 error\"; if `N` is supplied it must equal 1; `temperature` of 0 \"will be converted to `1e-8`\" (a float32 value > 0 and <= 2 is recommended); and audio `vtt` and `srt` response formats are not supported ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)). Model discovery is `GET https://api.groq.com/openai/v1/models`, returning \"a JSON list of all active models\" ([Supported models](https://console.groq.com/docs/models)).", "model_count_low": 11, "model_count_high": 11, "model_count_source_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party only: Groq runs its own inference on its own hardware (\"we pioneered the LPU\") and does not route to third-party provider APIs, so no provider count is published ([Groq pricing](https://groq.com/pricing); [Supported models](https://console.groq.com/docs/models)).", "first_party_vs_partner": "Vendor hardware serving open-weight third-party models: Groq hosts models such as Llama, GPT-OSS, Whisper, Qwen and MiniMax on its own LPU/LPX infrastructure and exposes them through its API; there is no third-party routing ([Supported models](https://console.groq.com/docs/models); [Groq pricing](https://groq.com/pricing)).", "custom_endpoint_support": "n.a. (no customer endpoint registration or custom-model upload documented on the pages fetched) ([Supported models](https://console.groq.com/docs/models)).", "byok_mode": "platform_credits_only", "merchant_of_record": "Groq: it publishes and charges the token, character and hourly prices for its hosted models (e.g. \"$0.15 input $0.60 output\" per 1M tokens, \"$40.00 per 1M characters\", \"$0.111 per hour\"), with batch at a 50% discount and enterprise items shown as \"Contact Sales\" ([Supported models](https://console.groq.com/docs/models); [Groq Batch API](https://console.groq.com/docs/batch)). The pricing page itself does not state markup, BYOK or credit mechanics: n.a. ([Groq pricing](https://groq.com/pricing)).", "key_management_note": "Single-vendor key model: requests carry a Groq API key in the `Authorization: Bearer` header, sourced from `GROQ_API_KEY` and created in the Groq Console (\"you can find your API key here\") ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Supported models](https://console.groq.com/docs/models)). No third-party provider secrets exist. Because inference is hosted, Groq necessarily processes request content; explicit retention or vault/KMS statements: n.a. on the pages fetched.", "api_openai_chat_completions_note": "Yes: OpenAI-compatible base URL `https://api.groq.com/openai/v1`, with `/v1/chat/completions` used as a batch endpoint target too ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch)).", "api_anthropic_messages_note": "n.a. (not documented on the pages fetched).", "api_openai_responses_note": "Yes: `POST https://api.groq.com/openai/v1/responses`, described as \"a more advanced interface for generating model responses\" supporting text and image inputs, stateful conversations using previous responses as context, and function calling ([Groq docs overview](https://console.groq.com/docs/overview); [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "api_embeddings_note": "n.a. (no embeddings endpoint documented on the pages fetched).", "api_images_note": "n.a. as generation; image inputs are supported by the Responses API and vision chat requests ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch)).", "api_audio_note": "Yes: \"Audio Transcription and Translation\" (STT) via `/v1/audio/transcriptions` and `/v1/audio/translations`, served by `whisper-large-v3` / `whisper-large-v3-turbo`; TTS-style models appear in the catalog (`canopylabs/orpheus-v1-english`, priced per 1M characters) ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch); [Supported models](https://console.groq.com/docs/models)).", "api_batch_note": "Yes: the Groq Batch API takes a JSONL file where \"each line in this file represents a single API call\", `POST` only, supporting `/v1/chat/completions`, `/v1/audio/transcriptions` and `/v1/audio/translations`, up to 50,000 lines and 200MB per file, up to 200 batch IDs per status query, multiple concurrent jobs, priced \"at a 50% cost discount compared to synchronous API pricing\", charging only successfully completed requests, and with rate limits separate from synchronous per-model limits ([Groq Batch API](https://console.groq.com/docs/batch)).", "deployment_modes_note": "hosted SaaS only (\"hosted models are directly accessible through the GroqCloud Models API endpoint\") ([Supported models](https://console.groq.com/docs/models)). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. on the pages fetched.", "byok_mode_note": "platform_credits_only in effect: usage is authenticated with a Groq API key (`GROQ_API_KEY`, `Authorization: Bearer $GROQ_API_KEY`) and priced per 1M tokens / per 1M characters / per hour by Groq, with no upstream provider keys to supply ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Supported models](https://console.groq.com/docs/models)).", "access_meta": { "qualified": [ "api_images" ], "model_count_kind": "enumerated", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "`not_documented` on the fetched Groq docs ([Groq text generation docs](https://console.groq.com/docs/text-chat)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([Groq text generation docs](https://console.groq.com/docs/text-chat)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": "`not_documented` ([Groq text generation docs](https://console.groq.com/docs/text-chat)).", "architecture_class": "vendor_saas", "architecture_note": "`vendor_saas` on the software side, with disclosed custom hardware: the LPU Inference Engine served through GroqCloud and the Groq API. No proxy runtime is disclosed ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).", "self_host_data_plane": "no", "self_host_artifact_note": "Only client SDKs are distributed (`pip install groq`, Groq Python and Groq TypeScript libraries) ([Groq text generation docs](https://console.groq.com/docs/text-chat), [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "latency_claims": [ { "url": "https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubling-down-on-performance", "unit": "s", "scope": "end_to_end", "value": "0.8", "metric": "total for 100 output tokens", "publisher": "independent", "conditions": "Llama 2 Chat 70B, ~100-token prompts generating ~200 tokens; estimated from latency and throughput, refreshed every three hours." } ], "throughput_claims": [ { "url": "https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubling-down-on-performance", "unit": "tokens/s", "scope": "generation_speed", "value": "241", "metric": "output speed", "publisher": "independent", "conditions": "Llama 2 Chat 70B, measured by ArtificialAnalysis.ai. Groq separately claims 300 tokens/s internally." } ], "streaming_support": "yes", "streaming_note": "Supported via `stream=True`, returning \"an iterator of completion deltas rather than a single, full completion\"; async streaming supported with `AsyncGroq`. Non-streaming caveats on the OpenAI-compatible surface: `logprobs`, `logit_bias`, `top_logprobs` and `messages[].name` are unsupported and return `400`; `N` must equal `1`; `temperature: 0` is converted to `1e-8`; audio transcription/translation do not support `vtt` or `srt` ([Groq text generation docs](https://console.groq.com/docs/text-chat), [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).", "guard_pii": "not_documented", "guard_pii_note": "`not_documented` — no PII guardrail feature ([Your data](https://console.groq.com/docs/your-data))", "guard_injection": "not_documented", "guard_injection_note": "Llama Prompt Guard 2 is offered as a model you call yourself ([Content moderation](https://console.groq.com/docs/content-moderation))", "guard_content": "not_documented", "guard_content_note": "`not_documented` as an inline platform control — moderation is available only as callable models: GPT-OSS-Safeguard 20B, Llama Guard 4, and Llama Prompt Guard 2. The application must make a second call and act on the verdict; Groq does not filter the request path ([Content moderation](https://console.groq.com/docs/content-moderation))", "guard_custom": "not_documented", "guard_custom_note": "GPT-OSS-Safeguard accepts a custom policy in its prompt, but this is a model capability rather than a gateway policy engine ([Content moderation](https://console.groq.com/docs/content-moderation))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages ([Your data](https://console.groq.com/docs/your-data))", "guard_execution_location": "not_documented", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([Content moderation](https://console.groq.com/docs/content-moderation))", "log_content": "metadata_only", "log_content_note": "`metadata_only` — request metadata is always retained; customer prompt/completion data is not retained by default, though abuse-detection logs may be held for up to 30 days ([Your data](https://console.groq.com/docs/your-data))", "log_opt_out": "yes", "log_opt_out_note": "Zero Data Retention is available through Data Controls; metadata retention is not optional ([Your data](https://console.groq.com/docs/your-data))", "tracing_support": "otel_integration", "tracing_note": "Via third-party OTel instrumentation: Arize Phoenix tracing is documented for Groq calls ([Arize + Groq](https://console.groq.com/docs/arize))", "export_destinations": [], "export_destinations_note": "Observability integrations are client-side: Arize, MLflow, and LangSmith ([Integrations](https://console.groq.com/docs/integrations)); Arize Phoenix specifically for OTel traces ([Arize + Groq](https://console.groq.com/docs/arize))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Integrations](https://console.groq.com/docs/integrations))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` from Groq; evaluation happens in the connected observability platform ([Integrations](https://console.groq.com/docs/integrations))", "observability_requires_saas": null, "log_retention_note": "Metadata retained (duration unspecified); abuse-related logs up to 30 days; no customer content retention by default ([Your data](https://console.groq.com/docs/your-data))", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "**0.8 seconds** total response time to receive 100 output tokens, \"estimated based on latency and throughput\", where latency is defined as \"Time to first tokens chunk received, in seconds, after API request sent\". Conditions: Llama 2 Chat (70B), unique prompts ~100 tokens generating ~200 output tokens, benchmark refreshed every three hours ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results), [Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).", "multi_surface": {}, "throughput_prose": "**241 tokens per second** on the Llama 2 Chat (70B) API as measured by ArtificialAnalysis.ai, \"more than double the speed of other hosting providers\"; throughput defined as tokens/sec received after the first chunk. Groq additionally states its internal benchmarks \"consistently\" reached **300 tokens per second**, and elsewhere claims customers run \"at 10x the speed\" with no value or conditions ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark), [Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results)).", "benchmark_independence": "Mixed, and the distinction matters: the 241 tok/s and 0.8 s figures originate from **independent third party ArtificialAnalysis.ai** but are reported on Groq's own pages; the 300 tok/s figure is **vendor internal**. The same third-party benchmark **includes competitors** (Microsoft Azure, Amazon Bedrock, Perplexity, Together.ai, Anyscale, Deepinfra, Fireworks, Lepton) — those rival numbers reach you via Groq's retelling, so verify against each rival before quoting ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "not_documented", "guard_custom_default": "not_documented", "rel_notes": "Groq is an inference provider with essentially no published reliability control surface: timeouts, retries, fallback, health and balancing are all your responsibility (or your gateway's). Also note the benchmark figures are dated Llama 2 70B-era results ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).", "guard_notes": "Groq is an inference provider, not a policy gateway: everything guardrail-shaped is a model in the catalogue and every observability integration is instrumentation you add in your own client ([Content moderation](https://console.groq.com/docs/content-moderation), [Integrations](https://console.groq.com/docs/integrations)).", "fit_agent_features_note": "The Responses API documents function calling, `code_interpreter`, `browser_search`, MCP tools, multi-step orchestration, exposed reasoning, tool discovery, and MCP approval flows using `mcp_approval_request` and `mcp_approval_response`. Groq’s `compound-beta` agentic tools system also supports code generation and execution, API calls, and webpage data retrieval; sessions, threads, and A2A are not documented. ([Groq remote MCP docs](https://console.groq.com/docs/tool-use/remote-mcp))", "fit_byok_required": "no", "fit_byok_required_note": "No. The first call uses `GROQ_API_KEY` ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Groq provides official Python and JavaScript/TypeScript libraries: Python package `groq` with the `Groq` class, and JavaScript/TypeScript package `groq-sdk` with the `Groq` class; both offer synchronous and asynchronous clients. Groq is also compatible with OpenAI’s client libraries, and its documentation shows Vercel AI SDK packages `ai` and `@ai-sdk/groq`. ([Groq client libraries docs](https://console.groq.com/docs/libraries))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "third_party_documented", "fit_cloudflare_workers_note": "Documented by Cloudflare, not Groq: Groq is a supported Cloudflare AI Gateway provider, with base URL `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/groq` replacing `https://api.groq.com/openai/v1`; no Workers binding is documented on that page. ([Cloudflare AI Gateway - Groq provider](https://developers.cloudflare.com/ai-gateway/usage/providers/groq/))", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented). No credit-card, free-tier or trial-credit wording appears on the quickstart, rate-limits, or pricing pages ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart), [Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits), [Groq On-Demand Pricing](https://groq.com/pricing))", "fit_ecosystem_notes": "Groq’s documentation lists Google Workspace Connectors for Gmail, Google Calendar, and Google Drive, along with integrations for external services. Its ecosystem also includes Vercel deployment documentation, the Groq developer community and API Cookbook, and MCP integrations with clients such as Claude Desktop, Cursor, and Windsurf. ([Groq platform overview](https://console.groq.com/docs/overview))", "fit_eval_prereqs": "A Groq API key, recommended as the `GROQ_API_KEY` environment variable. No account, credit card, cloud account, cluster or provider key is mentioned ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))", "fit_eval_shape": "sdk_install", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `client.chat.completions.create(messages=[{\"role\":\"user\",\"content\":\"Explain the importance of fast language models\"}], model=\"llama-3.3-70b-versatile\")` ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": null, "fit_kubernetes": "not_documented", "fit_kubernetes_note": "n.a. (not documented)", "fit_mcp": "mcp_tools_in_api", "fit_mcp_note": "Yes. Groq’s Responses API supports remote MCP over HTTPS using tools such as `type: \"mcp\"`, `server_label`, `server_url`, `headers`, `require_approval`, and `allowed_tools`; remote MCP is currently in beta. Groq also documents a `groq-mcp` MCP server package usable with Claude Desktop and other MCP clients. ([Groq remote MCP docs](https://console.groq.com/docs/tool-use/remote-mcp))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_credit_card", "fit_gateway_platform", "fit_kubernetes", "fit_model_gate", "fit_terraform" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented) ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart), [Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits))", "fit_onboarding_notes": "Rate limits are the practical gate rather than approvals; the Developer plan unlocks higher limits plus Batch and Flex processing ([Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits)). The quickstart also points to an onboarding video, the Playground, the developer community and the Groq API Cookbook ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Groq supports OpenAI’s client libraries when configured with the base URL `https://api.groq.com/openai/v1` and a Groq API key. ([Groq OpenAI compatibility docs](https://console.groq.com/docs/openai))", "fit_prod_prereqs": "n.a. (not documented) beyond rate-limit tiering: \"Need higher rate limits? Upgrade to Developer plan to access higher limits, Batch and Flex processing, and more.\" ([Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "sdk_install", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "Groq documents LangChain integration through `langchain-groq` and the `ChatGroq` class. LlamaIndex integration is also documented for Python and JavaScript, with the LlamaIndex page additionally documenting installation of LlamaIndex. ([Groq LangChain integration docs](https://console.groq.com/docs/langchain))", "fit_quickstart_steps": 0, "fit_quickstart_steps_caveat": "The quickstart is prose plus a code sample with no numbered procedure.", "fit_quickstart_url": "https://console.groq.com/docs/quickstart", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented)", "fit_terraform_ref": null, "fit_third_party_documented": { "cloudflare_workers": "Cloudflare" }, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get up and running with the Groq API in a few minutes\" ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Yes. Groq provides Vercel AI SDK integration through `@ai-sdk/groq` and `ai`; the provider can be imported from `@ai-sdk/groq` and used as `groq('llama-3.3-70b-versatile')`. ([Groq AI SDK integration docs](https://console.groq.com/docs/ai-sdk/))", "fit_vercel_ai_sdk_package": "@ai-sdk/groq" }, { "slug": "together-ai", "name": "Together AI", "company": "Together AI", "tagline": "Inference provider running open-weight models on its own GPUs, with fine-tuning and dedicated endpoints.", "category": "inference_provider", "deployment": "managed_only", "license": "Proprietary", "founded_year": 2022, "hq_country": "United States", "url_home": "https://www.together.ai/", "url_docs": "https://docs.together.ai/docs/introduction", "url_pricing": "https://www.together.ai/pricing", "url_status": "https://status.together.ai/", "url_github": null, "url_changelog": null, "url_models_api": "https://api.together.ai/v1/models", "github_stars": null, "model_count": 272, "model_count_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count": 40, "token_markup_pct": null, "credit_fee_pct": null, "credit_fee_min_usd": null, "byok_supported": null, "byok_fee_note": null, "seat_fee_usd_month": null, "free_tier": null, "enterprise_from_usd_year": null, "extra_fees": [ { "label": "Fine-tuning", "amount": "$0.48-$2.90 per 1M tokens with a $4.00 minimum per job" }, { "label": "Dedicated GPU endpoints", "amount": "H100 $5.49/hour, B200 $8.99/hour" }, { "label": "GPU clusters (on-demand)", "amount": "H100 $3.99, H200 $5.99, B200 $8.19 per GPU-hour" }, { "label": "Code Sandbox / Code Interpreter", "amount": "$0.0446 per vCPU, $0.03 per interpreter session" }, { "label": "Shared filesystem", "amount": "$0.16 per GiB per month" } ], "self_host_cost_note": "No self-host option; Together sets its own per-token prices rather than marking up another vendor's list price. Representative serverless rates include MiniMax M3 at $0.30 per 1M input, gpt-oss-120B at $0.15, DeepSeek V4 Flash at $0.14, and Kimi K3 at $3.00 input / $15.00 output. Discounted batch rates and provisioned throughput sold in PTUs are also published, plus embeddings from $0.02 per 1M tokens.", "openai_compatible": true, "requires_vendor_sdk": false, "data_export_available": null, "config_as_code": false, "portability_note": "Dashboard cost analytics only; no documented export.", "api_surfaces": [ "OpenAI Chat Completions", "Together Batch API", "Fine-tuning API", "Dedicated endpoints" ], "sdk_note": "Documented as an OpenAI-compatible API, so OpenAI SDKs work by changing the base URL; the model list endpoint returns id, type, organization, license, context_length and a pricing object.", "cap_failover": null, "cap_load_balancing": null, "cap_conditional_routing": null, "cap_cache_simple": null, "cap_cache_semantic": null, "cap_observability": null, "cap_prompt_management": null, "cap_evals": null, "cap_guardrails": true, "cap_pii_redaction": null, "cap_budgets": null, "cap_rate_limits": null, "cap_virtual_keys": null, "cap_mcp": null, "cap_embeddings": true, "cap_image_gen": true, "cap_audio": true, "cap_video": true, "cap_batch": true, "latency_overhead_ms": null, "latency_source_note": "Not a proxy, so there is no gateway overhead figure. Third-party measurements: a Telnyx-run streamed benchmark on GLM-5.2 (240 successful requests, 10 runs per provider, June 25 2026) recorded a 0.79 s median time-to-first-token for Together versus 1.28 s for Fireworks; a separate Python harness on Llama 3.1 8B reported roughly 250 tokens/sec median for Together versus 580 for Groq and 320 for Fireworks, with the author noting run-to-run variance.", "throughput_rps": null, "soc2": true, "hipaa_baa": true, "gdpr": null, "eu_residency": null, "eu_residency_note": null, "zdr": null, "zdr_note": "ZDR must be enabled by disabling prompt/response storage in organization privacy settings. It applies prospectively and disables passthrough models.", "air_gapped": null, "best_for": "Teams that want a broad open-weight catalog plus cheap fine-tuning and the option to move to dedicated GPUs on one vendor.", "strengths": [ "200+ open-weight models across text, vision, image, video, audio and embeddings in one OpenAI-compatible API", "Clear path from serverless to dedicated endpoints, GPU clusters and provisioned throughput as volume grows", "Inexpensive fine-tuning from $0.48 per 1M tokens with a $4 job minimum", "SOC 2 Type 2 completed and a public trust center, with HIPAA BAAs in place with partners", "Model list endpoint publishes per-model pricing, licence and context length for programmatic catalog checks" ], "weaknesses": [ "Single-source provider: no independent catalog of closed frontier models and no cross-provider routing or failover", "No published free tier, and third-party reporting says new accounts need a prepaid minimum balance", "GDPR posture, EU data residency and zero-data-retention are not documented on any vendor page found", "Serverless is multi-tenant, so throughput can vary with neighbouring load and model IDs can be rotated or deprecated" ], "avoid_if": "You need closed frontier models, cross-provider failover, or contractual EU data residency.", "reviews": [ { "url": "https://techcrunch.com/2026/07/01/neocloud-together-ai-raises-800m-leaps-to-8-3b-valuation/", "why": "TechCrunch situates Together's inference business in the neocloud market, explaining how its GPU-cluster economics and model-serving demand differ from routing platforms.", "date": "2026-07-01", "type": "written", "title": "Neocloud Together AI raises $800M, leaps to $8.3B valuation", "publisher": "TechCrunch" }, { "url": "https://www.youtube.com/watch?v=FybjHMBQV8M", "why": "Craig Smith interviews Together's VP of Research about the inference stack and infrastructure choices behind the hosted service.", "date": "2025-08-25", "type": "video", "title": "Inside Together AI: The Research Driving Next-Gen Open-Source Models", "publisher": "YouTube - Eye on AI" }, { "url": "https://www.youtube.com/watch?v=_GQfj3jhXVM", "why": "sentdex builds against Together's inference API hands-on - keys, streaming, prompt formatting, model parameters - and ships a TermGPT demo on top of it.", "date": "2023-12-24", "type": "video", "title": "Open Source AI Inference API w/ Together", "publisher": "YouTube - sentdex" }, { "url": "https://www.reddit.com/r/AI_Agents/comments/1ugzoe2/anyone_moved_away_from_togetherai_looking_for/", "why": "37-comment migration thread where users report latency inconsistency and failed model hosting on Together, and compare Featherless, Groq, Cerebras, DeepInfra and Fireworks.", "date": "2026-06-27", "type": "community", "title": "Anyone moved away from Together.ai? Looking for alternatives", "publisher": "Reddit - r/AI_Agents" }, { "url": "https://www.reddit.com/r/LocalLLaMA/comments/1qozrne/for_those_using_hosted_inference_providers/", "why": "27-comment thread with real spend and failure detail (e.g. ~$250/month pod bills, lost pod storage) comparing Together against other hosted inference providers.", "date": "2026-01-28", "type": "community", "title": "For those using hosted inference providers (Together, Fireworks, Baseten, RunPod, Modal) - what do you love and hate?", "publisher": "Reddit - r/LocalLLaMA" } ], "sources": { "name": "https://www.together.ai/pricing", "soc2": "https://www.together.ai/blog/soc-2-compliance", "company": "https://www.together.ai/about-us", "tagline": "https://docs.together.ai/docs/introduction", "sdk_note": "https://docs.together.ai/reference/models-1", "url_docs": "https://docs.together.ai/docs/introduction", "url_home": "https://www.together.ai/", "zdr_note": "https://docs.together.ai/docs/privacy-and-security", "cap_audio": "https://docs.together.ai/docs/serverless-models", "cap_batch": "https://www.together.ai/pricing", "cap_video": "https://docs.together.ai/docs/serverless-models", "hipaa_baa": "https://www.together.ai/blog/soc-2-compliance", "logs_note": "https://docs.together.ai/docs/privacy-and-security", "extra_fees": "https://www.together.ai/pricing", "hq_country": "https://www.forbes.com/companies/together-ai/", "url_status": "https://status.together.ai/", "zdr_status": "https://docs.together.ai/docs/privacy-and-security", "model_count": "https://api.together.ai/v1/models", "url_pricing": "https://www.together.ai/pricing", "api_surfaces": "https://docs.together.ai/docs/introduction", "founded_year": "https://www.forbes.com/companies/together-ai/", "product_fate": "https://www.together.ai/terms-of-service", "regions_note": "https://support.together.ai/articles/8079447813-eu-data-centers-and-dedicated-model-deployment", "cap_image_gen": "https://docs.together.ai/docs/serverless-models", "vendor_status": "https://www.reuters.com/legal/transactional/together-ai-raises-800-million-83-billion-valuation-2026-07-01/", "cap_embeddings": "https://www.together.ai/pricing", "cap_guardrails": "https://docs.together.ai/docs/serverless-models", "certifications": "https://www.together.ai/blog/soc-2-compliance", "config_as_code": "https://docs.together.ai/docs/billing-credits", "provider_count": "https://www.together.ai/models", "retention_days": "https://docs.together.ai/docs/privacy-and-security", "retention_note": "https://docs.together.ai/docs/privacy-and-security", "url_models_api": "https://docs.together.ai/reference/models-1", "logs_by_default": "https://docs.together.ai/docs/privacy-and-security", "model_count_low": "https://api.together.ai/v1/models", "tracing_support": "https://docs.together.ai/docs/dedicated-endpoints/monitoring", "model_count_high": "https://api.together.ai/v1/models", "model_count_note": "https://api.together.ai/v1/models", "together_pricing": "https://www.together.ai/pricing", "last_release_date": "https://docs.together.ai/docs/changelog", "openai_compatible": "https://docs.together.ai/docs/introduction", "log_retention_note": "https://docs.together.ai/docs/privacy-and-security", "guardrails_location": "https://docs.together.ai/docs/privacy-and-security", "latency_source_note": "https://markaicode.com/vs/together-ai-vs-fireworks-ai/", "requires_vendor_sdk": "https://docs.together.ai/docs/inference/openai-compatibility", "self_host_cost_note": "https://www.together.ai/pricing", "prompt_transits_vendor": "https://docs.together.ai/docs/privacy-and-security", "model_count_source_note": "https://api.together.ai/v1/models", "trains_on_customer_data": "https://docs.together.ai/docs/privacy-and-security", "together_serverless_overview": "https://docs.together.ai/docs/serverless/overview.md" }, "verified_at": { "name": "2026-08-29", "soc2": "2026-08-29", "company": "2026-08-29", "tagline": "2026-08-29", "sdk_note": "2026-08-29", "url_docs": "2026-08-29", "url_home": "2026-08-29", "zdr_note": "2026-09-05", "cap_audio": "2026-08-29", "cap_batch": "2026-08-29", "cap_video": "2026-08-29", "hipaa_baa": "2026-08-29", "logs_note": "2026-09-05", "extra_fees": "2026-08-29", "hq_country": "2026-08-29", "url_status": "2026-08-29", "zdr_status": "2026-09-05", "model_count": "2026-09-26", "url_pricing": "2026-08-29", "api_surfaces": "2026-08-29", "founded_year": "2026-08-29", "cap_image_gen": "2026-08-29", "cap_embeddings": "2026-08-29", "cap_guardrails": "2026-08-29", "config_as_code": "2026-08-29", "provider_count": "2026-08-29", "retention_days": "2026-09-05", "retention_note": "2026-09-05", "url_models_api": "2026-08-29", "logs_by_default": "2026-09-05", "model_count_low": "2026-09-26", "tracing_support": "2026-09-05", "model_count_high": "2026-09-26", "model_count_note": "2026-09-26", "openai_compatible": "2026-08-29", "log_retention_note": "2026-09-05", "latency_source_note": "2026-08-29", "requires_vendor_sdk": "2026-08-29", "self_host_cost_note": "2026-08-29", "model_count_source_note": "2026-09-26" }, "locked_fields": [], "published": true, "updated_at": "2026-09-26T04:17:00.252674+00:00", "vendor_status": "independent", "acquired_by": null, "acquired_date": null, "acquisition_terms": null, "product_fate": "Actively developed with near-daily platform changes. Raised an $800M Series C at an $8.3B valuation in July 2026. Its terms of service explicitly disclaim any uptime guarantee unless separately agreed in an order form.", "last_release_date": "2026-08-28", "sla_uptime_pct": null, "sla_url": null, "claim_basis": {}, "last_release_note": null, "prompt_transits_vendor": "yes", "prompt_path_note": "Third-party models hosted on Together run on its own infrastructure. Some models are offered as passthrough, meaning Together forwards your prompts and responses straight to the upstream provider — leave that toggle off if you do not want traffic leaving Together.", "logs_by_default": "full_content", "logs_note": "Prompts and responses are stored by default for product improvements. Organization administrators can disable storage to enable ZDR; training is a separate opt-in.", "retention_days": null, "retention_note": "Prompt storage is on by default; a retention duration is not published on the privacy page. Enabling organization ZDR stops persistence of future request content.", "zdr_status": "depends", "trains_on_customer_data": "opt_in", "training_note": "Data sharing for training is opt-in and not enabled by default; Together states it does not train on your data without explicit consent.", "subprocessors_url": null, "regions_note": "US by default. EU data centres exist for inference and storage inside the EU but only on Scale and Enterprise plans, serverless endpoints offer no region selection, and private networking and VPC-based deployments are supported.", "certifications": { "itar": { "status": "not_published" }, "soc2": { "detail": "a blog post reports a completed Type 2 examination; the trust center lists only Type 1", "status": "conflicting" }, "fedramp": { "status": "not_published" }, "gdpr_dpa": { "status": "not_published" }, "iso27001": { "status": "not_published" }, "hipaa_baa": { "detail": "adheres to HIPAA with BAAs signed with its partners, not offered to customers", "status": "alignment_only" } }, "guardrails_location": "none", "guardrails_note": "Deliberately none: models are hosted at full precision, and Together states it does not distil them, force system prompts or layer censorship on top.", "acquired_date_basis": null, "security_incidents": [], "pricing_model": "bundled_inference", "pricing_model_note": "Pay-for-own-inference across serverless per-token, Batch API, Provisioned Throughput (PTUs), dedicated single-tenant endpoints, GPU clusters, sandbox and storage. Enterprise quote for some SKUs. No separate gateway/platform fee — routing layer is $0.", "min_commitment": "None for on-demand serverless. Fine-tuning has a $4.00 minimum charge per job. PTU savings estimates assume continuous 24/7 provisioning (~43,800 min/mo) but no commitment is required.", "overage_note": "None stated. Cost risks are structural — dedicated endpoints and PTUs bill for provisioned capacity whether or not you use it; storage is billed per GiB/month.", "cache_support": "prefix", "cache_read_discount_pct": null, "cache_write_premium_pct": 0, "cache_pricing_note": "Automatic prefix caching on select serverless models, best-effort, plus per-replica caching on dedicated endpoints. Prefix-based: only the longest matching prefix counts. Serverless cache is shared across the fleet and entries are evicted as traffic shifts — cache hits aren't guaranteed and there's no configurable retention window. Docs say 'a steep discount', pricing table publishes numbers: MiniMax M3 $0.06 vs $0.30 (80% off), Kimi K3 $0.30 vs $3.00 (90% off), DeepSeek V4 Flash $0.03 vs $0.14 (~79% off), GLM-5.3 $0.26 vs $1.40 (~81% off), Qwen3.8-2.4T-A95B $0.25 vs $2.00 (87.5% off). No cache-write charge. Only dedicated endpoints make the saving reliable.", "cost_controls": { "enforcement": "not_stated", "spend_alerts": { "note": "Not stated.", "tier": null, "supported": null }, "virtual_keys": { "note": "Not stated.", "tier": null, "supported": null }, "rate_limiting": { "note": "Exists on serverless models but values and configurability not stated.", "tier": null, "supported": true }, "spend_webhooks": { "note": "Not stated.", "tier": null, "supported": null }, "model_allowlists": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_key": { "note": "Not stated.", "tier": null, "supported": null }, "budget_caps_per_team": { "note": "Not stated.", "tier": null, "supported": null } }, "cost_attribution": "Not stated.", "cost_export": "Not stated.", "product_kind_note": "Together AI's docs say \"Run, train, and serve open-source AI models on Together AI\" via \"our OpenAI-compatible API\", also described as the \"Together AI OpenAI compatibility layer\" ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)). It is a first-party inference, fine-tuning and GPU platform, not a router.", "deployment_modes": [ "saas" ], "deployment_note": "Serverless requires no provisioning and is charged on \"the tokens you use and the size of the model\"; dedicated endpoints serve \"a single model on hardware reserved only for you\", bill \"per-minute by hardware while the endpoint is running, regardless of your model or request volume\", bill each replica independently and stop billing \"as soon as they are scaled down\" ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)). Hardware options listed for dedicated inference are NVIDIA HGX H100, H200, B200, B300, GB200 NVL72 and GB300 NVL72, with GPU clusters \"pay as you go… on an hourly basis\" ([Together AI pricing](https://www.together.ai/pricing)). Migration is code-free: \"prototype with serverless, then switch to dedicated endpoints without changing your application code\", since \"dedicated endpoints use the same inference APIs as serverless models\" ([Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).", "api_openai_chat_completions": "yes", "api_anthropic_messages": "not_documented", "api_openai_responses": "not_documented", "api_embeddings": "yes", "api_images": "yes", "api_audio": "yes", "api_batch": "yes", "api_surface_note": "Drop-in base-URL swap: existing OpenAI Python/TypeScript or curl code moves over with \"two changes: the API key and base URL\" (`https://api.together.ai/v1`); first-party SDKs (`together`, `together-ai`) exist but are optional ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)). Documented gaps: `assistants.*`/`threads.*`/`runs.*` not supported (build agent loops with chat completions and function calling), OpenAI-shaped `fine_tuning.jobs.*` not supported (use Together's native fine-tuning API), `files.*` only partial (Together has its own Files API), `batches.*` not supported (native Batch API), `moderations.create` not supported (use Llama Guard through chat completions); `GET /v1/models` is supported for listing/retrieval ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).", "model_count_low": 272, "model_count_high": 272, "model_count_source_note": "Count of entries returned by the models API on 2026-09-26. Includes every entry exposed by that endpoint; not a count of unique base models.", "provider_count_low": null, "provider_count_high": null, "provider_count_source_note": "First-party only: \"Together AI hosts many popular models\" on its own serverless and dedicated infrastructure and does not route to third-party provider APIs, so no provider count is published ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).", "first_party_vs_partner": "Vendor-operated hardware serving open-weight third-party models: models are \"available via serverless or dedicated model inference\", a dedicated endpoint serves \"a single model on hardware reserved only for you\", and the \"Available models\" list is described as \"Together-hosted models you can deploy on dedicated endpoints\" ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)). Whether the GPUs are owned or leased is not stated: n.a. ([Together AI pricing](https://www.together.ai/pricing)).", "custom_endpoint_support": "Bring-your-own-weights, not bring-your-own-endpoint: \"Bring your own model — upload weights and serve them via the Together AI API\", and dedicated endpoints can serve \"custom fine-tuned models\" / uploaded \"own model weights\"; registering an external private URL, vLLM or SageMaker endpoint is not documented ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).", "byok_mode": "platform_credits_only", "merchant_of_record": "Together AI: serverless usage is charged on \"the tokens you use and the size of the model\", dedicated inference on GPU hours / per-minute hardware time, GPU clusters hourly, with separate published prices for audio minutes, TTS characters, batch, fine-tuning, sandboxes and storage ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints); [Together AI pricing](https://www.together.ai/pricing)).", "key_management_note": "Single-vendor key model: requests authenticate with `Authorization: Bearer $TOGETHER_API_KEY` obtained from Together's hosted API settings; no third-party provider secrets are stored ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)). Because inference is hosted, Together processes request content; explicit statements on retention, encryption, KMS or vendor visibility: n.a. on the pages fetched.", "api_openai_chat_completions_note": "Yes: `POST https://api.together.ai/v1/chat/completions`, including vision inputs, tools/function calling and `response_format` structured outputs ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)).", "api_anthropic_messages_note": "n.a. (explicitly not mentioned in the OpenAI-compatibility matrix) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).", "api_openai_responses_note": "n.a. (not mentioned) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).", "api_embeddings_note": "Yes: `POST /v1/embeddings` ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).", "api_images_note": "Yes: `POST /v1/images/generations` (with streaming image generation, e.g. \"stream real-time image generations with FLUX Schnell\") ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)).", "api_audio_note": "Yes, both directions: `POST /v1/audio/speech` (TTS), `POST /v1/audio/transcriptions` and `POST /v1/audio/translations` (STT), with per-audio-minute ASR pricing (Nemotron 3 ASR Streaming 0.6B, Parakeet TDT 0.6B v3, Nemotron 3.5 ASR, Whisper Large v3 and its streaming variant) and per-1M-character TTS pricing (Parakeet TDT 0.6B V3 Realtime, Cartesia Sonic-3, Orpheus TTS, Kokoro-82M TTS, Cartesia Sonic-2) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI pricing](https://www.together.ai/pricing)).", "api_batch_note": "Yes, but native rather than OpenAI-shaped: \"queue async generations and fetch results later\" via \"Together's native Batch API\", while `batches.*` in the OpenAI shape is \"not supported\"; batch prices are listed as \"Batch API price\" on the pricing page ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI pricing](https://www.together.ai/pricing)).", "deployment_modes_note": "hosted cloud service in several shapes: serverless inference, dedicated single-tenant model inference (\"provision single-tenant GPUs for predictable, isolated latency\"), dedicated containers \"on managed GPUs with autoscaling and observability\", GPU clusters (\"spin up H100 and B200 clusters with attached storage for training or large batch jobs\"), plus Sandbox VM environments ([Together AI docs overview](https://docs.together.ai/intro); [Together AI pricing](https://www.together.ai/pricing)). Self-host in customer infrastructure, hybrid/VPC, on-prem, air-gapped: n.a. on the pages fetched.", "byok_mode_note": "platform_credits_only in effect: all usage runs on a Together API key (`Authorization: Bearer $TOGETHER_API_KEY`) and is billed by Together; there are no upstream provider keys to supply ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).", "access_meta": { "qualified": [ "api_audio", "api_batch" ], "model_count_kind": "enumerated", "provider_count_kind": null }, "rel_timeout_config": "not_documented", "rel_timeout_note": "Timeout appears only as a documented response code: `504 Timeout` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "rel_retry_config": "not_documented", "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "rel_retry_default_note": null, "rel_fallback_config": "not_documented", "rel_fallback_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "rel_fallback_shape": "not_documented", "rel_health_check_config": "not_documented", "rel_health_check_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "rel_multi_region_config": "not_documented", "rel_multi_region_note": "Dedicated endpoints on Together Cloud are documented, but no regional placement or cross-region failover ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).", "rel_load_balancing_config": "not_documented", "rel_load_balancing_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "architecture_class": "vendor_saas", "architecture_note": "Serverless and dedicated endpoints at `api.together.ai`; no runtime disclosed beyond the named inference engine ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).", "self_host_data_plane": "no", "self_host_artifact_note": "Only client SDKs (`from together import Together`; `import Together from \"together-ai\"`) ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "latency_claims": [], "throughput_claims": [ { "url": "https://www.together.ai/blog/together-inference-engine-2", "unit": "tokens/s", "scope": "generation_speed", "value": "over 400", "metric": "decoding throughput", "publisher": "vendor", "conditions": "Meta Llama 3 8B on the Together Inference Engine 2.0." } ], "streaming_support": "yes", "streaming_note": "Supported. `stream` (boolean): when true it will \"stream tokens as Server-Sent Events as the model generates them\", and \"The stream terminates with `data: [DONE]`\"; the `200` response supports both `application/json` and `text/event-stream`, with schema `ChatCompletionStream` = `ChatCompletionEvent` | `StreamSentinel` and chunk type `chat.completion.chunk`. No streaming caveat is stated ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).", "guard_pii": "not_documented", "guard_pii_note": "`not_documented` ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))", "guard_injection": "not_documented", "guard_injection_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))", "guard_content": "sync_block", "guard_content_note": "`sync_block`-style filtering via an inline request parameter — adding `\"safety_model\": \"Meta-Llama/Llama-Guard-7b\"` to a chat completion makes Together run the safety model and filter the response before returning it. The control is per request, chosen by the caller, not an administrator policy ([Safety models](https://www.together.ai/blog/safety-models))", "guard_custom": "not_documented", "guard_custom_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))", "guard_model_policy": "not_documented", "guard_model_policy_note": "`not_documented` on the fetched pages ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))", "guard_execution_location": "vendor_saas", "guard_third_party": [], "guard_failure_mode": "not_documented", "guard_failure_mode_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))", "log_content": "metadata_only", "log_content_note": "`metadata_only` — zero data retention is the default; organisation-level toggles exist for storing prompts, allowing training, and provider passthrough ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))", "log_opt_out": "yes", "log_opt_out_note": "`yes` — prompt storage is an opt-in organisation setting, so the default already keeps metrics without bodies ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))", "tracing_support": "not_documented", "tracing_note": "`n.a.` — no OpenTelemetry or agent tracing documented; the endpoint feed is an audit event stream, not request traces ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))", "export_destinations": [], "export_destinations_note": "A Prometheus-compatible metrics endpoint, `GET https://o11y-de2-metrics.cloud.together.ai/organizations/{org_id}/metrics`, working with Prometheus, Grafana Agent, the Datadog OpenMetrics integration, and Vector; it is in beta and \"access may need to be enabled for your organization\". Events are also readable via `tg beta endpoints events` with `--json` ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))", "feedback_capture": "no", "feedback_capture_note": "`n.a.` ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))", "eval_hooks": "no", "eval_hooks_note": "`n.a.` — the API exposes `logprobs` for scoring outputs client-side, but no eval service on logged traffic ([Logprobs](https://docs.together.ai/docs/logprobs))", "observability_requires_saas": null, "log_retention_note": "Default prompt storage has no published duration on the reviewed privacy page. Organization ZDR disables future content storage; endpoint event retention is not specified. [Privacy and security](https://docs.together.ai/docs/privacy-and-security).", "production_meta": { "sources": {}, "qualified": [], "latency_prose": "`n.a.` No latency-metric figure (no TTFT, p50, p99 or overhead) is published on the fetched pages; all published numbers are throughput/speedup ratios ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).", "multi_surface": {}, "throughput_prose": "All from the engine announcement, metric = decoding throughput (tokens/sec) unless noted: - \"over **400 tokens per second** on Meta Llama 3 8B\" - \"decoding throughput **4x faster** than open-source vLLM\" - Together Turbo: \"up to **4.5x** performance improvement over vLLM (version 0.5.1)\" on Llama-3-8B-Instruct and Llama-3-70B-Instruct; **2.8x–4.5x** and **2.6x–4.3x** decoding speedup respectively, with Llama-3-8B-Instruct on a single H100 and Llama-3-70B-Instruct on 8×H100 - Together Turbo: \"up to **7x the capacity** of vLLM (version 0.5.1)\" and up to 7x cost reduction; on 4×H100 Turbo lands \"within 8%\" of Together Reference on 8×H100 - Together Lite: \"**12x reduction in cost**\" vs vLLM, and on two A100s it \"outperforms vLLM FP16 and FP8 running on eight H100 GPUs by up to **30%**\" - Together Reference: \"**4x speedup** over the state-of-the-art inference engine vLLM across normal serving regimes\" Payload/prompt length, concurrency and cache conditions are not stated for any of these ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).", "benchmark_independence": "All vendor self-published, and **heavily comparative against rivals**: vLLM 0.5.1 (open source), and the claim that the engine \"outperforms commercial solutions including Amazon Bedrock, Azure AI, Fireworks, and Octo AI by 1.3x to 2.5x\", with Together Reference \"over 2x faster tokens per second than Amazon Bedrock, Microsoft Azure, or Octo AI; and over 30% faster than Fireworks FP8 models\". Those competitor figures are vendor claims about rivals and are not verifiable from the rivals' own docs ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)). Together also appears as a compared provider in the third-party ArtificialAnalysis.ai benchmark reported by Groq ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))." }, "guard_pii_default": "not_documented", "guard_injection_default": "not_documented", "guard_content_default": "choose_at_setup", "guard_custom_default": "not_documented", "rel_notes": "Quality/speed tiering (Turbo / Lite / Reference) is the main reliability-adjacent lever, and the published comparisons are ratios against vLLM rather than absolute latency, which makes them hard to translate into an SLO. Streaming ASR models are listed separately in the serverless catalog ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2), [Together serverless models](https://docs.together.ai/docs/serverless-models)).", "guard_notes": "Together's observability is infrastructure-shaped, not request-shaped: edge/router/worker metric layers, latency histograms, and `_total` counters for dedicated endpoints, with no per-request log of prompts to inspect ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring)).", "fit_agent_features_note": "Together supports function calling through `chat.completions.create` with `tools` and `tool_choice`. `assistants.*`, `threads.*`, and `runs.*` are not supported; the documentation instructs users to build agent loops on top of chat completions and function calling. ([Together OpenAI compatibility docs](https://docs.together.ai/docs/inference/openai-compatibility))", "fit_byok_required": "no", "fit_byok_required_note": "No. The first call uses `TOGETHER_API_KEY` ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))", "fit_client_sdk_langs": [ "Python", "TypeScript" ], "fit_client_sdks_note": "Together documents official Python and TypeScript SDKs: Python package `together` and TypeScript npm package `together-ai`. Its integration documentation also covers Hugging Face with `huggingface_hub` and `InferenceClient` or `@huggingface/inference` and `HfInference`, LangChain with `langchain-together` and `ChatTogether`, and LlamaIndex with `llama-index` and `OpenAILike`. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))", "fit_cloud_iam": "not_documented", "fit_cloud_iam_note": "n.a. (not documented)", "fit_cloudflare_workers": "not_documented", "fit_cloudflare_workers_note": "n.a. (not documented)", "fit_credit_card": "not_documented", "fit_credit_card_note": "n.a. (not documented). Neither the quickstart nor the pricing page states a credit-card requirement, free tier, or trial credits ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))", "fit_ecosystem_notes": "Together documents a REST API at `https://api.together.ai/v1`, an OpenAI SDK-compatible interface, dedicated model inference, fine-tuning, and GPU clusters. Its integration documentation lists Hugging Face, Vercel AI SDK, LangChain, LlamaIndex, Helicone, CrewAI, LangGraph, DSPy, PydanticAI, AutoGen (AG2), Agno, Composio, and RAG integrations; Together-native endpoints include video generation, image edits and inpainting beyond `images.generate`, reasoning controls and `reasoning_content`, and the logprobs surface that are not exposed by the OpenAI SDKs. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))", "fit_eval_prereqs": "None stated separately; account registration is folded into step 1. No credit card, cloud account, cluster or provider key is mentioned ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))", "fit_eval_shape": "sdk_install", "fit_eval_snippet": "yes", "fit_eval_snippet_note": "Yes: `client.chat.completions.create(model=\"MiniMaxAI/MiniMax-M3\", messages=[{\"role\":\"user\",\"content\":\"What are the top 3 things to do in New York?\"}])`, run from a saved file or directly via cURL ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))", "fit_gateway_platform": "not_documented", "fit_gateway_platform_note": "n.a. (not documented)", "fit_helm_chart": "only a third-party Volcano scheduler chart on Together GPU clusters", "fit_kubernetes": "generic_platform_only", "fit_kubernetes_note": "No Helm chart or operator is documented for Together's inference gateway itself; Together's Kubernetes documentation covers installing the third-party Volcano scheduler chart (`volcano-sh/volcano` from `https://volcano-sh.github.io/helm-charts`) on Together GPU clusters. ([Together docs - Volcano on GPU clusters](https://docs.together.ai/docs/volcano-on-gpu-clusters))", "fit_mcp": "docs_mcp_only", "fit_mcp_note": "Together AI documents a docs MCP server at `https://docs.together.ai/mcp` (server ids `together-docs` / `TogetherAIDocs`) that lets coding agents query Together's documentation; an MCP gateway/registry or tool proxying for inference traffic is not documented there. ([Together AI docs - agent skills / MCP server](https://docs.together.ai/docs/agent-skills))", "fit_meta": { "not_documented_fields": [ "fit_cloud_iam", "fit_cloudflare_workers", "fit_credit_card", "fit_gateway_platform", "fit_model_gate", "fit_terraform" ] }, "fit_model_gate": "not_documented", "fit_model_gate_note": "n.a. (not documented) ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))", "fit_onboarding_notes": "\"New API keys are shown only once; save the value safely.\" Optional `uv init --no-workspace` scaffolding is shown, and \"After a few seconds, you should see the response printed to your terminal.\" Python and TypeScript SDKs plus OpenAI-SDK and REST paths are offered ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))", "fit_openai_dropin": "yes", "fit_openai_dropin_note": "Yes. Together documents OpenAI SDK compatibility by changing the API key and setting the base URL to `https://api.together.ai/v1`; the rest of the code can remain the same. ([Together OpenAI compatibility docs](https://docs.together.ai/docs/inference/openai-compatibility))", "fit_prod_prereqs": "n.a. (not documented) ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))", "fit_prod_selfhost_shape": null, "fit_prod_shape": "sdk_install", "fit_python_frameworks": [ "LangChain", "LlamaIndex" ], "fit_python_frameworks_note": "Together documents integrations for LangChain using the `langchain-together` package and `ChatTogether`, and for LlamaIndex using `llama-index`, `OpenAILike`, and dedicated Together embedding classes. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))", "fit_quickstart_steps": 3, "fit_quickstart_steps_caveat": null, "fit_quickstart_url": "https://docs.together.ai/docs/quickstart", "fit_self_host_docs": "no_self_host", "fit_self_host_install": null, "fit_terraform": "not_documented", "fit_terraform_note": "n.a. (not documented)", "fit_terraform_ref": null, "fit_third_party_documented": { "kubernetes": "the Volcano project" }, "fit_vendor_time_claim": "Vendor claim, verbatim: \"Make your first request to Together AI in a few minutes.\" ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))", "fit_vercel_ai_sdk": "official_provider", "fit_vercel_ai_sdk_note": "Together documents an official Vercel AI SDK provider package, `@ai-sdk/togetherai`, installable with `npm i ai @ai-sdk/togetherai`. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))", "fit_vercel_ai_sdk_package": "@ai-sdk/togetherai" } ] }