{
  "meta": {
    "title": "GatewayScore",
    "last_updated": "2026-08-31T04:37:47.93379+00:00",
    "source": "supabase",
    "provider_count": 20,
    "field_count": 113,
    "license": "Catalog values are facts published by each vendor. Reuse freely; please link back and re-verify before relying on any figure.",
    "methodology": "https://gatewayscore.com/methodology",
    "caveats": [
      "A null value means the vendor has not published the figure. It does not mean \"no\".",
      "Latency and throughput figures are usually vendor self-published benchmarks.",
      "Model counts are not comparable across vendors — counting conventions differ.",
      "Each value carries its source URL in the `sources` object and its check date in `verified_at`."
    ]
  },
  "providers": [
    {
      "slug": "openrouter",
      "name": "OpenRouter",
      "company": "OpenRouter, Inc.",
      "tagline": "Hosted marketplace that routes one OpenAI-compatible API to models from many inference providers.",
      "category": "managed_marketplace",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2023,
      "hq_country": "United States",
      "url_home": "https://openrouter.ai",
      "url_docs": "https://openrouter.ai/docs/faq",
      "url_pricing": "https://openrouter.ai/pricing",
      "url_status": "https://status.openrouter.ai",
      "url_github": null,
      "url_changelog": "https://openrouter.ai/docs/changelog",
      "url_models_api": "https://openrouter.ai/api/v1/models",
      "github_stars": null,
      "model_count": 500,
      "model_count_note": "Pricing page advertises 500+ models on pay-as-you-go and Enterprise; a live call to https://openrouter.ai/api/v1/models on 2026-08-29 returned 396 catalog entries across 58 creator prefixes.",
      "provider_count": 80,
      "token_markup_pct": 0,
      "credit_fee_pct": 5.5,
      "credit_fee_min_usd": 0.8,
      "byok_supported": true,
      "byok_fee_note": "Free up to $25,000/mo of list-price inference (pay-as-you-go) or $200,000/mo (Enterprise), then 5% of the equivalent OpenRouter cost; measured by list-price cost, not request count.",
      "seat_fee_usd_month": null,
      "free_tier": "Free tier with 25+ free models and 50 free-model requests/day; 1,000/day after purchasing at least $10 in credits.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Crypto (Coinbase) credit purchase fee",
          "amount": "5%"
        },
        {
          "label": "Card credit purchase fee minimum",
          "amount": "$0.80 per purchase"
        }
      ],
      "self_host_cost_note": null,
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": false,
      "portability_note": "Aggregated usage export (CSV/PDF), not per-request logs.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenRouter Management API"
      ],
      "sdk_note": "OpenAI-compatible endpoint plus a TypeScript SDK and a Management API under /api/v1/keys for provisioning per-customer keys.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": false,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": false,
      "cap_pii_redaction": false,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": true,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": true,
      "cap_audio": null,
      "cap_video": true,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No vendor-published overhead figure. Third-party estimates diverge: Respan cites roughly 25-40 ms over direct provider calls, TrueFoundry cites about 40 ms typical production overhead, mcp.directory estimates a 100-150 ms hop, and an interleaved TTFT benchmark by LLM Gateway (2026-07-22) measured OpenRouter median cold TTFT of 1392 ms end-to-end rather than isolated gateway overhead.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": null,
      "gdpr": true,
      "eu_residency": true,
      "eu_residency_note": "In-region routing in the EU and US for enterprise accounts on request via https://eu.openrouter.ai and https://us.openrouter.ai; prompts and completions stay in the selected region.",
      "zdr": true,
      "zdr_note": "Zero data retention is OpenRouter's own policy on all accounts with no plan gate, and can be enforced globally, per model group, per guardrail or per request.",
      "air_gapped": null,
      "best_for": "Teams that want the broadest possible model and provider catalog behind one OpenAI-compatible key with unified billing.",
      "strengths": [
        "Largest published catalog: 500+ models, 80+ providers",
        "No token markup; provider list prices pass through",
        "Public models endpoint with per-model pricing",
        "ZDR routing, EU/US in-region routing, SOC 2 and GDPR posture",
        "Per-key credit limits with daily/weekly/monthly resets"
      ],
      "weaknesses": [
        "5.5% credit-purchase fee (min $0.80) makes small top-ups expensive - $5 buys $5 of credit for $5.80",
        "BYOK documentation describes two different meters ($25k list-price cost vs 1M requests)",
        "No native content guardrails or PII redaction",
        "Managed only - no self-hosting or VPC option"
      ],
      "avoid_if": "You need self-hosting, air-gapped deployment, or a gateway that charges nothing on credit purchases.",
      "reviews": [
        {
          "url": "https://techcrunch.com/2026/08/19/stripe-didnt-really-buy-openrouter-because-of-the-singularity/",
          "why": "TechCrunch's reporters analyse why a payments company paid $7B+ for an inference router, arguing the real prize is OpenRouter's position in metering and billing AI spend rather than model access.",
          "date": "2026-08-19",
          "type": "written",
          "title": "Stripe didn't really buy OpenRouter because of the 'singularity'",
          "publisher": "TechCrunch"
        },
        {
          "url": "https://dev.to/goose_oss/how-openrouter-unlocked-our-workshop-strategy-5082",
          "why": "Rizel Scarlett explains how the Goose team used OpenRouter to hand out one key covering many models for live workshops, including the practical limits they hit.",
          "date": "2025-07-30",
          "type": "written",
          "title": "How OpenRouter Unlocked Our Workshop Strategy",
          "publisher": "DEV Community (Goose / Block open-source team)"
        },
        {
          "url": "https://www.youtube.com/watch?v=fjd2hm6-qtM",
          "why": "Hands-on walkthrough on an established data/AI teaching channel: API-key creation, OpenAI-SDK compatibility, auto-router, model rankings, logs and credit tracking.",
          "date": "2026-03-25",
          "type": "video",
          "title": "What is OpenRouter | All about OpenRouter in 10 minutes",
          "publisher": "YouTube - codebasics"
        },
        {
          "url": "https://www.youtube.com/watch?v=DBgINesX__k",
          "why": "Datadog's Ryan MacLean demos OpenRouter's request builder, curl/jq scripting, pricing and privacy settings, and a live chat app against multiple providers.",
          "date": "2025-09-11",
          "type": "video",
          "title": "OpenRouter: Unified LLM Interface and Model Orchestration",
          "publisher": "YouTube - Datadog Community"
        },
        {
          "url": "https://news.ycombinator.com/item?id=49330139",
          "why": "Practitioner thread on the acquisition where commenters discuss shifting production load onto and away from OpenRouter, credits, and self-host/migration options.",
          "date": "2026-08-17",
          "type": "community",
          "title": "Stripe to Buy OpenRouter for $7B",
          "publisher": "Hacker News"
        },
        {
          "url": "https://www.reddit.com/r/LocalLLaMA/comments/1jydnif/anyone_use_openrouter_in_production/",
          "why": "21-comment thread with real numbers: ~2M tokens/day in production and 2-3 overload errors per 100 requests on direct Anthropic versus OpenRouter fallbacks to Bedrock and Vertex.",
          "date": "2025-04-13",
          "type": "community",
          "title": "Anyone use openrouter in production?",
          "publisher": "Reddit - r/LocalLLaMA"
        }
      ],
      "sources": {
        "zdr": "https://openrouter.ai/enterprise",
        "gdpr": "https://openrouter.ai/enterprise",
        "soc2": "https://openrouter.ai/enterprise",
        "company": "https://en.wikipedia.org/wiki/OpenRouter",
        "license": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "sdk_note": "https://openrouter.ai/docs/features/provisioning-api-keys",
        "url_docs": "https://openrouter.ai/docs/faq",
        "zdr_note": "https://openrouter.ai/docs/faq",
        "cap_video": "https://openrouter.ai/models",
        "free_tier": "https://openrouter.ai/pricing",
        "deployment": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "extra_fees": "https://openrouter.ai/docs/faq",
        "hq_country": "https://en.wikipedia.org/wiki/OpenRouter",
        "url_status": "https://status.openrouter.ai",
        "zdr_status": "https://openrouter.ai/docs/guides/features/zdr",
        "acquired_by": "https://stripe.com/newsroom/news/stripe-agrees-to-acquire-openrouter",
        "cap_budgets": "https://openrouter.ai/pricing",
        "model_count": "https://openrouter.ai/pricing",
        "url_pricing": "https://openrouter.ai/pricing",
        "api_surfaces": "https://openrouter.ai/docs/features/provisioning-api-keys",
        "cap_failover": "https://openrouter.ai/enterprise",
        "eu_residency": "https://openrouter.ai/docs/features/privacy-and-logging",
        "founded_year": "https://en.wikipedia.org/wiki/OpenRouter",
        "product_fate": "https://openrouter.ai/blog/announcements/openrouter-is-joining-stripe/",
        "regions_note": "https://openrouter.ai/enterprise",
        "acquired_date": "https://www.reuters.com/technology/payments-firm-stripe-buy-ai-developer-platform-openrouter-2026-08-19/",
        "byok_fee_note": "https://openrouter.ai/pricing",
        "cap_image_gen": "https://openrouter.ai/models",
        "url_changelog": "https://openrouter.ai/docs/changelog",
        "vendor_status": "https://openrouter.ai/blog/announcements/openrouter-is-joining-stripe/",
        "byok_supported": "https://openrouter.ai/docs/use-cases/byok",
        "cap_guardrails": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "certifications": "https://openrouter.ai/enterprise",
        "config_as_code": "https://openrouter.ai/docs/guides/features/presets",
        "credit_fee_pct": "https://openrouter.ai/docs/faq",
        "openrouter_faq": "https://openrouter.ai/docs/faq",
        "provider_count": "https://openrouter.ai/pricing",
        "retention_days": "https://openrouter.ai/docs/guides/privacy/data-collection",
        "url_models_api": "https://openrouter.ai/docs/faq",
        "cap_rate_limits": "https://openrouter.ai/pricing",
        "logs_by_default": "https://openrouter.ai/docs/guides/privacy/data-collection",
        "openrouter_byok": "https://openrouter.ai/docs/use-cases/byok",
        "cap_cache_simple": "https://openrouter.ai/pricing",
        "cap_virtual_keys": "https://openrouter.ai/docs/features/provisioning-api-keys",
        "model_count_note": "https://openrouter.ai/api/v1/models",
        "token_markup_pct": "https://openrouter.ai/docs/faq",
        "acquisition_terms": "https://www.reuters.com/technology/payments-firm-stripe-buy-ai-developer-platform-openrouter-2026-08-19/",
        "cap_observability": "https://openrouter.ai/enterprise",
        "cap_pii_redaction": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "eu_residency_note": "https://openrouter.ai/docs/features/privacy-and-logging",
        "last_release_date": "https://openrouter.ai/docs/changelog",
        "openai_compatible": "https://openrouter.ai/docs/faq",
        "openrouter_limits": "https://openrouter.ai/docs/api-reference/limits",
        "subprocessors_url": "https://openrouter.ai/authorized-sub-processors",
        "cap_cache_semantic": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "cap_load_balancing": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway",
        "credit_fee_min_usd": "https://openrouter.ai/docs/faq",
        "guardrails_location": "https://openrouter.ai/docs/guides/features/guardrails",
        "latency_source_note": "https://www.respan.ai/market-map/compare/openrouter-vs-vercel-ai-gateway",
        "requires_vendor_sdk": "https://openrouter.ai/docs/quickstart",
        "data_export_available": "https://openrouter.ai/docs/cookbook/administration/activity-export",
        "openrouter_enterprise": "https://openrouter.ai/enterprise",
        "prompt_transits_vendor": "https://openrouter.ai/privacy",
        "cap_conditional_routing": "https://openrouter.ai/pricing",
        "trains_on_customer_data": "https://openrouter.ai/privacy",
        "openrouter_prompt_caching": "https://openrouter.ai/docs/features/prompt-caching"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_video": "2026-08-29",
        "free_tier": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "eu_residency": "2026-08-29",
        "founded_year": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "url_models_api": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "cap_virtual_keys": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "credit_fee_min_usd": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:44.171859+00:00",
      "vendor_status": "acquisition_pending",
      "acquired_by": "Stripe",
      "acquired_date": "2026-08-19",
      "acquisition_terms": "Terms undisclosed; Reuters reported slightly more than $8B",
      "product_fate": "Stripe signed a definitive agreement to acquire OpenRouter on 19 August 2026, subject to customary closing conditions and expected to close in the coming weeks. OpenRouter says it will continue to operate as it is: same mission, same name, same product, same roadmap, and that nothing about an existing integration changes.",
      "last_release_date": "2026-08-19",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Hosted router only; there is no self-hosted option. Your input is transmitted to whichever model provider you select.",
      "logs_by_default": "metadata_only",
      "logs_note": "Both logging toggles are off by default. Token counts, latency and similar metadata are stored; prompt and completion content is not.",
      "retention_days": 0,
      "retention_note": "Prompt retention is always opt-in. Metadata is retained with no published window.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "OpenRouter does not train on your inputs or outputs, and offers a 1% discount if you opt in to letting it. Downstream model providers are a separate question: some of them may train on what you send.",
      "subprocessors_url": "https://openrouter.ai/authorized-sub-processors",
      "regions_note": "US servers by default. Enterprise adds EU region locking and in-region routing. No self-hosted or customer-VPC option.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II, asserted on the enterprise page",
          "status": "claimed"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "described as GDPR compatible",
          "status": "claimed"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "PII detection and redaction run inside OpenRouter's gateway, so the data has already reached them before it is redacted.",
      "acquired_date_basis": "announced",
      "security_incidents": [],
      "pricing_model": "markup_only",
      "pricing_model_note": "Credit top-up fee (5.5% Stripe / 5% crypto) with a $0.80 minimum; 0% token markup; BYOK charged 5% above a monthly list-price allowance ($25k Pay-as-you-go / $200k Enterprise).",
      "min_commitment": "None stated. Unused credits may expire one year after purchase.",
      "overage_note": "No log/trace-retention or request-volume overage. Only after-the-fact charge is the BYOK 5% once the monthly list-price allowance is exceeded.",
      "cache_support": "passthrough",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Provider caching flows through unchanged. Read discounts vary by provider: 75%/50% for OpenAI, 90% for Anthropic/Alibaba/DeepSeek, 50% for Groq, ~80% for Z.AI, 75% for Gemini implicit. Cache-write premium: 0% for pre-GPT-5.6 OpenAI/Grok/Moonshot/Groq/Gemini; +25% for GPT-5.6+, Alibaba explicit and Anthropic 5-min; +100% for Anthropic 1-hour. OpenRouter itself adds 0% on cached traffic.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Not stated as spend alerts; auto top-up threshold is the closest control.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "API keys plus Management API for programmatic key management; 'virtual key' terminology not used.",
          "tier": null,
          "supported": true
        },
        "rate_limiting": {
          "note": "Platform-governed, not user-configurable. 20 req/min and 50 or 1,000 req/day on free models; global capacity governance across accounts.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Per-key credit limits enforced pre-request via limit/limit_remaining.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Per model, provider and API key on the Activity page; session_id grouping across turns. Per-user/team/customer not stated.",
      "cost_export": "API only (/api/v1/key, credits API, /api/v1/generation). CSV, webhook, S3 and warehouse export not stated.",
      "product_kind_note": "Routing proxy in front of many providers. The vendor's FAQ says \"OpenRouter is a proxy that sends your requests to the model provider for it to be completed\" and \"OpenRouter is a drop-in replacement for OpenAI\" ([OpenRouter FAQ](https://openrouter.ai/docs/faq)); the quickstart calls it \"access to hundreds of AI models through a single API endpoint\" that \"handles fallbacks automatically\" ([OpenRouter Quickstart](https://openrouter.ai/docs/quickstart)).",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "Nothing to install: customers call `https://openrouter.ai/api/v1` ([Quickstart](https://openrouter.ai/docs/quickstart)). The \"self-host\"-like lever is BYOK/own capacity: \"Use your AWS/GCP/Azure credits and get the best throughput combining your limits plus ours\" ([Enterprise](https://openrouter.ai/enterprise)). Tier gating exists for BYOK volume, not deployment: pay-as-you-go includes $25,000/month of BYOK inference free, Enterprise $200,000 ([FAQ](https://openrouter.ai/docs/faq)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap: \"any SDKs that support OpenAI by default also support OpenRouter\" ([FAQ](https://openrouter.ai/docs/faq)); first-party SDKs (`@openrouter/sdk`, Python `openrouter`, Agent SDK) are optional ([Quickstart](https://openrouter.ai/docs/quickstart)). Schemas are \"very similar to the OpenAI Chat API\" but not identical, unsupported params are silently ignored, and streaming is SSE with an occasional comment payload to ignore; usage arrives once in the final chunk ([API overview](https://openrouter.ai/docs/api_reference/overview)). Batch is text-only — image/audio/video/file parts are rejected ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).",
      "model_count_low": 400,
      "model_count_high": 500,
      "model_count_source_note": "\"400+ models and providers\" on the docs models page (undated) ([Models](https://openrouter.ai/docs/guides/overview/models)); the enterprise marketing page says \"500+ AI models\" (undated) ([Enterprise](https://openrouter.ai/enterprise)); the quickstart only says \"hundreds of AI models\" ([Quickstart](https://openrouter.ai/docs/quickstart)). See contradictions.",
      "provider_count_low": 83,
      "provider_count_high": 83,
      "provider_count_source_note": "\"83 of 83 providers\" listed on the providers directory, page undated ([OpenRouter Providers](https://openrouter.ai/providers)).",
      "first_party_vs_partner": "Routed to third-party providers: \"We pass through the pricing of the underlying providers; there is no markup on inference pricing\" and OpenRouter \"sends your requests to the model provider\" ([FAQ](https://openrouter.ai/docs/faq)); routing falls back \"to other providers or GPUs\" ([API overview](https://openrouter.ai/docs/api_reference/overview)). No statement of OpenRouter-owned inference hardware on any page fetched.",
      "custom_endpoint_support": "Partially: not documented for arbitrary vLLM/Ollama URLs, but BYOK supports customer-specific Azure deployment URLs — per-deployment config takes an explicit `endpoint_url` such as `https://example-project.openai.azure.com/openai/deployments/...` plus `api_key`, `model_id`, `model_slug` ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)). Generic private-URL registration: not documented.",
      "byok_mode": "both",
      "merchant_of_record": "Differs by mode. Credits mode: OpenRouter invoices — \"Credits are simply deposits on OpenRouter that you use for LLM inference… we deduct the request cost from your credits\" ([FAQ](https://openrouter.ai/docs/faq)). BYOK mode: the upstream provider bills — \"the provider bills you directly for inference and OpenRouter charges only the BYOK fee\" (5% of list, deducted from credits, above the plan allowance) ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart), [BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).",
      "key_management_note": "\"Your provider keys are securely encrypted and used for all requests routed through the specified provider\"; keys live in workspace BYOK settings, can be Prioritized or Fallback, and can be filtered by `allowed_models`, `allowed_api_key_hashes`, `allowed_user_ids` (100 entries max each) ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)). Content visibility: \"OpenRouter does not store your prompts or responses, unless you opt in\"; metadata (tokens, latency) is always stored and \"does not include the content of your prompts or responses\" ([Data collection](https://openrouter.ai/docs/guides/privacy/data-collection), [FAQ](https://openrouter.ai/docs/faq)). Batch inputs/results are stored as JSONL in Google Cloud Storage and deleted after 30 days ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).",
      "api_openai_chat_completions_note": "yes: `POST /api/v1/chat/completions`, \"implements the OpenAI API specification for /completions and /chat/completions endpoints\" ([API overview](https://openrouter.ai/docs/api_reference/overview), [FAQ](https://openrouter.ai/docs/faq)).",
      "api_anthropic_messages_note": "yes: native Anthropic Messages surface documented as `POST /messages` on the `https://openrouter.ai/api/v1` server ([Create a message](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-a-message)); also a valid Batch API shape as `/v1/messages` ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).",
      "api_openai_responses_note": "yes: \"OpenAI-compatible Responses API endpoints\" are listed in the API reference navigation ([API reference](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-a-message)) and `/v1/responses` is an accepted Batch endpoint shape ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).",
      "api_embeddings_note": "yes: `/v1/embeddings` is a documented batch shape (\"Embeddings are rolling out on providers that support them\") and models can be filtered with `/api/v1/models?output_modalities=embeddings` ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart), [Models](https://openrouter.ai/docs/guides/overview/models)).",
      "api_images_note": "yes: `POST /api/v1/images` ([Image generation](https://openrouter.ai/docs/guides/overview/multimodal/image-generation)).",
      "api_audio_note": "yes: STT and TTS endpoint categories are listed in the API reference, and audio input/output runs through `/api/v1/chat/completions` ([List all models](https://openrouter.ai/docs/api/api-reference/models/list-all-models-and-their-properties), [Audio](https://openrouter.ai/docs/guides/overview/multimodal/audio)).",
      "api_batch_note": "yes: `POST https://openrouter.ai/api/beta/batches`, 24h completion window, text-only ([Batch quickstart](https://openrouter.ai/docs/batch-quickstart)).",
      "deployment_modes_note": "Hosted SaaS only (documented). The enterprise page frames the model as \"Bring your own capacity and use OpenRouter as SaaS\" ([OpenRouter Enterprise](https://openrouter.ai/enterprise)); no self-host, VPC, on-prem or air-gapped mode appears on the quickstart, FAQ, API overview, BYOK or enterprise pages fetched ([FAQ](https://openrouter.ai/docs/faq), [BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).",
      "byok_mode_note": "both: \"OpenRouter supports both OpenRouter credits and the option to bring your own provider keys (BYOK)\" ([BYOK](https://openrouter.ai/docs/guides/overview/auth/byok)).",
      "access_meta": {
        "qualified": [],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "No request-timeout header or key appears on the fetched routing/performance docs. The closest published control is a routing preference, not a timeout: `preferred_max_latency` (with `preferred_min_throughput`), evaluated against provider p50/p90/p99 over a rolling 5-minute window ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "No retry-count key or backoff strategy is published; the documented failure behavior is provider fallback, not a retry counter. Default retry count: `n.a.` Backoff: `n.a.` ([OpenRouter model fallbacks](https://openrouter.ai/docs/guides/routing/model-fallbacks)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "ORDERED. `provider.order` sets an explicit ordered provider list and `allow_fallbacks` (default `true`) controls whether other providers may be used ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)). Model-level fallback uses the `models` array, tried in order; via the Anthropic-compatible surface the key is `fallbacks`, limited to 3 entries ([OpenRouter model fallbacks](https://openrouter.ai/docs/guides/routing/model-fallbacks)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "It is automatic unhealthy-provider ejection, not a configurable health check or circuit breaker: OpenRouter tracks provider errors and removes a failing provider from rotation using a 30-second outage window ([OpenRouter reliability and failover](https://openrouter.ai/blog/insights/reliability-failover/)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "The fetched pages describe edge execution on Cloudflare Workers but no configurable cross-region failover ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
      "rel_load_balancing_config": "per_request",
      "rel_load_balancing_note": "Weights are NOT user-settable: OpenRouter load-balances with price-inverse-square weighting internally; the user-facing controls are `order`, `only`, `sort` and the latency/throughput preferences ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)).",
      "architecture_class": "edge_worker",
      "architecture_note": "Stated runtime: \"Edge computing using Cloudflare Workers\" ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
      "self_host_data_plane": "not_documented",
      "self_host_artifact_note": "No Docker image, Helm chart, binary or npm package for a self-hosted data plane appears on the fetched pages ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported. Documented caveat: on stream cancellation some upstreams still bill — Bedrock, Groq, Google and Mistral are named as continuing to charge for cancelled streams ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Guardrails include a DLP scanner with 7 built-in sensitive-data types, using regex plus Presidio NLP for names/addresses; per-type action is Flag, Redact, or Block, and Block returns a 403 before the model is called, so it both redacts and blocks depending on configuration ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails), [OpenRouter guardrails announcement](https://openrouter.ai/blog/announcements/guardrails/))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Prompt-injection detection is regex/pattern-based (30+ patterns mapped to OWASP LLM risks), not a model classifier, and shares the Flag/Redact/Block actions ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))",
      "guard_content": "not_documented",
      "guard_content_note": "The Guardrails reference documents DLP and prompt-injection scanners; no toxicity/moderation category is described ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policies take the form of configurable scanner rules with per-type actions; regex patterns underpin the DLP and injection scanners ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Model allowlists and provider allowlists are enforced and intersected on the request path ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [
        "Microsoft Presidio"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "No fail-open/fail-closed statement on the guardrails pages ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails))",
      "log_content": "metadata_only",
      "log_content_note": "Prompt and completion text is not stored unless input/output logging is explicitly enabled ([Input/output logging](https://openrouter.ai/docs/guides/features/input-output-logging), [Data collection](https://openrouter.ai/docs/guides/privacy/data-collection))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Body logging is opt-in, not opt-out: leaving I/O logging off keeps usage metrics while storing no prompt/completion text ([Input/output logging](https://openrouter.ai/docs/guides/features/input-output-logging))",
      "tracing_support": "not_documented",
      "tracing_note": "`n.a.` — No OpenTelemetry or agent-trace support documented on the fetched pages ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "export_destinations": [
        "CSV export"
      ],
      "export_destinations_note": "Activity export as CSV and PDF from the activity/administration API; no log/metric/trace streaming destinations documented ([Activity export cookbook](https://openrouter.ai/docs/cookbook/administration/activity-export))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — No feedback or score endpoint documented ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` — No eval or online-scoring product documented ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "observability_requires_saas": null,
      "log_retention_note": "Opt-in prompt/completion logs are retained for at least 3 months ([Data collection](https://openrouter.ai/docs/guides/privacy/data-collection))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` No numeric latency figure is published on the fetched pages. Only the measurement machinery is documented: provider p50/p90/p99 latency over a rolling 5-minute window used for `preferred_max_latency` ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` Only `preferred_min_throughput` as a routing threshold; no measured RPS or tokens/sec figure ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance)).",
        "benchmark_independence": "No figures published, so nothing to attribute ([OpenRouter latency and performance](https://openrouter.ai/docs/guides/best-practices/latency-and-performance))."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "not_documented",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Reliability here is largely vendor-operated rather than user-configured: you choose ordering and preferences, the platform does ejection and weighting. `allow_fallbacks: false` is the switch that makes routing deterministic at the cost of resilience ([OpenRouter provider selection](https://openrouter.ai/docs/guides/routing/provider-selection)).",
      "guard_notes": "Guardrails are enforced alongside Zero-Data-Retention routing: ZDR enforcement is applied on the same request path as the model/provider allowlists ([OpenRouter Guardrails docs](https://openrouter.ai/docs/guides/features/guardrails)).",
      "fit_agent_features_note": "The `@openrouter/agent` Agent SDK provides multi-turn conversation loops, tool execution, stop conditions, and state management through `callModel`. It sends prompts, handles model tool calls, executes tools, feeds results back, and returns the final response; sessions/threads and A2A are not documented. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. Only an OpenRouter key is used in the examples ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart)); the FAQ states OpenRouter \"provides a unified API to access all the major LLM models\" with no BYO-key requirement ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "fit_client_sdk_langs": [
        "TypeScript",
        "Python",
        "Go"
      ],
      "fit_client_sdks_note": "Official client SDKs include the TypeScript package `@openrouter/sdk`, the Python package `openrouter` with the `OpenRouter` class, and the Go package `go-sdk` at `github.com/OpenRouterTeam/go-sdk`. The Python SDK supports synchronous and asynchronous clients. ([OpenRouter client SDK overview](https://openrouter.ai/docs/client-sdks/overview))",
      "fit_cloud_iam": "upstream_credentials_only",
      "fit_cloud_iam_note": "Partially: OpenRouter's BYOK docs accept AWS credentials (`accessKeyId`, `secretAccessKey`, `region`) for an AWS IAM user/role and a Google Cloud service-account key JSON for Vertex; Azure is configured with an `api_key`. Calls to OpenRouter still authenticate with an OpenRouter API key, so this is upstream credential reuse rather than IAM-based access to the gateway. ([OpenRouter BYOK authentication docs](https://openrouter.ai/docs/guides/overview/auth/byok))",
      "fit_cloudflare_workers": "vendor_runtime_only",
      "fit_cloudflare_workers_note": "OpenRouter states that it uses edge computing with Cloudflare Workers to stay close to applications. This documents OpenRouter’s own infrastructure use, not a user-facing Cloudflare Workers integration. ([OpenRouter latency and performance guide](https://openrouter.ai/docs/guides/best-practices/latency-and-performance))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated as required; a free allowance exists: \"All new users receive a small free allowance to test out OpenRouter,\" and free models have \"low rate limits (50 requests per day total)\" ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "fit_ecosystem_notes": "OpenRouter provides access to hundreds of models through a unified API, with automatic fallbacks and provider/model routing; its SDK documentation describes access to 400+ models. OpenRouter also documents remote MCP hosting and uses Cloudflare Workers, edge caching, and provider routing in its own service infrastructure; self-hosting and single-cloud requirements are not documented. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))",
      "fit_eval_prereqs": "An OpenRouter API key (examples use the `<OPENROUTER_API_KEY>` placeholder). No account, credit card, cloud account, cluster, or separate provider key is stated as a prerequisite on the quickstart ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))",
      "fit_eval_shape": "base_url_swap",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes. `POST https://openrouter.ai/api/v1/chat/completions` with a bearer OpenRouter key and body `{\"model\": \"~openai/gpt-latest\", \"messages\": [{\"role\":\"user\",\"content\":\"What is the meaning of life?\"}]}`, shown in Python, TypeScript `fetch`, and shell `curl` ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))",
      "fit_gateway_platform": "third_party_documented",
      "fit_gateway_platform_note": "OpenRouter itself documents no gateway plugin, but Apache APISIX's `ai-proxy` / `ai-proxy-multi` plugins list `openrouter` as a supported provider, proxying to `openrouter.ai` via the OpenAI-compatible driver (third-party gateway documentation, not OpenRouter's). ([Apache APISIX ai-proxy plugin docs](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))",
      "fit_helm_chart": null,
      "fit_kubernetes": "not_documented",
      "fit_kubernetes_note": "n.a. (not documented)",
      "fit_mcp": "hosted_mcp_server",
      "fit_mcp_note": "OpenRouter documents remote MCP support through `@openrouter/mcp`, connecting to MCP servers over Streamable HTTP or SSE and exposing their tools as first-class `callModel` tools. Its hosted MCP server is `https://mcp.openrouter.ai/mcp`; the quickstart states that it requires no installation and uses OAuth login. ([OpenRouter Agent SDK MCP tools](https://openrouter.ai/docs/agent-sdk/call-model/mcp-tools))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_kubernetes"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota, or waitlist gate is mentioned; \"OpenRouter gives you access to hundreds of AI models through a single API endpoint\" ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart)); the FAQ also states no gate ([OpenRouter FAQ](https://openrouter.ai/docs/faq))",
      "fit_onboarding_notes": "Optional client SDKs exist (`npm install @openrouter/sdk`, `pip install openrouter`) and an Agent SDK (`npm install @openrouter/agent`), but the plain REST/OpenAI-SDK path needs no new dependency. OpenRouter headers `HTTP-Referer` and `X-OpenRouter-Title` are explicitly \"Optional.\" A remote MCP server at `https://mcp.openrouter.ai/mcp` is offered with \"nothing to install\" ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. OpenRouter documents OpenAI-compatible usage with the exact base URL `https://openrouter.ai/api/v1`; its Responses API endpoint is `https://openrouter.ai/api/v1/responses` and requires an OpenRouter API key. ([OpenRouter quickstart docs](https://openrouter.ai/docs/quickstart))",
      "fit_prod_prereqs": "n.a. (not documented) — no database, Redis, cluster, load balancer, or control-plane requirement appears on the quickstart ([OpenRouter Quickstart Guide](https://openrouter.ai/docs/quickstart))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "base_url_swap",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LangChain integration is documented for Python and JavaScript applications, and LlamaIndex integration is documented for Python and TypeScript RAG applications. LlamaIndex also documents the `llama-index-llms-openrouter` package and `llama_index.llms.openrouter.OpenRouter` class. ([OpenRouter frameworks and integrations overview](https://openrouter.ai/docs/guides/community/frameworks-and-integrations-overview))",
      "fit_quickstart_steps": 0,
      "fit_quickstart_steps_caveat": "The quickstart is prose plus a code sample with no numbered procedure, so there is no step count to report.",
      "fit_quickstart_url": "https://openrouter.ai/docs/quickstart",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "community_provider",
      "fit_terraform_note": "Terraform Registry lists an OpenRouter provider at `cloudopsworks/openrouter` and another at `mcmarkj/openrouter`. The documented versions are `0.3.0` and `0.0.3`, respectively, and both support Terraform `0.13+` per the registry pages. ([Terraform Registry cloudopsworks/openrouter provider](https://registry.terraform.io/providers/cloudopsworks/openrouter/latest))",
      "fit_terraform_ref": "cloudopsworks/openrouter 0.3.0 and mcmarkj/openrouter 0.0.3 on the Registry — neither is published by OpenRouter",
      "fit_third_party_documented": {
        "gateway_platform": "Apache APISIX"
      },
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "OpenRouter documents the official provider package `@openrouter/ai-sdk-provider` for the Vercel AI SDK, including use in a Next.js application. ([OpenRouter Vercel AI SDK provider](https://github.com/OpenRouterTeam/ai-sdk-provider))",
      "fit_vercel_ai_sdk_package": "@openrouter/ai-sdk-provider"
    },
    {
      "slug": "requesty",
      "name": "Requesty",
      "company": "Requesty",
      "tagline": "Hosted router with a flat 5% fee on inference, EU data residency and enterprise governance controls.",
      "category": "managed_marketplace",
      "deployment": "managed_only",
      "license": null,
      "founded_year": null,
      "hq_country": "United Kingdom",
      "url_home": "https://requesty.ai/",
      "url_docs": "https://docs.requesty.ai/",
      "url_pricing": "https://www.requesty.ai/pricing",
      "url_status": "https://status.requesty.ai",
      "url_github": null,
      "url_changelog": "https://docs.requesty.ai/changelog",
      "url_models_api": "https://router.requesty.ai/v1/models",
      "github_stars": null,
      "model_count": 600,
      "model_count_note": "Homepage and pricing page state 600+ models and 20+ providers; docs say 300+. A live unauthenticated call to https://router.requesty.ai/v1/models on 2026-08-29 returned 686 entries with per-model pricing.",
      "provider_count": 20,
      "token_markup_pct": 5,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "0% markup on your own provider contracts when using BYOK; keeps negotiated provider and committed-use pricing.",
      "seat_fee_usd_month": 0,
      "free_tier": "Free plan: all free models, 200 requests/day, routing, caching, fallbacks, EU residency, no credit card.",
      "enterprise_from_usd_year": null,
      "extra_fees": [],
      "self_host_cost_note": null,
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": false,
      "portability_note": "Analytics export is CSV/PDF summaries.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Anthropic Messages",
        "Embeddings",
        "Audio",
        "Images"
      ],
      "sdk_note": "Base URL https://router.requesty.ai/v1 works with the OpenAI SDK; documented integrations for LangChain, Vercel AI SDK, LlamaIndex, Haystack, Pydantic AI and Claude Code.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": 16,
      "latency_source_note": "Vendor benchmark (Requesty blog, 2026-04-28) claims roughly 16 ms of added overhead including fallback, load balancing, caching and observability, versus about 55 ms for managed OpenRouter and 124 ms for self-hosted LiteLLM; no methodology is published. A third-party comparison repeats an approximate 8 ms P50 figure, also without methodology.",
      "throughput_rps": null,
      "soc2": false,
      "hipaa_baa": null,
      "gdpr": true,
      "eu_residency": true,
      "eu_residency_note": "EU traffic pinned to AWS eu-central-1 (Frankfurt) via https://router.eu.requesty.ai/v1, with request processing, logging, caching and analytics kept in region; US data in Virginia and APAC in Singapore.",
      "zdr": true,
      "zdr_note": "Organisation-wide zero data retention is enforced server-side but must be requested in writing, and it also disables Requesty's own caching.",
      "air_gapped": false,
      "best_for": "European teams that want one router with EU data residency, PII scrubbing and a predictable flat 5% fee.",
      "strengths": [
        "Flat 5% fee with no seat fees, no minimum spend and no top-up fee",
        "0% markup when bringing your own provider keys",
        "EU-only routing endpoint with in-region logging, caching and analytics",
        "Semantic caching, PII redaction, guardrails and budget controls",
        "Public models endpoint returning live pricing"
      ],
      "weaknesses": [
        "5% markup is charged on every token, unlike zero-markup gateways",
        "Its own pages state 600+, 400+ and 300+ models inconsistently",
        "SOC 2 Type II is only in progress, not attested",
        "Governance features (RBAC, PII detection, guardrails, SSO) are gated to Enterprise",
        "Managed SaaS only: no self-hosting, VPC or air-gapped option"
      ],
      "avoid_if": "You need a zero-markup gateway, a completed SOC 2 attestation today, or a self-hosted deployment.",
      "reviews": [
        {
          "url": "https://www.datacamp.com/tutorial/requesty-tutorial",
          "why": "Independent step-by-step tutorial that builds against Requesty's OpenAI-compatible endpoint and compares its routing, caching and cost controls with OpenRouter and LiteLLM.",
          "date": "2025-12-01",
          "type": "written",
          "title": "Requesty Tutorial: The Unified LLM Gateway",
          "publisher": "DataCamp"
        },
        {
          "url": "https://www.youtube.com/watch?v=aXw0ew7GEvw",
          "why": "Vasilij Nevlev runs the same prompt through both gateways across four rounds, testing auto-routing, caching, budget caps, PII scrubbing and EU-only model filtering.",
          "date": "2026-07-17",
          "type": "video",
          "title": "OpenRouter vs Requesty: Which AI Gateway You Should Run?",
          "publisher": "YouTube - AiGentic Lab"
        }
      ],
      "sources": {
        "zdr": "https://requesty.ai/",
        "gdpr": "https://requesty.ai/",
        "soc2": "https://requesty.ai/",
        "cap_mcp": "https://www.requesty.ai/pricing",
        "sdk_note": "https://docs.requesty.ai/",
        "url_docs": "https://docs.requesty.ai/",
        "zdr_note": "https://requesty.ai/",
        "cap_audio": "https://docs.requesty.ai/",
        "free_tier": "https://www.requesty.ai/pricing",
        "air_gapped": "https://www.truefoundry.com/blog/requesty-vs-openrouter",
        "deployment": "https://www.truefoundry.com/blog/requesty-vs-openrouter",
        "hq_country": "https://www.businessinsider.com/pitch-deck-ai-startup-requesty-seed-2025-9",
        "url_status": "https://status.requesty.ai",
        "zdr_status": "https://www.requesty.ai/security",
        "cap_budgets": "https://requesty.ai/",
        "model_count": "https://www.requesty.ai/pricing",
        "url_pricing": "https://www.requesty.ai/pricing",
        "api_surfaces": "https://docs.requesty.ai/",
        "cap_failover": "https://requesty.ai/",
        "eu_residency": "https://requesty.ai/",
        "product_fate": "https://docs.requesty.ai/changelog",
        "regions_note": "https://www.requesty.ai/security",
        "byok_fee_note": "https://requesty.ai/",
        "cap_image_gen": "https://docs.requesty.ai/",
        "url_changelog": "https://docs.requesty.ai/changelog",
        "vendor_status": "https://www.requesty.ai/blog/requesty-raises-3m",
        "byok_supported": "https://requesty.ai/",
        "cap_embeddings": "https://docs.requesty.ai/",
        "cap_guardrails": "https://requesty.ai/",
        "certifications": "https://www.requesty.ai/security",
        "config_as_code": "https://docs.requesty.ai/features/latency-routing",
        "provider_count": "https://www.requesty.ai/pricing",
        "retention_days": "https://www.requesty.ai/security",
        "sla_uptime_pct": "https://www.requesty.ai/enterprise",
        "url_models_api": "https://docs.requesty.ai/features/supported-models",
        "cap_rate_limits": "https://requesty.ai/",
        "logs_by_default": "https://www.requesty.ai/security",
        "cap_cache_simple": "https://docs.requesty.ai/",
        "model_count_note": "https://continuumcode.ai/guides/requesty-review/",
        "requesty_pricing": "https://www.requesty.ai/pricing",
        "token_markup_pct": "https://www.requesty.ai/pricing",
        "cap_observability": "https://requesty.ai/",
        "cap_pii_redaction": "https://requesty.ai/",
        "eu_residency_note": "https://www.requesty.ai/blog/agentic-routing-benchmarked",
        "last_release_date": "https://docs.requesty.ai/changelog",
        "openai_compatible": "https://docs.requesty.ai/",
        "subprocessors_url": "https://www.requesty.ai/privacy/subprocessors",
        "cap_cache_semantic": "https://requesty.ai/",
        "cap_load_balancing": "https://docs.requesty.ai/",
        "seat_fee_usd_month": "https://requesty.ai/",
        "guardrails_location": "https://www.requesty.ai/security",
        "latency_overhead_ms": "https://www.requesty.ai/blog/agentic-routing-benchmarked",
        "latency_source_note": "https://www.requesty.ai/blog/agentic-routing-benchmarked",
        "requesty_api_limits": "https://docs.requesty.ai/features/api-limits",
        "requires_vendor_sdk": "https://docs.requesty.ai/frameworks/openai",
        "cap_prompt_management": "https://requesty.ai/",
        "data_export_available": "https://docs.requesty.ai/features/analytics-exports",
        "requesty_auto_caching": "https://docs.requesty.ai/features/auto-caching",
        "prompt_transits_vendor": "https://www.requesty.ai/security",
        "cap_conditional_routing": "https://www.requesty.ai/pricing",
        "trains_on_customer_data": "https://www.requesty.ai/privacy"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "cap_mcp": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "free_tier": "2026-08-29",
        "air_gapped": "2026-08-29",
        "deployment": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "eu_residency": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "provider_count": "2026-08-29",
        "url_models_api": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:46.577984+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed and sold to new customers, with three features shipped on 26 August 2026. Raised a $3M seed led by 20VC.",
      "last_release_date": "2026-08-26",
      "sla_uptime_pct": 99.99,
      "sla_url": "https://www.requesty.ai/enterprise",
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Hosted gateway; all traffic flows through Requesty's infrastructure in Frankfurt.",
      "logs_by_default": "full_content",
      "logs_note": "On self-serve plans prompt and output logging is on by default, encrypted and held in the EU. You can disable it per API key.",
      "retention_days": 30,
      "retention_note": "Thirty days on self-serve, reducible to zero by disabling logging per key.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "Absolute on any paid plan. The free plan is different: models labelled Training Permitted retain prompts and outputs and train on them, and those models are free-plan only.",
      "subprocessors_url": "https://www.requesty.ai/privacy/subprocessors",
      "regions_note": "EU only — all infrastructure in Frankfurt, with an EU endpoint. No US region and no customer-VPC option.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II expected Q3 2026",
          "status": "in_progress"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPA on request; EU residency in Frankfurt",
          "status": "available"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "Requesty runs its own PII detection model before the prompt reaches the model provider — inside its gateway, not yours.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "markup_only",
      "pricing_model_note": "Flat 5% per-token markup, no seats, no minimum spend. Free tier caps free-model use at 200 req/day; Enterprise is custom.",
      "min_commitment": "None — 'no minimum spend'.",
      "overage_note": "No retention, request-volume or export overage stated on the pricing page.",
      "cache_support": "passthrough",
      "cache_read_discount_pct": 90,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Passthrough of provider caching with automatic breakpoint insertion; Requesty explicitly does not store responses server-side. Vendor says cache hits are billed 'at a fraction of the normal input token cost (up to 90% savings)'; cache writes 'have extra costs' for some providers, no number published. A per-request `false` toggle lets you avoid write premiums.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Slack + webhook alerts fire as limits are approached (reported after spend).",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Org admins issue keys; policies scope to a key.",
          "tier": null,
          "supported": true
        },
        "rate_limiting": {
          "note": "Requesty imposes no rate limits of its own; upstream 429s are handled by routing retries.",
          "tier": null,
          "supported": false
        },
        "spend_webhooks": {
          "note": "Webhook alerts on approaching spend limits.",
          "tier": null,
          "supported": true
        },
        "model_allowlists": {
          "note": "'Approved models & policies' is an Enterprise bullet; 'Approved models whitelist' in the comparison table.",
          "tier": "enterprise",
          "supported": true
        },
        "budget_caps_per_key": {
          "note": "Each API key has its own monthly spend cap, also per service account; enforced pre-request.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "'Groups & team budgets' listed; 'Teams, groups & spend controls' is an Enterprise bullet. Tier assignment ambiguous.",
          "tier": "enterprise",
          "supported": true
        }
      },
      "cost_attribution": "Per API key and per service account (each carries its own cap and usage). User/team/tag/customer splits listed as feature names without confirmation.",
      "cost_export": "'Data exports' is listed in the comparison table; a Management API can be queried programmatically. Specific formats (CSV/S3/warehouse) not stated.",
      "product_kind_note": "Routing proxy: \"One API. 300+ models. OpenAI-compatible\" and the docs call it a \"pass-through gateway\" that \"normalizes the schema across models and providers through a single API\" ([Requesty quickstart](https://docs.requesty.ai/), [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)). Marketed as an \"Enterprise AI Gateway\" ([Requesty Enterprise](https://www.requesty.ai/enterprise)).",
      "deployment_modes": [
        "saas",
        "self_host",
        "air_gapped"
      ],
      "deployment_note": "SaaS endpoints `https://router.requesty.ai/v1` and `https://api.requesty.ai/v1` ([quickstart](https://docs.requesty.ai/), [Enterprise](https://www.requesty.ai/enterprise)); the Helm guide covers a Kubernetes chart with `/health` and `/ready` probes and air-gapped installs ([Helm guide](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369)). Enterprise tier gates SSO, RBAC, custom SLAs and EU residency ([Enterprise](https://www.requesty.ai/enterprise)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap: \"If you're already using the OpenAI SDK, point it at Requesty and you're done. No SDK changes, no new client to learn\"; LangChain, Vercel AI SDK, LlamaIndex, Haystack and Pydantic AI work out of the box ([quickstart](https://docs.requesty.ai/)). Streaming caveat: you must pass `stream_options: {\"include_usage\": true}` to get a final usage chunk ([quickstart](https://docs.requesty.ai/)). Model IDs are provider-prefixed (e.g. `openai/gpt-4o`) ([quickstart](https://docs.requesty.ai/)).",
      "model_count_low": 160,
      "model_count_high": 600,
      "model_count_source_note": "\"300+ models\" on the docs (undated) ([Supported models](https://docs.requesty.ai/features/supported-models), [quickstart](https://docs.requesty.ai/)); \"600+\" AI models on the enterprise page (undated) ([Enterprise](https://www.requesty.ai/enterprise)); \"160+ supported models\" in the self-hosting blog post ([Helm guide](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369)). See contradictions.",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "n.a. as a number. The BYOK page lists 6 connectable providers (OpenAI, Anthropic, Google AI Studio, Vertex AI, Mistral AI, xAI) and says \"More providers will be added over time\"; the models page names OpenAI, Anthropic, Google, DeepSeek, xAI, Mistral \"and more\". Both pages undated ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys), [Supported models](https://docs.requesty.ai/features/supported-models)).",
      "first_party_vs_partner": "Routed to third-party providers; open-source models are \"Hosted or bring-your-own endpoints\" ([quickstart](https://docs.requesty.ai/)). No Requesty-owned inference hardware is claimed on any page fetched.",
      "custom_endpoint_support": "Partially documented: the quickstart lists \"bring-your-own endpoints\" for open-source models ([quickstart](https://docs.requesty.ai/)), but no page fetched documents how to register a vLLM/Ollama/SageMaker URL — mechanism n.a.",
      "byok_mode": "both",
      "merchant_of_record": "Differs by mode. Credits: Requesty (account credits consumed per request, `cost` returned in `usage`) ([quickstart](https://docs.requesty.ai/)). BYOK: the upstream provider — \"Cost Control: Use your own billing relationships with providers\" and \"keep provider discounts and committed-use pricing\" ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys), [quickstart](https://docs.requesty.ai/)). Explicit invoicing language: not stated.",
      "key_management_note": "Upstream keys are added per provider in the dashboard, \"at most one key per provider\", usable in fallback policies, with usage tracked across the organization ([BYOK](https://docs.requesty.ai/features/bring-your-own-keys)). Requesty's own API keys are \"hashed at rest\", shown once, and scoped service-account keys are supported; prompts may be logged up to 30 days unless disabled, and org-wide Zero Data Retention requires a written request ([Security](https://www.requesty.ai/security)). Enterprise page claims zero data retention and no training on customer data ([Enterprise](https://www.requesty.ai/enterprise)). Encryption/KMS specifics for upstream keys: n.a.",
      "api_openai_chat_completions_note": "yes: `POST https://router.requesty.ai/v1/chat/completions` ([quickstart](https://docs.requesty.ai/), [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).",
      "api_anthropic_messages_note": "yes: `POST /v1/messages`, documented as `https://router.requesty.ai/anthropic/v1/messages` for the Anthropic SDK ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis), [quickstart](https://docs.requesty.ai/)).",
      "api_openai_responses_note": "yes: `POST /v1/responses` ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).",
      "api_embeddings_note": "yes (\"Create vector embeddings from text…\"), documented under the inference API family ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).",
      "api_images_note": "yes: image generation and editing \"using DALL-E, Stable Diffusion, and other image models\" ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).",
      "api_audio_note": "yes, both: Text to Speech and Speech to Text endpoints ([Inference APIs](https://docs.requesty.ai/api-reference/inference-apis)).",
      "api_batch_note": "n.a. (no batch/async bulk endpoint on any Requesty page fetched: [Inference APIs](https://docs.requesty.ai/api-reference/inference-apis), [quickstart](https://docs.requesty.ai/)).",
      "deployment_modes_note": "Hosted SaaS (EU/Frankfurt region available) ([Enterprise](https://www.requesty.ai/enterprise), [Security](https://www.requesty.ai/security)). Self-host on Kubernetes via Helm, incl. air-gapped, is documented in a vendor blog post ([Self-hosting Requesty on Kubernetes](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369)) while the enterprise page says self-hosting is \"not offered at this time\" ([Enterprise](https://www.requesty.ai/enterprise)) — see contradictions. — self_host and air_gapped are contested: a vendor engineering post documents a Helm/Kubernetes install including air-gapped, while the enterprise page states self-hosting is not offered at this time.",
      "byok_mode_note": "both: platform credits (\"New accounts include free credits to start routing immediately\") plus BYOK (\"Use your own API keys with Requesty… Choose between Requesty's keys or your own keys for each model\") ([quickstart](https://docs.requesty.ai/), [BYOK](https://docs.requesty.ai/features/bring-your-own-keys)).",
      "access_meta": {
        "qualified": [
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "No timeout header or key on the fetched fallback/load-balancing/latency-routing/limits pages ([Requesty API limits](https://docs.requesty.ai/features/api-limits.md)).",
      "rel_retry_config": "dashboard_only",
      "rel_retry_note": "Retries are part of a policy configured in the Requesty console: 0–10 retries per model, exponential backoff 500ms → 1s → 2s → 4s with ±10% jitter ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "dashboard_only",
      "rel_fallback_note": "ORDERED. Fallback policies are created in the dashboard and referenced from the request as `policy/<name>` in the `model` field ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "No upstream health check, circuit breaker or provider-ejection mechanism is documented on the fetched pages; latency routing selects by observed latency instead ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "Regional Bedrock endpoints can be targeted as separate models, but no cross-region failover feature is documented ([Requesty load balancing policies](https://docs.requesty.ai/features/load-balancing-policies)).",
      "rel_load_balancing_config": "dashboard_only",
      "rel_load_balancing_note": "Weights ARE supported and must total 100% across the models in a load-balancing policy ([Requesty load balancing policies](https://docs.requesty.ai/features/load-balancing-policies)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "Runtime is never disclosed on the fetched pages; only the hosted endpoint `https://router.requesty.ai/v1` is documented ([Requesty streaming](https://docs.requesty.ai/features/streaming)).",
      "self_host_data_plane": "not_documented",
      "self_host_artifact_note": "No self-host artifact appears on the fetched pages ([Requesty streaming](https://docs.requesty.ai/features/streaming)).",
      "latency_claims": [
        {
          "url": "https://docs.requesty.ai/features/streaming",
          "unit": "",
          "scope": "marketing",
          "value": "up to 80% lower",
          "metric": "perceived wait time",
          "publisher": "vendor",
          "conditions": "Streaming marketing claim. Not a gateway-overhead measurement; conditions not stated."
        }
      ],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported over Server-Sent Events with `\"stream\": true`. Caveats: streaming responses do not include a `usage` object by default — you must pass `stream_options: {\"include_usage\": true}`, which adds an extra chunk before `data: [DONE]` with an empty `choices` array; the docs also tell you to implement your own retry/non-streaming fallback for stream interruptions ([Requesty streaming](https://docs.requesty.ai/features/streaming)).",
      "guard_pii": "sync_observe",
      "guard_pii_note": "PII scanners (`pii/v2/fast`, `pii/v2/ml`) support actions Disabled, Report, and Mask; masking rewrites the request but the docs document no hard block/deny action ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))",
      "guard_injection": "not_documented",
      "guard_injection_note": "The guardrails page documents PII and secrets scanners only ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))",
      "guard_content": "not_documented",
      "guard_content_note": "No toxicity/moderation scanner documented ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))",
      "guard_custom": "not_documented",
      "guard_custom_note": "Policy form is a choice of vendor-supplied scanners plus an action; no regex, word list, or JSON-schema enforcement documented ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails), [Managed policies](https://docs.requesty.ai/features/managed-policies))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Approved Models restricts the callable catalog, and Access Lists apply org-, group-, and key-level allowlists ([Approved models](https://docs.requesty.ai/features/approved-models), [Access lists](https://docs.requesty.ai/features/access-lists))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "No timeout or failure behaviour stated ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails))",
      "log_content": "full_request_and_response",
      "log_content_note": "The Logs view shows full message history for a request, i.e. prompt and completion text is stored ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "log_opt_out": "not_documented",
      "log_opt_out_note": "No documented header or setting to suppress bodies while keeping metrics ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "tracing_support": "otel",
      "tracing_note": "Requesty groups related requests into Traces keyed by `trace_id`, giving multi-step visibility; OpenTelemetry support is not documented ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "export_destinations": [],
      "export_destinations_note": "`n.a.` — No external log/metric/trace destinations documented ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "feedback_capture": "partial",
      "feedback_capture_note": "Logs carry a Feedback column, but no feedback API endpoint or header is documented ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` — No eval or scoring product documented ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "observability_requires_saas": null,
      "log_retention_note": "`n.a.` — No retention window stated ([Requesty Logs](https://docs.requesty.ai/features/logs))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "One marketing figure only: streaming reduces \"perceived wait time by up to 80%\" — metric is perceived wait time, not gateway overhead; conditions not stated ([Requesty streaming](https://docs.requesty.ai/features/streaming)). No p50/p99/overhead figures ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` ([Requesty latency routing](https://docs.requesty.ai/features/latency-routing)).",
        "benchmark_independence": "The single \"up to 80%\" figure is a vendor claim about its own product, with no methodology ([Requesty streaming](https://docs.requesty.ai/features/streaming))."
      },
      "guard_pii_default": "sync_observe",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "not_documented",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Everything reliability-related is a named dashboard \"policy\" invoked as `policy/<name>` in the `model` field, so reliability config is not in your repo and not per-request ([Requesty fallback policies](https://docs.requesty.ai/features/fallback-policies)).",
      "guard_notes": "Guardrail verdicts are written into the logs, so Requesty's scanners function primarily as inline detectors with masking, and the docs never promise request denial ([Requesty Guardrails](https://docs.requesty.ai/features/guardrails)).",
      "fit_agent_features_note": "Requesty documents function calling, agent routing policies with preferred models, fallback chains, and cost caps, and agent analytics for latency, cost, and success rates. Session reconstruction can replay full conversation sessions for debugging; sessions/threads as an agent primitive, multi-step orchestration, and A2A are not stated. ([Requesty homepage](https://www.requesty.ai/))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. Only `REQUESTY_API_KEY` is needed; \"Bring your own keys\" is listed as an optional feature ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))",
      "fit_client_sdk_langs": [
        "Python",
        "TypeScript"
      ],
      "fit_client_sdks_note": "Requesty documents the OpenAI SDK for Python and TypeScript/Node.js using the `openai` package, installed with `pip install openai` or `npm install openai`, and the `OpenAI` client class. The Anthropic SDK is also documented through `https://router.requesty.ai/anthropic/v1/messages`; cURL requires no installation. ([Requesty quickstart](https://docs.requesty.ai/quickstart))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "user_facing",
      "fit_cloudflare_workers_note": "Requesty can run behind Cloudflare Workers, which can handle authentication, rate limiting, request validation, forwarding, response caching, and global edge deployment. ([Requesty behind Cloudflare Workers](https://www.requesty.ai/blog/edge-deployments-running-requesty-behind-cloudflare-workers-1751655483))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated as required; \"New accounts include free credits to start routing immediately.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart)). Enterprise page: \"You can start on the pay-as-you-go plan with no commitment\" ([Requesty Enterprise](https://www.requesty.ai/enterprise))",
      "fit_ecosystem_notes": "Requesty is documented as a hosted, OpenAI-compatible gateway with unified routing, fallback, caching, analytics, BYOK, and access to hundreds of models through a single endpoint; documented model counts include 300+ and 600+. Official evidence also documents EU routing at `https://router.eu.requesty.ai/v1` and self-hosting on Kubernetes, including air-gapped deployments. ([Requesty homepage](https://www.requesty.ai/))",
      "fit_eval_prereqs": "A Requesty account (\"Sign up at app.requesty.ai\") and an API key exported as `REQUESTY_API_KEY`. No credit card, cloud account, cluster, or separate provider key is stated ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))",
      "fit_eval_shape": "base_url_swap",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes. OpenAI-SDK Python/TypeScript plus curl: `client.chat.completions.create(model=\"openai/gpt-4o\", messages=[{\"role\":\"user\",\"content\":\"Hello, who are you?\"}])` against `https://router.requesty.ai/v1`, returning an OpenAI-compatible completion ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "Helm repo https://helm.requesty.ai (chart name not stated)",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Requesty documents self-hosting on Kubernetes, including air-gapped environments. The official Helm repository is named `requesty` and is located at `https://helm.requesty.ai`; the chart name is not stated. ([Requesty Kubernetes Helm deployment guide](https://www.requesty.ai/blog/self-hosting-requesty-on-kubernetes-the-complete-helm-deployment-guide-1751655369))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "Requesty provides an MCP Gateway that authenticates and routes requests to multiple MCP servers, supports tool discovery and whitelisting, and integrates with MCP-compatible tools such as Claude Code, Cursor, and Roo Code. It supports `streamable-http` and `sse`; `stdio` is coming soon, and the Roo Code endpoint is `https://router.requesty.ai/mcp`. ([Requesty MCP Gateway docs](https://docs.requesty.ai/features/mcp-gateway))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_gateway_platform",
          "fit_terraform"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "None for self-serve: \"Every model lives behind one endpoint. Swap `model` in the request to switch providers.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart)). Restriction is an admin option, not a gate: \"Enterprise plans let admins restrict access to an approved list of models and providers\" ([Requesty Enterprise](https://www.requesty.ai/enterprise))",
      "fit_onboarding_notes": "Only two config changes are required (Requesty key instead of `OPENAI_API_KEY`, Requesty base URL instead of `https://api.openai.com/v1`); an Anthropic-SDK path exists at `https://router.requesty.ai/anthropic/v1/messages`; OpenAI-compatible frameworks (LangChain, Vercel AI SDK, LlamaIndex, Haystack, Pydantic AI) \"work with Requesty out of the box\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Requesty documents OpenAI SDK drop-in compatibility with `base_url=\"https://router.requesty.ai/v1\"`; the EU endpoint is `https://router.eu.requesty.ai/v1`. ([Requesty OpenAI framework docs](https://docs.requesty.ai/frameworks/openai))",
      "fit_prod_prereqs": "Nothing infrastructural; enterprise adds a sales-led path of 3 steps (\"Talk to our team\" → \"Custom onboarding\" → \"Go live\") and \"Enterprise plans are priced on request\" for SSO/RBAC/SLA ([Requesty Enterprise](https://www.requesty.ai/enterprise))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "base_url_swap",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Requesty documents LangChain integration using `langchain_openai.ChatOpenAI`, `langchain_core.prompts.PromptTemplate`, and `langchain_core.runnables.RunnableLambda`. LlamaIndex is also documented as an integration, including a TypeScript integration, but its package and class names are not stated. ([Requesty LangChain framework docs](https://docs.requesty.ai/frameworks/langchain))",
      "fit_quickstart_steps": 4,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.requesty.ai/quickstart",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "not_documented",
      "fit_terraform_note": "n.a. (not documented)",
      "fit_terraform_ref": null,
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": "Vendor claim, verbatim: \"One API. 300+ models. OpenAI-compatible. Route your first request in under 2 minutes.\" ([Quickstart - Requesty Docs](https://docs.requesty.ai/quickstart))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Yes. The official Vercel AI SDK provider package is `@requesty/ai-sdk`, with the provider instance `requesty` imported from `@requesty/ai-sdk`. ([Requesty Vercel AI SDK docs](https://docs.requesty.ai/frameworks/vercel-ai-sdk))",
      "fit_vercel_ai_sdk_package": "@requesty/ai-sdk"
    },
    {
      "slug": "braintrust-gateway",
      "name": "Braintrust Gateway",
      "company": "Braintrust",
      "tagline": "Multi-provider gateway inside Braintrust's eval and observability platform, with caching and span-level tracing.",
      "category": "managed_gateway",
      "deployment": "both",
      "license": "Open core",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://www.braintrust.dev",
      "url_docs": "https://www.braintrust.dev/docs/deploy/gateway",
      "url_pricing": "https://www.braintrust.dev/pricing",
      "url_status": "https://status.braintrust.dev",
      "url_github": "https://github.com/braintrustdata/braintrust-proxy",
      "url_changelog": "https://www.braintrust.dev/docs/reference/release-notes",
      "url_models_api": null,
      "github_stars": 409,
      "model_count": 100,
      "model_count_note": "Proxy docs state over 100 models; the supported-models page enumerates 18 providers (OpenAI, Azure, Anthropic, Google, Vertex AI, Bedrock, Together, Fireworks, Groq, Mistral, Lepton, Cerebras, Perplexity, xAI, Replicate, Ollama, Databricks, Baseten). Model-list endpoints require authentication.",
      "provider_count": 18,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Provider keys are configured at organization or project level (project keys override org keys); the proxy can even be used with only a provider key and no Braintrust account.",
      "seat_fee_usd_month": 0,
      "free_tier": "Starter $0/month: unlimited users, $10 model credits, 1 GB processed data, 10k scores, 14-day retention. The Braintrust-hosted Gateway is free during public preview, with pricing to be announced before GA.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Pro platform fee",
          "amount": "$249/month (includes $249 model credits, 5 GB data, 50k scores)"
        },
        {
          "label": "Processed data overage",
          "amount": "$4/GB (Starter), $3/GB (Pro)"
        },
        {
          "label": "Scores overage",
          "amount": "$2.50 per 1,000 (Starter), $1.50 per 1,000 (Pro)"
        },
        {
          "label": "Extended retention",
          "amount": "$0.50 per GB per month beyond the included 30 days (Pro, up to 180 days)"
        }
      ],
      "self_host_cost_note": "Self-hosted Braintrust deployments run the Gateway inside the customer's own data plane so LLM traffic stays in their account; the standalone proxy is MIT-licensed on GitHub. On-prem/hybrid deployment is an Enterprise-plan feature, and independent reviews note there is no free self-hosted tier below Enterprise.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": null,
      "portability_note": "Logs and spans export to your own S3/GCS bucket as JSONL or Parquet.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Anthropic Messages",
        "Google Gemini",
        "AWS Bedrock",
        "OpenAI Realtime"
      ],
      "sdk_note": "Point the OpenAI, Anthropic or Gemini SDK at https://gateway.braintrust.dev with a Braintrust user key (sk-) or service token (bt-st-); cross-SDK calls are supported.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": null,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": true,
      "cap_guardrails": false,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": true,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": null,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No published gateway overhead; the global endpoint uses latency-based DNS routing with health checks across five regions (us-east-1, us-west-2, eu-west-1, ap-southeast-1). A third-party comparison confirms no measured Braintrust proxy overhead is published.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": null,
      "eu_residency": true,
      "eu_residency_note": "EU West Gateway endpoint https://gateway.euw.braintrust.dev runs in eu-west-1 (Ireland); Gateway routing is separate from the organization's data-plane region.",
      "zdr": null,
      "zdr_note": "Zero data retention applies only to Braintrust's own built-in Topics inference endpoints, where it is on by default. There is no ZDR mode for your trace data; the equivalents are retention policies or running the data plane yourself.",
      "air_gapped": null,
      "best_for": "Teams that already run evals and tracing in Braintrust and want their model traffic to flow through the same platform.",
      "strengths": [
        "Hosted Gateway is free during public preview, with unlimited users on all plans",
        "Deep eval/tracing integration: spans, scorers, datasets, CI quality gates",
        "Cross-SDK access (OpenAI, Anthropic, Gemini, Bedrock) plus Realtime audio and embeddings",
        "End-to-end encrypted cache with 1-week default TTL and per-request cache control",
        "Time-limited temporary credentials for frontend/mobile clients; MIT-licensed proxy on GitHub"
      ],
      "weaknesses": [
        "Gateway pricing after general availability is unannounced, so cost is unpredictable",
        "Narrower provider list (18) and no public model-catalog endpoint",
        "No real-time guardrails - evaluation happens after the fact",
        "Steep jump from free Starter to $249/month Pro, with per-GB and per-score overages that agent workloads can inflate",
        "Self-hosting is effectively Enterprise-only"
      ],
      "avoid_if": "You want a standalone routing marketplace or inline guardrails rather than an eval-centric platform.",
      "reviews": [
        {
          "url": "https://www.lennysnewsletter.com/p/how-braintrust-uses-ai-agents-evals",
          "why": "Long-form independent interview with Braintrust CEO Ankur Goyal on how the platform (proxy, evals, CI) is used in practice, including agent-driven benchmarking work.",
          "date": "2026-06-15",
          "type": "written",
          "title": "How Braintrust uses AI agents, evals, and CI to ship better AI products",
          "publisher": "Lenny's Newsletter"
        },
        {
          "url": "https://www.youtube.com/watch?v=97iykOemOn4",
          "why": "Hamel Husain, an independent AI-evals practitioner, records a hands-on walkthrough of Braintrust (datasets, tracing, BTQL, SME review) as part of a tool-by-tool comparison.",
          "date": "2025-10-01",
          "type": "video",
          "title": "LLM Eval Tools Compared: Braintrust",
          "publisher": "YouTube - Hamel Husain"
        }
      ],
      "sources": {
        "soc2": "https://www.braintrust.dev/pricing",
        "license": "https://www.braintrust.dev/docs/guides/proxy",
        "sdk_note": "https://www.braintrust.dev/docs/deploy/gateway",
        "url_docs": "https://www.braintrust.dev/docs/deploy/gateway",
        "zdr_note": "https://www.braintrust.dev/docs/guides/proxy",
        "cap_audio": "https://www.braintrust.dev/docs/guides/proxy",
        "cap_evals": "https://www.braintrust.dev/pricing",
        "free_tier": "https://www.braintrust.dev/docs/deploy/gateway",
        "hipaa_baa": "https://www.braintrust.dev/pricing",
        "deployment": "https://www.braintrust.dev/docs/deploy/gateway",
        "extra_fees": "https://www.braintrust.dev/pricing",
        "hq_country": "https://vantaige.io/ai-tool/braintrust",
        "url_github": "https://github.com/braintrustdata/braintrust-proxy",
        "url_status": "https://status.braintrust.dev",
        "zdr_status": "https://www.braintrust.dev/docs/security",
        "model_count": "https://www.braintrust.dev/docs/guides/proxy",
        "url_pricing": "https://www.braintrust.dev/pricing",
        "api_surfaces": "https://www.braintrust.dev/docs/deploy/gateway",
        "cap_failover": "https://www.braintrust.dev/docs/deploy/gateway",
        "eu_residency": "https://www.braintrust.dev/docs/deploy/gateway",
        "github_stars": "https://api.github.com/repos/braintrustdata/braintrust-proxy",
        "product_fate": "https://www.braintrust.dev/docs/changelog",
        "regions_note": "https://www.braintrust.dev/legal/dpa",
        "byok_fee_note": "https://www.braintrust.dev/docs/guides/proxy",
        "url_changelog": "https://www.braintrust.dev/docs/reference/release-notes",
        "vendor_status": "https://www.braintrust.dev/blog/announcing-series-b",
        "byok_supported": "https://www.braintrust.dev/docs/deploy/gateway",
        "cap_embeddings": "https://www.braintrust.dev/docs/deploy/gateway",
        "cap_guardrails": "https://vantaige.io/ai-tool/braintrust",
        "certifications": "https://www.braintrust.dev/docs/security",
        "provider_count": "https://www.braintrust.dev/docs/deploy/supported-models",
        "retention_days": "https://www.braintrust.dev/docs/admin/data-management/retention",
        "logs_by_default": "https://www.braintrust.dev/docs/observe",
        "cap_cache_simple": "https://www.braintrust.dev/docs/guides/proxy",
        "cap_virtual_keys": "https://www.braintrust.dev/docs/guides/proxy",
        "model_count_note": "https://www.braintrust.dev/docs/deploy/supported-models",
        "cap_observability": "https://www.braintrust.dev/docs/deploy/gateway",
        "eu_residency_note": "https://www.braintrust.dev/docs/deploy/gateway",
        "openai_compatible": "https://www.braintrust.dev/docs/deploy/gateway",
        "subprocessors_url": "https://www.braintrust.dev/legal/dpa",
        "braintrust_pricing": "https://www.braintrust.dev/pricing",
        "cap_load_balancing": "https://www.braintrust.dev/docs/guides/proxy",
        "seat_fee_usd_month": "https://www.braintrust.dev/pricing",
        "guardrails_location": "https://www.braintrust.dev/blog/security-data-control",
        "latency_source_note": "https://www.braintrust.dev/docs/deploy/gateway",
        "requires_vendor_sdk": "https://www.braintrust.dev/docs/deploy/gateway",
        "self_host_cost_note": "https://www.braintrust.dev/docs/deploy/gateway",
        "cap_prompt_management": "https://www.braintrust.dev/docs/guides/proxy",
        "data_export_available": "https://www.braintrust.dev/docs/admin/data-management/export",
        "braintrust_billing_faq": "https://www.braintrust.dev/docs/admin/billing/faq",
        "prompt_transits_vendor": "https://www.braintrust.dev/docs/security",
        "trains_on_customer_data": "https://www.braintrust.dev/blog/security-data-control",
        "braintrust_ai_proxy_blog": "https://www.braintrust.dev/blog/ai-proxy",
        "braintrust_plans_and_limits": "https://www.braintrust.dev/docs/plans-and-limits"
      },
      "verified_at": {
        "soc2": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_evals": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_github": "2026-08-29",
        "url_status": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "eu_residency": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "provider_count": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "cap_virtual_keys": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_observability": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:35.832825+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed, with an $80M Series B led by ICONIQ announced in February 2026 and new dashboard features in August 2026.",
      "last_release_date": null,
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": "Braintrust dates its changelog by month only, so no exact release date is published. The latest entry is August 2026.",
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "Three shapes: multi-tenant SaaS, bring-your-own-cloud where Braintrust operates the data plane inside your account, or full self-hosting. In BYOC the SDK and browser talk straight to your data plane, so no AI data crosses Braintrust's control plane.",
      "logs_by_default": "full_content",
      "logs_note": "Once instrumented, Braintrust captures every request as a trace. That is the product, not a side effect — the data plane stores prompts, outputs, datasets and traces.",
      "retention_days": 14,
      "retention_note": "Fourteen days on Starter, thirty on Pro (extendable to 180 in 30-day steps), up to 365 on Enterprise.",
      "zdr_status": "depends",
      "trains_on_customer_data": "no",
      "training_note": "Prompts, traces and customer conversations are never used to improve models.",
      "subprocessors_url": "https://www.braintrust.dev/legal/dpa",
      "regions_note": "Hosted data plane in the US or the EU on Pro. BYOC and self-hosted keep data in your own cloud account and region.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPAs can be executed",
          "status": "available"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "detail": "BAAs can be executed",
          "status": "available"
        }
      },
      "guardrails_location": "both",
      "guardrails_note": "Global masking functions redact PII before logging. In BYOC or self-hosted that masking runs inside your own cloud.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "hybrid",
      "pricing_model_note": "Flat monthly platform fee + included model credits then per-token rates + usage overages (processed data, scores, retention). Starter $0 with $10/mo credits; Pro $249/mo with $249/mo credits (drops to $100/mo credits after 2026-09-01); Enterprise annual invoice. AI proxy blog says the gateway itself is free 'as a beta', but Braintrust's docs index the proxy page as 'deprecated' — treat the free-proxy framing as unverified.",
      "min_commitment": "None for Starter/Pro. Enterprise is annual-invoiced.",
      "overage_note": "Processed data: 1 GB then +$4/GB (Starter) / 5 GB then +$3/GB (Pro). Scores: 10k then $2.50/1k (Starter) / 50k then $1.50/1k (Pro). Retention: 14 days (Starter) / 30 days then +$0.50/GB/month up to 180 days (Pro). Model credits do not roll over. Retention bills what you ingest into each month you keep beyond the base window, not what's currently stored. Deleting data does not reduce processed-data charges.",
      "cache_support": "exact",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact-request response caching in the proxy: requests with temperature=0 or `seed` are cached, encrypted per API key, controllable via x-bt-use-cache. No gateway-layer cached-token pricing; Braintrust's own model rates do publish cache-hit prices (GLM-5.2 81% off, Kimi K3 90% off). Cache hits avoid the provider call entirely, but cached traffic still counts as processed data at ingestion.",
      "cost_controls": {
        "enforcement": "after_the_fact",
        "spend_alerts": {
          "note": "Automated emails at 60% and 100% of Topics credit; Starter alerts at 80/90/100% of included usage. Reported after spend.",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Platform-wide only (10,000 ops/10s), not configurable — returns HTTP 429.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated as cost splits. Environments tag object versions but are not described as cost attribution.",
      "cost_export": "S3 / Google Cloud Storage scheduled export automations on Enterprise only; price not stated. CSV, webhook and warehouse export not stated.",
      "product_kind_note": "An evaluation/observability platform whose gateway also proxies: the vendor calls it the \"Braintrust AI proxy\", giving \"unified access to models from OpenAI, Anthropic, Google, AWS, Mistral, and third-party providers through a single API\" with \"automatic caching, observability, and multi-provider support\" ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). The proxy is now deprecated in favour of the Braintrust Gateway ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway docs](https://www.braintrust.dev/docs/deploy/gateway)).",
      "deployment_modes": [
        "saas",
        "self_host",
        "hybrid_vpc"
      ],
      "deployment_note": "Hybrid model: Braintrust-managed control plane plus a customer-hosted data plane — AWS via Terraform on ECS/EC2, GCP and Azure via Kubernetes/Helm; the self-hosted data-plane URL is configured in organization settings ([Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting)). \"Self-hosted Braintrust deployments include a built-in proxy that runs in your environment\" and the self-hosted Gateway serves `/v1/proxy` ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)). The OSS proxy ships deployment wrappers for Vercel, Cloudflare, AWS Lambda and Express ([GitHub](https://github.com/braintrustdata/braintrust-proxy)). Tier gating for hybrid: not stated on pages fetched; note self-hosted deployments have built-in models disabled by default while SaaS enables them ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "not_documented",
      "api_audio": "partial",
      "api_batch": "not_documented",
      "api_surface_note": "Base-URL swap with the standard OpenAI drivers (\"You can communicate with the proxy via the standard OpenAI drivers/API\"), no proprietary SDK required ([GitHub](https://github.com/braintrustdata/braintrust-proxy)). Caveats: paths sit under `/v1/proxy`; temporary credentials and WebSocket realtime work only on the standalone `https://braintrustproxy.com/v1` host, not `api.braintrust.dev/v1/proxy`; requests to `api.braintrust.dev/v1/proxy` now route to the Gateway with \"no code or URL changes\"; caching triggers when `temperature=0` or a seed is set ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [API ref](https://www.braintrust.dev/docs/api-reference/proxy/proxy-chatcompletions)). Self-hosted reasoning support needs data plane v0.0.74+, project-level provider precedence v2.0.0+ ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "model_count_low": 100,
      "model_count_high": 100,
      "model_count_source_note": "\"over 100 models\" (page undated) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "provider_count_low": 9,
      "provider_count_high": 9,
      "provider_count_source_note": "n.a. as a number; standard providers named are OpenAI, Anthropic, Google, AWS Bedrock, Azure OpenAI, Together AI, Fireworks, Groq, Replicate (page undated) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "first_party_vs_partner": "Third-party routing with a Braintrust-billed \"built-in models\" option: built-in open-source models (e.g. `glm-5.2`, `kimi-k3`, `deepseek-v4-flash-0731`) consume Braintrust model credits at on-demand rates and are disabled by default on self-hosted deployments ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)). Whether models run on Braintrust hardware: not stated ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "custom_endpoint_support": "yes: \"Add custom models or endpoints to use with the proxy\", with custom providers supporting \"self-hosted models\", fine-tuned models and proprietary AI services, configured at org or project level with endpoint, headers, streaming and cost metadata ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "byok_mode": "both",
      "merchant_of_record": "BYOK: the upstream provider (customer's own provider keys; project-level keys let you \"manage separate billing\") ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). Built-in models: Braintrust, via model credits and on-demand rates ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)). Explicit \"who invoices\" language: not stated.",
      "key_management_note": "Provider keys are configured at organization or project scope, project overriding org; project-level keys isolate usage and billing ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)). Keys are stored as one-way cryptographic hashes, never plaintext ([AI providers](https://www.braintrust.dev/docs/admin/ai-providers)); in self-hosting, project-level encrypted provider secrets stay in the customer data plane while org-level encrypted secrets sit in the control plane, and Braintrust servers/employees do not require data-plane access ([Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting)). The Gateway page states Braintrust \"cannot see your data and does not store or log API keys\" ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)); the hosted OSS proxy is described as having \"end-to-end encryption\" ([GitHub](https://github.com/braintrustdata/braintrust-proxy)).",
      "api_openai_chat_completions_note": "yes: `POST /v1/proxy/chat/completions`, \"fully compatible with the OpenAI SDK\" by setting base URL `https://api.braintrust.dev/v1/proxy` ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Proxy chat/completions API ref](https://www.braintrust.dev/docs/api-reference/proxy/proxy-chatcompletions)). Note the path is prefixed with `/v1/proxy`, not bare `/v1`.",
      "api_anthropic_messages_note": "yes: native Anthropic schema at `/v1/proxy/anthropic/messages`; `anthropic-version` and `x-api-key` headers are not required ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)); the Gateway page also lists Anthropic messages support ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "api_openai_responses_note": "yes: Responses is listed among the Gateway's supported OpenAI surfaces ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "api_embeddings_note": "yes: `/embeddings` (and Gemini `embedContent`/`batchEmbedContents`) ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "api_images_note": "n.a. (no image-generation endpoint on the proxy or gateway pages fetched: [AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "api_audio_note": "Partly: the proxy supports the OpenAI Realtime API over WebSockets at `/realtime` with audio output modalities and audio-transcript stream events; standalone STT/TTS endpoints are not documented ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "api_batch_note": "n.a. for generic batch inference; only Gemini `batchEmbedContents` is listed ([Gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "deployment_modes_note": "Hosted SaaS (`https://api.braintrust.dev/v1/proxy`, `https://gateway.braintrust.dev` in public preview), self-host / BYOC data plane (hybrid in the customer's own cloud account), and open-source self-deploy of the proxy ([Gateway](https://www.braintrust.dev/docs/deploy/gateway), [Self-hosting](https://www.braintrust.dev/docs/admin/self-hosting), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers), [GitHub braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)). On-prem/air-gapped: not documented.",
      "byok_mode_note": "both: you can use the proxy with \"your API key from any supported provider\" (even without a Braintrust account) or use \"a single Braintrust API key to access all AI providers through one interface\", and built-in models bill against Braintrust model credits ([AI proxy docs](https://www.braintrust.dev/docs/deploy/ai-proxy), [AI providers](https://www.braintrust.dev/docs/admin/ai-providers)).",
      "access_meta": {
        "qualified": [
          "api_batch"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "enumerated"
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "No request-timeout header or key on the fetched proxy/gateway docs ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "Default count and backoff: `n.a.` The documented failure path is failover to the next endpoint, not a retry counter ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "ORDERED, comma-separated: header `x-bt-fallback-providers`. Response headers `x-bt-used-endpoint`, `x-bt-failover-from` and `x-bt-failover-to` report what happened; `x-bt-endpoint-name` pins a specific endpoint ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "`not_configurable`, and what exists is health checking at the gateway's own edge rather than upstream provider ejection: the global endpoint `gateway.braintrust.dev` uses DNS latency-based routing plus health checks across `us-east-1`, `eu-west-1`, `us-west-2` and `ap-southeast-1` ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "rel_multi_region_config": "not_configurable",
      "rel_multi_region_note": "The vendor runs multiple regions behind one hostname with DNS health-checked routing; there is no user-facing cross-region failover config ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
      "rel_load_balancing_config": "not_configurable",
      "rel_load_balancing_note": "`not_configurable` at the weight level: the proxy automatically load-balances across multiple API keys configured for the same provider, and no weight key is documented ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy)).",
      "architecture_class": "edge_worker",
      "architecture_note": "Repo is TypeScript 99.0% and is documented as deployable to Vercel, Cloudflare Workers, AWS Lambda or Express ([braintrustdata/braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "MIT-licensed repo with deploy targets for Vercel, Cloudflare, AWS Lambda and a Node/Express server ([braintrustdata/braintrust-proxy](https://github.com/braintrustdata/braintrust-proxy)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported, and failover is documented as working for streaming requests. Caveat: caching is disabled for requests that went through failover ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "The AI proxy documentation describes routing, caching, and logging; no PII scanner ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))",
      "guard_injection": "not_documented",
      "guard_injection_note": "Same ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))",
      "guard_content": "not_documented",
      "guard_content_note": "Same ([Braintrust proxy repo](https://github.com/braintrustdata/braintrust-proxy))",
      "guard_custom": "not_documented",
      "guard_custom_note": "No request-path policy engine documented; policies exist as scorers applied to logs, not as blocking checks ([View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "No admin model allow/deny list documented for the proxy ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))",
      "guard_execution_location": "not_documented",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))",
      "log_content": "configurable",
      "log_content_note": "Logging through the proxy requires passing an `x-bt-parent` header to name the destination project; without it, requests are not logged, and the repo lists automatic logging as \"coming soon\" ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy), [Braintrust proxy repo](https://github.com/braintrustdata/braintrust-proxy)). When logging is enabled, full request/response spans including prompt and completion text are stored ([View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Omitting `x-bt-parent` sends the request through without logging ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry is supported natively: Braintrust exposes an OTel-compatible backend endpoint and documents OpenLLMetry and Vercel AI SDK instrumentation, with multi-step spans forming agent traces ([Braintrust OpenTelemetry](https://www.braintrust.dev/docs/integrations/sdk-integrations/opentelemetry), [View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "export_destinations": [
        "Braintrust"
      ],
      "export_destinations_note": "CSV and JSON download from the logs UI, plus `bt sync pull` for NDJSON extraction to local/customer storage; the OTel endpoint is an ingest, not an export ([View logs](https://www.braintrust.dev/docs/observe/view-logs), [Braintrust OpenTelemetry](https://www.braintrust.dev/docs/integrations/sdk-integrations/opentelemetry))",
      "feedback_capture": "yes",
      "feedback_capture_note": "`POST /v1/project_logs/{project_id}/feedback` attaches scores, comments, and metadata to a logged span ([Feedback API](https://www.braintrust.dev/docs/api-reference/logs/feedback-for-project-logs-events))",
      "eval_hooks": "yes",
      "eval_hooks_note": "Yes — online scoring runs automated scorers against logged production traffic inside Braintrust; it is a Braintrust platform feature ([View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "observability_requires_saas": "The proxy can be self-hosted in the customer environment, but logs, spans, and online scoring are Braintrust platform features, so full observability is tied to the Braintrust control plane ([Braintrust AI proxy](https://www.braintrust.dev/docs/deploy/ai-proxy), [View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "log_retention_note": "`n.a.` — No default retention window found on the fetched pages ([View logs](https://www.braintrust.dev/docs/observe/view-logs))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` No numeric latency figure on the fetched pages ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway)).",
        "benchmark_independence": "No figures published ([Braintrust gateway](https://www.braintrust.dev/docs/deploy/gateway))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "not_documented",
      "guard_custom_default": "not_documented",
      "rel_notes": "The response headers (`x-bt-failover-from` / `x-bt-failover-to` / `x-bt-used-endpoint`) make failover observable per request, which is unusual and valuable for debugging silent degradation ([Braintrust proxy fallback reference](https://www.braintrust.dev/docs/api-reference/proxy/proxy-any-openai-request-fallback)).",
      "guard_notes": "This is an eval/observability platform with a proxy attached rather than a policy gateway: the strongest features are span-level logs, OTel ingest, feedback, and online scorers, with no request-path enforcement ([View logs](https://www.braintrust.dev/docs/observe/view-logs)).",
      "fit_agent_features_note": "Braintrust documents agent observability and tooling features rather than gateway passthrough capabilities. These include tool/function-call spans, Vercel AI SDK agent classes such as `Agent`, `ToolLoopAgent`, `WorkflowAgent`, and `HarnessAgent`, multi-step tool interactions, OpenAI Responses API function calls and web search spans, Anthropic server-side tool use, and tracing for Claude Code, Codex, OpenCode, and pi sessions. Sessions/threads and A2A are not documented. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))",
      "fit_byok_required": "optional",
      "fit_byok_required_note": "Optional, not required: with an account \"you can use a single Braintrust API key to access all AI providers\"; without an account \"you can use the proxy with individual provider API keys.\" For tracing, \"you must use a `BRAINTRUST_API_KEY` rather than a provider's key.\" ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))",
      "fit_client_sdk_langs": [
        "TypeScript",
        "Python",
        "Go",
        "Java",
        "Ruby",
        "C#"
      ],
      "fit_client_sdks_note": "Documented provider clients include the OpenAI SDK (`openai` for TypeScript/Python), Anthropic SDK (`@anthropic-ai/sdk` and `anthropic`), Google GenAI SDK (`@google/genai` and `google.genai`), and cURL. Braintrust’s official/native SDKs are documented for TypeScript/JavaScript (`braintrust`), Python (`braintrust`), Go, Java, Ruby, and C#/.NET; `@braintrust/otel` provides OpenTelemetry functionality. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Braintrust documents keyless cloud authentication and IAM reuse. AWS Bedrock supports STS `AssumeRole`; Azure AI Foundry and Azure OpenAI support Microsoft Entra Workload Identity using Braintrust-signed OIDC tokens; Google Vertex AI supports Workload Identity Federation using Braintrust-signed OIDC tokens. Self-hosted GCP deployments also use Workload Identity for GCS, and Azure uses workload identity for Key Vault. ([Braintrust changelog](https://www.braintrust.dev/docs/changelog))",
      "fit_cloudflare_workers": "user_facing",
      "fit_cloudflare_workers_note": "Cloudflare Workers are supported. Documented integrations include `OpenAIRealtimeWebSocket` from `openai/realtime/websocket` for realtime connections and `wrapCloudflareAIChat()` from `braintrust` for Cloudflare AI Chat tracing; Cloudflare Workers require `nodejs_compat`, a `BRAINTRUST_API_KEY` Wrangler secret, and flushing with `ctx.waitUntil(logger.flush())`. ([Braintrust Cloudflare AI Chat integration](https://www.braintrust.dev/docs/integrations/agent-frameworks/cloudflare-ai-chat))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated on the proxy page ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy)); the hosted Gateway \"is described as in public preview and free to use; pricing is announced before general availability\" ([Use the Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway))",
      "fit_ecosystem_notes": "Braintrust offers Braintrust-hosted, BYOC, and self-hosted deployments across AWS, GCP, and Azure. The hosted Gateway is at `https://gateway.braintrust.dev`, while self-hosted data planes can run the Gateway and serve `/v1/proxy`; the AI proxy documentation says the AI proxy is deprecated and recommends the Gateway for production use. Hosted Gateway logging writes to the organization’s configured data plane, and provider API keys are configured in Braintrust. ([Braintrust Gateway documentation](https://www.braintrust.dev/docs/deploy/gateway))",
      "fit_eval_prereqs": "Either a Braintrust API key or a provider key: \"You can use the proxy without a Braintrust account by providing your API key from any supported provider.\" No credit card, cloud account, or cluster is stated ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))",
      "fit_eval_shape": "base_url_swap",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes. `client.chat.completions.create({model: \"gpt-4o-mini\", messages: [{role:\"user\", content:\"What is a proxy?\"}], seed: 1})` with `baseURL: \"https://api.braintrust.dev/v1/proxy\"`; cURL equivalent posts to `.../v1/proxy/chat/completions` ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))",
      "fit_gateway_platform": "service_mesh",
      "fit_gateway_platform_note": "Chart-managed Istio integration is documented through an Istio `VirtualService`; enable `virtualService.workloadIsolation.enabled: true`, along with `virtualService.enabled: true` and `api.workloadIsolation.enabled: true`. No Kong, APISIX, Envoy, nginx, or other gateway plugin is documented. ([Braintrust self-hosting scaling configuration](https://ai-proxy-5k9q3mg0z.preview.braintrust.dev/docs/admin/self-hosting/configure/scaling))",
      "fit_helm_chart": "braintrust from oci://public.ecr.aws/braintrust/helm/braintrust",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Official self-hosting deployment uses the Helm chart named `braintrust`, installed from `oci://public.ecr.aws/braintrust/helm/braintrust`, with the chart repository at `https://github.com/braintrustdata/helm`. Other evidence documents chart versions `6.2.1`, `6.13.0+`, and `v5.0.1+` for specific self-hosting and scaling features. ([Braintrust self-hosting deployment](https://www.braintrust.dev/docs/admin/self-hosting/deploy))",
      "fit_mcp": "hosted_mcp_server",
      "fit_mcp_note": "Braintrust provides a hosted MCP server at `https://api.braintrust.dev/mcp` using streamable HTTP, with support for Claude Code, Cursor, Codex, VS Code, and other MCP-compatible clients. It supports reading and writing Braintrust data, querying logs, authoring prompts and scorers, configuring monitoring, and running evals; self-hosted deployments use the MCP URL shown in organization data-plane settings. ([Braintrust MCP integration](https://www.braintrust.dev/docs/integrations/developer-tools/mcp))",
      "fit_meta": {
        "not_documented_fields": []
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "No model gate: \"The proxy supports over 100 models... New models are added regularly.\" One org-level enablement gate exists for infra monitoring: \"The Infra dashboard option is available once Braintrust has enabled infrastructure monitoring for your organization.\" ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy), [Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))",
      "fit_onboarding_notes": "The proxy is deprecated: \"Deprecated. The AI proxy is deprecated and will no longer be regularly maintained,\" directing users to the gateway \"for production-grade reliability.\" WebSocket/realtime use requires a different base URL (`https://braintrustproxy.com/v1`). Temporary, time-limited credentials can be minted via `/credentials` for browser/mobile clients. Source is open at github.com/braintrustdata/braintrust-proxy ([AI proxy - Braintrust](https://www.braintrust.dev/docs/guides/proxy))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. The AI proxy is “fully compatible with the OpenAI SDK”; set `baseURL`/`base_url` to `https://api.braintrust.dev/v1/proxy`. The Gateway documentation also supports keeping the OpenAI SDK and changing the base URL to `https://gateway.braintrust.dev`, with a Braintrust API key. ([Braintrust Gateway documentation](https://www.braintrust.dev/docs/deploy/gateway))",
      "fit_prod_prereqs": "Hosted Gateway: only `BRAINTRUST_API_KEY` ([Use the Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway)). Self-hosted data plane: Braintrust API, PostgreSQL 17+ (8+ vCPU, 64GB+ RAM, 1000GB+, 15,000+ IOPS), Redis 7+, object storage and Brainstore (NVMe, 150,000+ IOPS, dedicated Kubernetes nodes; 2+ readers, 2+ fast readers, 1+ writer), production API 4+ instances ([Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))",
      "fit_prod_selfhost_shape": "infra_rollout",
      "fit_prod_shape": "base_url_swap",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LangChain and LlamaIndex integrations are documented. The LlamaIndex integration uses packages including `braintrust`, `llama-index`, `llama-index-llms-openai`, and `openinference-instrumentation-llama-index`, with `braintrust.auto_instrument()`, `setup_llamaindex()`, `LlamaIndexInstrumentor`, and `BraintrustSpanProcessor`. ([Braintrust LlamaIndex integration](https://www.braintrust.dev/docs/integrations/sdk-integrations/llamaindex))",
      "fit_quickstart_steps": 0,
      "fit_quickstart_steps_caveat": "The page carrying the quickstart and first-call snippet is marked deprecated, and it publishes no numbered procedure. The current Gateway page has no quickstart at all.",
      "fit_quickstart_url": "https://www.braintrust.dev/docs/guides/proxy",
      "fit_self_host_docs": "documented_no_command",
      "fit_self_host_install": "No literal command is published; self-hosting uses \"official Terraform modules\" on AWS (ECS/EC2) and GCP/Azure (Kubernetes and Helm), with Braintrust hosting the control plane ([Self-hosting Braintrust](https://www.braintrust.dev/docs/admin/self-hosting))",
      "fit_terraform": "official_modules",
      "fit_terraform_note": "Official Terraform modules are documented for AWS, GCP, and Azure: `github.com/braintrustdata/terraform-aws-braintrust-data-plane`, `github.com/braintrustdata/terraform-google-braintrust-data-plane`, and `github.com/braintrustdata/terraform-azure-braintrust-data-plane`. Additional documentation identifies Terraform module versions `v5.2.1` and `v6.0` for specific AWS deployments; no Terraform Registry name is stated. ([Braintrust self-hosting deployment](https://www.braintrust.dev/docs/admin/self-hosting/deploy))",
      "fit_terraform_ref": "terraform-aws-braintrust-data-plane, and Google and Azure equivalents, on GitHub; no Registry name",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "integration_documented",
      "fit_vercel_ai_sdk_note": "Braintrust documents a Vercel AI SDK integration. Install `braintrust`, `ai`, and `zod` with `pnpm add braintrust ai zod` or `npm install braintrust ai zod`; use `wrapAISDK` for AI SDK v3–v6 or `braintrustAISDKTelemetry()` for AI SDK v7. ([Braintrust Vercel AI SDK integration](https://www.braintrust.dev/docs/integrations/sdk-integrations/vercel))",
      "fit_vercel_ai_sdk_package": "braintrust wrapAISDK (telemetry wrapper, not a provider)"
    },
    {
      "slug": "helicone",
      "name": "Helicone",
      "company": "Helicone (Mintlify)",
      "tagline": "Open-source LLM observability platform with an OpenAI-compatible AI gateway attached.",
      "category": "managed_gateway",
      "deployment": "both",
      "license": "Apache-2.0",
      "founded_year": 2023,
      "hq_country": "United States",
      "url_home": "https://www.helicone.ai",
      "url_docs": "https://docs.helicone.ai/gateway/overview",
      "url_pricing": "https://www.helicone.ai/pricing",
      "url_status": "https://status.helicone.ai",
      "url_github": "https://github.com/Helicone/helicone",
      "url_changelog": null,
      "url_models_api": "https://api.helicone.ai/v1/public/model-registry/models",
      "github_stars": 6109,
      "model_count": 111,
      "model_count_note": "Live public model registry returned 111 models spanning 21 provider endpoints on 2026-08-29; docs describe the gateway as 100+ models / 100+ providers. The simpler https://ai-gateway.helicone.ai/v1/models endpoint returns the same 111 models without pricing.",
      "provider_count": 21,
      "token_markup_pct": 0,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Bring your own provider keys, or add Helicone credits and pay provider rates with 0% markup while Helicone manages provider keys.",
      "seat_fee_usd_month": 0,
      "free_tier": "Hobby: free, 1 seat, 10,000 requests/month, 1 GB storage, 7-day retention, 10 logs/min ingestion.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Pro plan",
          "amount": "$79/month, unlimited seats, 10k requests + 1 GB free then usage-based"
        },
        {
          "label": "Team plan",
          "amount": "$799/month, 5 organizations, 3-month retention"
        },
        {
          "label": "Usage overage",
          "amount": "Metered per request and per GB of storage beyond included amounts"
        }
      ],
      "self_host_cost_note": "Main platform repo is Apache-2.0 and self-hostable (infrastructure cost only); the standalone Rust AI gateway repo is GPL-3.0 and was last pushed 2025-11-21, which an independent review flags as effectively stale.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions"
      ],
      "sdk_note": "Point the OpenAI SDK at https://ai-gateway.helicone.ai with a Helicone API key; requests are logged automatically.",
      "cap_failover": true,
      "cap_load_balancing": null,
      "cap_conditional_routing": null,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": true,
      "cap_guardrails": false,
      "cap_pii_redaction": false,
      "cap_budgets": null,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No vendor-published managed-gateway overhead. An independent review (Chatforest, researched May 2026) cites a self-hosted proxy overhead of under 1 ms; another (Continuum, 2026-08-19) says the proxy adds a hop and recommends measuring on your own traffic.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": true,
      "eu_residency": true,
      "eu_residency_note": "Choice of US or EU data region, which Helicone presents as the route to GDPR compliance.",
      "zdr": true,
      "zdr_note": "Not branded as ZDR, but async logging can capture metadata only, and you can decline to send bodies to the backend at all.",
      "air_gapped": null,
      "best_for": "Teams whose main need is per-request, per-user LLM observability with a thin gateway bolted on, ideally self-hosted.",
      "strengths": [
        "0% markup on tokens with credits or BYOK",
        "Apache-2.0 platform that can be self-hosted for full data control",
        "Strong observability: sessions, user analytics, custom properties, HQL, alerts",
        "Public model registry endpoint exposing per-endpoint pricing",
        "US or EU data region choice, SOC 2 Type II and HIPAA listed on plans"
      ],
      "weaknesses": [
        "Acquired by Mintlify on 2026-03-03 and reported to be in maintenance mode with no new features planned",
        "Log- and storage-based pricing gets expensive for very high request counts",
        "Hobby plan drops logs above 10 logs/min, which can silently truncate agent traces",
        "Observability-first: routing, guardrails and PII redaction are weaker than Portkey or OpenRouter",
        "Standalone Rust gateway repo (GPL-3.0) has not had a functional commit since July 2025"
      ],
      "avoid_if": "You need an actively developed routing-first gateway with guardrails and policy enforcement.",
      "reviews": [
        {
          "url": "https://clickhouse.com/blog/helicones-migration-from-postgres-to-clickhouse-for-advanced-llm-monitoring",
          "why": "Third-party engineering writeup of Helicone's proxy/observability architecture and why its request-log workload outgrew Postgres, with the actual migration path described.",
          "date": "2026-04-17",
          "type": "written",
          "title": "Helicone's Migration from Postgres to ClickHouse for Advanced LLM Monitoring",
          "publisher": "ClickHouse"
        }
      ],
      "sources": {
        "zdr": "https://docs.helicone.ai/faq/compliance",
        "gdpr": "https://docs.helicone.ai/faq/compliance",
        "soc2": "https://www.helicone.ai/pricing",
        "company": "https://chatforest.com/reviews/helicone-llm-observability-gateway/",
        "license": "https://docs.helicone.ai/references/open-source",
        "sdk_note": "https://docs.helicone.ai/getting-started/quick-start",
        "url_docs": "https://docs.helicone.ai/gateway/overview",
        "zdr_note": "https://docs.helicone.ai/faq/compliance",
        "cap_evals": "https://www.helicone.ai/pricing",
        "free_tier": "https://www.helicone.ai/pricing",
        "hipaa_baa": "https://www.helicone.ai/pricing",
        "deployment": "https://docs.helicone.ai/faq/compliance",
        "extra_fees": "https://www.helicone.ai/pricing",
        "hq_country": "https://www.ycombinator.com/companies/helicone",
        "url_github": "https://github.com/Helicone/helicone",
        "url_status": "https://status.helicone.ai",
        "zdr_status": "https://docs.helicone.ai/features/advanced-usage/omit-logs",
        "acquired_by": "https://www.mintlify.com/blog/mintlify-acquires-helicone",
        "model_count": "https://api.helicone.ai/v1/public/model-registry/models",
        "url_pricing": "https://www.helicone.ai/pricing",
        "api_surfaces": "https://docs.helicone.ai/gateway/overview",
        "cap_failover": "https://docs.helicone.ai/gateway/overview",
        "eu_residency": "https://docs.helicone.ai/faq/compliance",
        "founded_year": "https://www.ycombinator.com/companies/helicone",
        "github_stars": "https://api.github.com/repos/Helicone/helicone",
        "product_fate": "https://www.helicone.ai/blog/joining-mintlify",
        "regions_note": "https://docs.helicone.ai/faq/compliance",
        "acquired_date": "https://www.mintlify.com/blog/mintlify-acquires-helicone",
        "byok_fee_note": "https://docs.helicone.ai/gateway/overview",
        "vendor_status": "https://www.mintlify.com/blog/mintlify-acquires-helicone",
        "byok_supported": "https://docs.helicone.ai/getting-started/quick-start",
        "cap_guardrails": "https://continuumcode.ai/guides/helicone-review/",
        "certifications": "https://docs.helicone.ai/faq/soc2",
        "config_as_code": "https://github.com/Helicone/ai-gateway",
        "provider_count": "https://api.helicone.ai/v1/public/model-registry/models",
        "retention_days": "https://www.helicone.ai/pricing",
        "url_models_api": "https://docs.helicone.ai/rest/models/get-v1public-model-registry-models",
        "cap_rate_limits": "https://docs.helicone.ai/gateway/overview",
        "logs_by_default": "https://docs.helicone.ai/features/advanced-usage/omit-logs",
        "cap_cache_simple": "https://docs.helicone.ai/gateway/overview",
        "helicone_caching": "https://docs.helicone.ai/features/advanced-usage/caching",
        "helicone_pricing": "https://www.helicone.ai/pricing",
        "model_count_note": "https://docs.helicone.ai/gateway/overview",
        "token_markup_pct": "https://docs.helicone.ai/gateway/overview",
        "cap_observability": "https://docs.helicone.ai/gateway/overview",
        "cap_pii_redaction": "https://continuumcode.ai/guides/helicone-review/",
        "eu_residency_note": "https://docs.helicone.ai/faq/compliance",
        "last_release_date": "https://github.com/Helicone/helicone/releases/tag/v2025.08.21-1",
        "openai_compatible": "https://docs.helicone.ai/gateway/overview",
        "seat_fee_usd_month": "https://www.helicone.ai/pricing",
        "guardrails_location": "https://docs.helicone.ai/faq/compliance",
        "latency_source_note": "https://chatforest.com/reviews/helicone-llm-observability-gateway/",
        "requires_vendor_sdk": "https://docs.helicone.ai/getting-started/how-to-integrate",
        "self_host_cost_note": "https://continuumcode.ai/guides/helicone-review/",
        "helicone_rate_limits": "https://docs.helicone.ai/features/advanced-usage/custom-rate-limits",
        "cap_prompt_management": "https://docs.helicone.ai/gateway/overview",
        "data_export_available": "https://docs.helicone.ai/rest/request/post-v1requestquery-clickhouse",
        "prompt_transits_vendor": "https://docs.helicone.ai/faq/compliance",
        "helicone_gateway_overview": "https://docs.helicone.ai/gateway/overview",
        "helicone_custom_properties": "https://docs.helicone.ai/features/advanced-usage/custom-properties",
        "helicone_how_we_calculate_cost": "https://docs.helicone.ai/references/how-we-calculate-cost"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_evals": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_github": "2026-08-29",
        "url_status": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "eu_residency": "2026-08-29",
        "founded_year": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "provider_count": "2026-08-29",
        "url_models_api": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:40.755599+00:00",
      "vendor_status": "maintenance",
      "acquired_by": "Mintlify",
      "acquired_date": "2026-03-03",
      "acquisition_terms": "Terms undisclosed",
      "product_fate": "Mintlify announced the acquisition on 3 March 2026 and put Helicone into maintenance mode: security updates, bug fixes and new models keep shipping, but Mintlify says it will work closely with every customer to support a smooth migration to another platform. Helicone says its services remain live for the foreseeable future. The signup flow still advertises a free trial, so the maintenance status is easy to miss.",
      "last_release_date": "2025-08-21",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": "Latest tagged GitHub release v2025.08.21-1. The hosted changelog stops at 26 November 2025 — the stalest product in this catalogue by roughly a year.",
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "Hosted cloud in the US or EU, or self-hosted entirely inside your own perimeter.",
      "logs_by_default": "full_content",
      "logs_note": "Bodies are stored unless you use Omit Logs. Note the limit of that feature: it stops storage but the request and response are still sent to Helicone's backend.",
      "retention_days": 7,
      "retention_note": "Seven days on Hobby, one month on Pro, three months on Team, indefinite on Enterprise.",
      "zdr_status": "depends",
      "trains_on_customer_data": "not_published",
      "training_note": "No statement found in the privacy policy, terms or data-autonomy docs. Get it in writing.",
      "subprocessors_url": null,
      "regions_note": "US or EU for the hosted service, or self-host on your own Docker, Kubernetes or cloud.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II, report on request",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "described as GDPR-compliant in a blog post",
          "status": "claimed"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "detail": "cloud described as HIPAA-compliant; no executable BAA offer found",
          "status": "claimed"
        }
      },
      "guardrails_location": "none",
      "guardrails_note": "No guardrails or PII-redaction feature found. The nearest control is omitting logs.",
      "acquired_date_basis": "announced",
      "security_incidents": [],
      "pricing_model": "hybrid",
      "pricing_model_note": "Flat per-tier platform fee (Hobby free, Pro $79/mo, Team $799/mo, Enterprise custom) plus usage-based requests/storage. Gateway credits at 0% markup. 50% off first year for startups; $100 credit for OSS; free for students.",
      "min_commitment": "None stated for Pro/Team. Enterprise not stated.",
      "overage_note": "Pro/Team/Enterprise all include 10K requests + 1 GB storage with 'usage-based ↑' after — the per-unit overage rates are not published. Calculator implies storage pricing (0.30 GB → $0.97) without a stated rate.",
      "cache_support": "exact",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact-match gateway cache on Cloudflare's edge; cache key hashes seed, URL, full body, relevant headers and bucket index. Semantic cache not stated. Can combine with provider prompt caching. Helicone charges no cache surcharge, but cached responses count against your storage allowance.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "'Alerts' listed as a feature without tier marks.",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Global, per-user and per-property segments; pre-request. Helicone's rate limit is bounded by your provider's policy.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "'Webhooks' listed as a feature without tier marks.",
          "tier": null,
          "supported": true
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated as budgets, but rate limits can be denominated in cost — policies control 'request count or cost in cents within a time window' via Helicone-RateLimit-Policy, enforced pre-request with a 429.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Per user via Helicone-User-Id header; per tag/property via Custom Properties (project, feature, workflow stage, ticket). Per key/team not stated. Costs are estimates based on the model returned in the response body, not billed pass-through.",
      "cost_export": "Query API for filtered retrieval; property-filtered webhooks. CSV, S3 and warehouse export not stated; pricing page 'Data export' line has no format, tier or price.",
      "product_kind_note": "Observability platform that also proxies/routes: the repo calls it \"Helicone AI Gateway… The fastest, lightest, and easiest-to-integrate AI Gateway on the market\", \"The NGINX of LLMs\", \"Open-source, lightweight, and built on Rust\" ([GitHub Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)); the self-host Docker page calls Helicone an \"LLM observability platform\" whose Jawn service is the \"Jawn API + LLM Proxy\" ([Docker self-host](https://docs.helicone.ai/getting-started/self-host/docker)).",
      "deployment_modes": [
        "saas",
        "self_host"
      ],
      "deployment_note": "Self-hosting options are Manual, Docker Compose, Kubernetes with Helm charts, and cloud infrastructure ([Self-hosting overview](https://docs.helicone.ai/getting-started/self-host/overview)); the all-in-one image is `helicone/helicone-all-in-one:latest` for a \"local machine or server\" or \"Production Setup (Remote Server) — EC2, VPS, etc.\" ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)). Important gating: in the self-hosted version \"Other providers (Vertex AI, AWS Bedrock, Azure OpenAI) are not supported\" — only OpenAI and Anthropic ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "not_documented",
      "api_images": "not_documented",
      "api_audio": "not_documented",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap with \"any OpenAI SDK\" — \"No new SDKs to learn, no integrations to maintain\" — but model names must become provider-qualified (`gpt-4o` → `openai/gpt-4o`) ([GitHub](https://github.com/Helicone/ai-gateway)). Routing selectors are expressed in the model string, e.g. `gpt-4o-mini/openai` for a specific provider or `gpt-4o-mini/azure/clm1a2b3c` for a configured deployment ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "model_count_low": 100,
      "model_count_high": 100,
      "model_count_source_note": "\"1 API. 100+ models\" ([GitHub](https://github.com/Helicone/ai-gateway)); \"100+ Models Available\" ([Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)). Both undated.",
      "provider_count_low": 20,
      "provider_count_high": 100,
      "provider_count_source_note": "\"100+\" providers on the gateway docs (undated) ([Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview), [Gateway overview](https://docs.helicone.ai/gateway/overview)); the GitHub README says \"OpenAI, Anthropic, Google, AWS Bedrock, and 20+ more providers\" ([GitHub](https://github.com/Helicone/ai-gateway)); the self-hosted build supports only 2 (OpenAI, Anthropic) ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)). See contradictions.",
      "first_party_vs_partner": "Third-party routing to \"LLM Providers\" (OpenAI, Anthropic, AWS Bedrock, Google Vertex, \"20+ more\") ([GitHub](https://github.com/Helicone/ai-gateway)); with credits \"we manage provider API keys for you\" so requests still reach those providers ([Gateway overview](https://docs.helicone.ai/gateway/overview)). No Helicone-owned inference hardware claimed.",
      "custom_endpoint_support": "n.a.: not documented on the pages fetched; the closest is a \"configured deployment\" ID for provider deployments such as Azure ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "byok_mode": "both",
      "merchant_of_record": "Credits mode: Helicone invoices, at \"0% markup\", \"customers pay exactly provider charges\" ([Gateway overview](https://docs.helicone.ai/gateway/overview), [Provider routing](https://docs.helicone.ai/gateway/provider-routing)). BYOK/self-host mode: the upstream provider, since your own `OPENAI_API_KEY`/`ANTHROPIC_API_KEY` are used ([GitHub](https://github.com/Helicone/ai-gateway)). Explicit invoicing language: not stated.",
      "key_management_note": "Self-hosted: provider keys stay in your `.env`/environment as `PROVIDER_API_KEY` values, and the gateway itself \"handles API keys\", with the Helicone key only needed when authentication is enabled ([GitHub](https://github.com/Helicone/ai-gateway)). Cloud: keys are added in Provider Settings and BYOK keys are attempted before Helicone-managed keys ([Provider routing](https://docs.helicone.ai/gateway/provider-routing)). Rate limits can be scoped per user, team, per-API-key or globally ([GitHub](https://github.com/Helicone/ai-gateway)). Encryption specifics and whether Helicone can read request bodies in cloud mode: n.a. on pages fetched.",
      "api_openai_chat_completions_note": "yes: self-hosted path `http://YOUR_IP:8585/v1/gateway/oai/v1/chat/completions`; cloud/router usage is `client.chat.completions.create` against a gateway base URL such as `https://ai-gateway.helicone.ai/ai` or `http://localhost:8080/router/<name>` ([Docker](https://docs.helicone.ai/getting-started/self-host/docker), [GitHub](https://github.com/Helicone/ai-gateway)).",
      "api_anthropic_messages_note": "yes: `http://YOUR_IP:8585/v1/gateway/anthropic/v1/messages` ([Docker](https://docs.helicone.ai/getting-started/self-host/docker)).",
      "api_openai_responses_note": "n.a. (not mentioned on the gateway or self-host pages fetched: [GitHub](https://github.com/Helicone/ai-gateway), [Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)).",
      "api_embeddings_note": "n.a. (not documented on pages fetched: [Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Integrations overview](https://docs.helicone.ai/gateway/integrations/overview)).",
      "api_images_note": "n.a. (same pages, not documented).",
      "api_audio_note": "n.a. (same pages, not documented).",
      "api_batch_note": "n.a. (same pages, not documented).",
      "deployment_modes_note": "Hosted SaaS (`https://ai-gateway.helicone.ai/ai`), self-host (local binary via `npx @helicone/ai-gateway@latest`, Docker/Docker Compose, Kubernetes/Helm, cloud deployment on AWS/GCP/Azure) ([GitHub](https://github.com/Helicone/ai-gateway), [Self-hosting overview](https://docs.helicone.ai/getting-started/self-host/overview)); on-prem is implied by the `NEXT_PUBLIC_IS_ON_PREM` flag ([Docker self-host](https://docs.helicone.ai/getting-started/self-host/docker)). Hybrid/VPC and air-gapped: not documented.",
      "byok_mode_note": "both: BYOK keys are tried first with Helicone-managed keys as fallback, and credits give access to \"100+ other providers without signing up for each one\" at \"0% markup\" ([Provider routing](https://docs.helicone.ai/gateway/provider-routing), [Gateway overview](https://docs.helicone.ai/gateway/overview)).",
      "access_meta": {
        "qualified": [],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` as a settable value. Timeout appears only as a failover trigger: `408` \"Timeout errors\" causes the gateway to try the next provider ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)). Retry headers include `Helicone-Retry-Min-Timeout` / `Helicone-Retry-Max-Timeout`, but these bound backoff, not the request ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).",
      "rel_retry_config": "per_request",
      "rel_retry_note": "`per_request` via headers: `Helicone-Retry-Enabled`, `Helicone-Retry-Num` (default `5`), `Helicone-Retry-Factor` (default `2`), `Helicone-Retry-Min-Timeout` (default `1000` ms), `Helicone-Retry-Max-Timeout` (default `10000` ms). Backoff is exponential: 1s → 2s → 4s → 8s → 10s, on `429`, `500`, `502`, `503`, `504` ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).",
      "rel_retry_default_note": "`per_request` via headers: `Helicone-Retry-Enabled`, `Helicone-Retry-Num` (default `5`), `Helicone-Retry-Factor` (default `2`), `Helicone-Retry-Min-Timeout` (default `1000` ms), `Helicone-Retry-Max-Timeout` (default `10000` ms).",
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "`per_request` for the cloud gateway: ORDERED. `Helicone-Fallbacks` JSON header with `target-url`, `onCodes` and `bodyKeyOverride` per entry; the response header `Helicone-Fallback-Index` reports which entry served ([Helicone gateway fallbacks](https://docs.helicone.ai/getting-started/integration-method/gateway-fallbacks)). In the AI Gateway the ordered chain is expressed inside the `model` string, e.g. `model: \"gpt-4o-mini/azure,gpt-4o-mini/openai,gpt-4o-mini\"`, \"tried in the exact order you specify\" ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "`not_configurable`, and it is passive unhealthy-provider avoidance rather than a configurable circuit breaker: the gateway is \"always aware of provider uptimes and your rate limits\" and instantly tries the next provider on `429`, `401`, `400`, `408`, `500+` ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway), [Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "`not_documented` as failover. You can pin a specific regional deployment (`model: \"gpt-4o/azure/eu-frankfurt-deployment\"`, \"No data leaves the EU\"), but that is region pinning, not cross-region failover ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Weights ARE supported. YAML key `routers.<name>.load-balance.chat.strategy` with strategies: `model-latency`, provider latency P2C + PeakEWMA, weighted distribution based on model weight, and cost optimization ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)). In the cloud gateway, \"equal-cost providers are load balanced\" and BYOK keys are always tried before Helicone managed keys ([Helicone provider routing](https://docs.helicone.ai/gateway/provider-routing)).",
      "architecture_class": "compiled_binary",
      "architecture_note": "\"built on Rust\"; repo language breakdown Rust 96.7% ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "`yes`. npm-launched binary `npx @helicone/ai-gateway@latest --config config.yaml`, plus Docker and Kubernetes self-hosting ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).",
      "latency_claims": [
        {
          "url": "https://github.com/Helicone/ai-gateway",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "<5",
          "metric": "p95",
          "publisher": "vendor",
          "conditions": "README comparison table. Whether this is added overhead or full round trip is not stated; no RPS, payload, hardware or cache state given."
        },
        {
          "url": "https://github.com/Helicone/ai-gateway",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "~100",
          "metric": "cold start",
          "publisher": "vendor",
          "conditions": "Same README table, against a \"Typical Setup\" baseline of ~2s. Conditions not stated."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://github.com/Helicone/ai-gateway",
          "unit": "req/s",
          "scope": "capacity",
          "value": "~3,000",
          "metric": "sustained",
          "publisher": "vendor",
          "conditions": "README table against a \"Typical Setup\" baseline of ~500. Hardware not stated."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported; the fetched pages do not state a streaming-specific caveat for the AI Gateway ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)). Note the retry headers are documented independently of streaming ([Helicone retries](https://docs.helicone.ai/features/advanced-usage/retries)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "No PII scanner documented; the privacy control is body omission rather than detection ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))",
      "guard_injection": "not_documented",
      "guard_injection_note": "`not_documented` ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))",
      "guard_content": "sync_block",
      "guard_content_note": "With the header `Helicone-Moderations-Enabled: true`, Helicone calls OpenAI's moderation endpoint before the chat completion and returns HTTP 400 with `PROMPT_FLAGGED_FOR_MODERATION` when flagged, so the request never reaches the model ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))",
      "guard_custom": "not_documented",
      "guard_custom_note": "No customer-defined regex/word-list/JSON-schema policy engine documented ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "The gateway router config selects load-balance targets and rate limits per key, but no admin allow/deny policy on models is documented ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "OpenAI Moderation"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "No stated behaviour if the moderation call fails or times out ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations))",
      "log_content": "full_request_and_response",
      "log_content_note": "Request and response bodies (prompt and completion text) are logged unless omission is requested ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Headers `Helicone-Omit-Request: true` and `Helicone-Omit-Response: true`, or `disable_content_tracing()` in the SDK, keep metrics while dropping bodies ([Omit logs](https://docs.helicone.ai/features/advanced-usage/omit-logs))",
      "tracing_support": "otel",
      "tracing_note": "The AI Gateway repo states \"OpenTelemetry support for logs, metrics, and traces\" alongside built-in Helicone observability; representation of multi-step agent traces is not detailed on the fetched pages ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))",
      "export_destinations": [
        "OpenTelemetry",
        "Datadog",
        "Webhook",
        "PostHog"
      ],
      "export_destinations_note": "PostHog via `Helicone-Posthog-Key` / `Helicone-Posthog-Host` headers ([PostHog integration](https://docs.helicone.ai/gateway/integrations/posthog)); OpenTelemetry logs/metrics/traces from the gateway ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))",
      "feedback_capture": "yes",
      "feedback_capture_note": "`POST /v1/request/{id}/feedback` with a boolean `rating` ([Feedback](https://docs.helicone.ai/features/advanced-usage/feedback)); arbitrary numeric scores via `POST /v1/request/{requestId}/score` ([Scores](https://docs.helicone.ai/features/advanced-usage/scores))",
      "eval_hooks": "partial",
      "eval_hooks_note": "Scores can be pushed against logged requests, but the docs state plainly that \"Helicone doesn't run evaluations for you\" — you compute scores externally and post them ([Scores](https://docs.helicone.ai/features/advanced-usage/scores))",
      "observability_requires_saas": "The gateway is fully open-source and self-hostable, and the repo presents observability as part of it; the fetched pages do not enumerate which dashboard features are cloud-only ([Helicone AI Gateway repo](https://github.com/Helicone/ai-gateway))",
      "log_retention_note": "Plan-based and configurable only by upgrading: Hobby 7 days, Pro 1 month, Team 3 months, Enterprise forever ([Helicone pricing](https://www.helicone.ai/pricing))",
      "production_meta": {
        "sources": {},
        "qualified": [
          "guard_content"
        ],
        "latency_prose": "README \"Scalable for production\" table: **P95 latency `<5ms`** for Helicone AI Gateway vs `~60-100ms` for a \"Typical Setup\"; **cold start `~100ms`** vs `~2s`. What was measured is not stated (gateway overhead vs round trip is ambiguous), and no RPS, payload, hardware or cache conditions are given — the README defers to `benchmarks/README.md` ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).",
        "multi_surface": {},
        "throughput_prose": "Same table: **`~3,000` requests/sec** vs `~500` for a \"Typical Setup\". Hardware and conditions not stated ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).",
        "benchmark_independence": "All vendor-published and self-measured. **Flag:** the comparison column (\"Typical Setup\", `~60-100ms`, `~500` req/s) is a vendor figure about unnamed competing setups, with no methodology or named product — treat it as marketing, not a benchmark ([Helicone/ai-gateway](https://github.com/Helicone/ai-gateway))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "off_by_default",
      "guard_custom_default": "not_documented",
      "rel_notes": "Two distinct products share the brand: the header-driven cloud proxy (`Helicone-Retry-*`, `Helicone-Fallbacks`) and the Rust AI Gateway with YAML routers. Their config surfaces do not overlap, so docs for one do not describe the other ([Helicone gateway fallbacks](https://docs.helicone.ai/getting-started/integration-method/gateway-fallbacks), [Helicone/ai-gateway](https://github.com/Helicone/ai-gateway)).",
      "guard_notes": "Helicone's one true request-path control is moderation, and it is opt-in per request via a header rather than an org-wide policy — a per-call flag is easy for a client to omit ([Moderations](https://docs.helicone.ai/features/advanced-usage/moderations)).",
      "fit_agent_features_note": "Helicone documentation shows tool/function calling, session tracking, and grouping related API calls. The Vercel AI SDK integration documents multi-step orchestration using `Experimental_Agent` with `stopWhen: stepCountIs(5)`, while A2A is not documented. ([Helicone Vercel AI SDK integration](https://docs.helicone.ai/gateway/integrations/vercel-ai-sdk))",
      "fit_byok_required": "hosted_no_selfhost_yes",
      "fit_byok_required_note": "No for the hosted gateway (\"Want more control? You can bring your own provider keys instead,\" presented as optional) ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)). Yes in the self-hosted Docker path, whose first call sends `Authorization: Bearer $OPENAI_API_KEY` alongside `Helicone-Auth` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))",
      "fit_client_sdk_langs": [
        "Python",
        "TypeScript"
      ],
      "fit_client_sdks_note": "Documented clients and packages include the OpenAI Python package (`openai`, `from openai import OpenAI`), the OpenAI JavaScript package (`openai`), `@helicone/ai-sdk-provider`, `@ai-sdk/gateway`, `@ai-sdk/openai`, `llama-index-llms-helicone`, `boto3`, and `helicone-async`. Bedrock examples also document the classes `BedrockRuntimeClient`, `InvokeModelCommand`, `ConverseCommand`, and `HeliconeAsyncLogger`. ([Helicone client integrations](https://docs.helicone.ai/integrations/openai/python))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "vendor_runtime_only",
      "fit_cloudflare_workers_note": "Helicone states that its proxy leverages Cloudflare Workers running on Cloudflare’s global network to process requests at the edge. ([Helicone latency reference](https://docs.helicone.ai/references/latency-affect))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated as required for the first call; the signup link is labeled \"Sign up for free\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)) and pricing lists \"Hobby — Free\" plus \"Pro — 7-day free trial\" ([Helicone Pricing](https://www.helicone.ai/pricing))",
      "fit_ecosystem_notes": "Helicone documents hosted gateways including `https://ai-gateway.helicone.ai`, access to 100+ models or providers, logging and observability, routing, fallbacks, and BYOK through Helicone Providers. It also explicitly documents self-hosting a complete Helicone stack on Kubernetes/Helm, with AWS S3, Aurora, and EKS resources supported through Terraform; some older OpenAI integration pages are described as maintained but no longer actively developed. ([Helicone quick start](https://docs.helicone.ai/getting-started/quick-start))",
      "fit_eval_prereqs": "A Helicone account plus a Helicone API key. No credit card, cloud account, cluster, or provider key: \"Helicone maintains the keys for you.\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))",
      "fit_eval_shape": "base_url_swap",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes, in TypeScript, Python and curl: `POST https://ai-gateway.helicone.ai/chat/completions` with `Authorization: Bearer $HELICONE_API_KEY` and `{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, world!\"}]}` ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "helicone-core, helicone-infrastructure, helicone-monitoring, helicone-argocd",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Helicone explicitly supports self-hosting on Kubernetes and provides an official Helm chart. The chart repository is the Helicone Helm repository, with components including `helicone-core`, `helicone-infrastructure`, `helicone-monitoring`, and `helicone-argocd`; the Helm deployment uses `helm upgrade --install helicone ./helicone -f values.yaml`. ([Helicone Kubernetes self-hosting](https://docs.helicone.ai/getting-started/self-host/kubernetes))",
      "fit_mcp": "client_tooling",
      "fit_mcp_note": "Helicone provides an MCP server available as the npm package `@helicone/mcp`, including the tools `query_requests` and `query_sessions`. It supports MCP-compatible clients such as Claude Desktop, Claude Code, Cursor, and Codex, and `query_sessions` provides session search, filtering, pagination, and session performance metrics. ([Helicone MCP integration](https://docs.helicone.ai/integrations/tools/mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_gateway_platform"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "n.a. (not documented). No approval/quota/waitlist step; users can \"Switch between 100+ models by just changing the model name\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))",
      "fit_onboarding_notes": "Hosted quickstart needs no library install at all (curl works). Self-hosted is materially narrower: \"Self-hosted version supports OpenAI and Anthropic only,\" port 8585 \"has no proxy authentication; firewall restriction is required,\" and email verification must be performed manually after creating an account at `/signup` ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start), [Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Helicone documents OpenAI SDK-compatible integrations by changing the endpoint and authentication. Documented base URLs include `https://oai.helicone.ai/v1`, `https://ai-gateway.helicone.ai`, `https://ai-gateway.helicone.ai/v1`, and `https://vercel.helicone.ai/v1`, depending on the integration. ([Helicone OpenAI Python integration](https://docs.helicone.ai/integrations/openai/python))",
      "fit_prod_prereqs": "Hosted: credits on the account (\"You simply add credits to your account, and we handle the rest\") ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start)). Self-hosted: Docker; ports 3000/8585/9080 reachable; internal PostgreSQL (5432) and ClickHouse (8123); MinIO S3 storage; public IP/domain for remote deployments; Docker volumes for persistence; a reverse proxy for HTTPS; and production env vars `SITE_URL`, `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `NEXT_PUBLIC_APP_URL`, `NEXT_PUBLIC_HELICONE_JAWN_SERVICE`, `NEXT_PUBLIC_IS_ON_PREM=true`, `S3_ENDPOINT` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))",
      "fit_prod_selfhost_shape": "cli_or_container",
      "fit_prod_shape": "base_url_swap",
      "fit_python_frameworks": [
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LlamaIndex is documented with the packages `llama-index-llms-helicone`, `llama_index.core`, and `llama_index.llms.openai`, including the `Helicone` and `OpenAI` classes. LangChain and LlamaIndex are also discussed in a comparison blog post, which mentions `PromptTemplate`, `LLMChain`, `IndexToolConfig`, and `LlamaIndexTool`; the LangChain details are from that comparison post rather than a dedicated integration page. ([Helicone LlamaIndex integration](https://docs.helicone.ai/gateway/integrations/llamaindex))",
      "fit_quickstart_steps": 2,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.helicone.ai/getting-started/quick-start",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`docker pull helicone/helicone-all-in-one:latest` then `docker run -d --name helicone -p 3000:3000 -p 8585:8585 -p 9080:9080 helicone/helicone-all-in-one:latest` ([Docker - Helicone](https://docs.helicone.ai/getting-started/self-host/docker))",
      "fit_terraform": "shipped_code",
      "fit_terraform_note": "Helicone documents Terraform directories for creating AWS S3, Aurora, and EKS resources: `terraform/s3`, `terraform/aurora`, and `terraform/eks`. An official Helicone Terraform provider, module, or Terraform Registry name is not stated. ([Helicone Kubernetes self-hosting](https://docs.helicone.ai/getting-started/self-host/kubernetes))",
      "fit_terraform_ref": "terraform/s3, terraform/aurora and terraform/eks directories in the repo; no provider or module published",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get your first LLM request logged with Helicone in under 2 minutes using the AI Gateway.\" ([Quickstart - Helicone](https://docs.helicone.ai/getting-started/quick-start))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Helicone documents Vercel AI SDK integrations using `@helicone/ai-sdk-provider` with `createHelicone`, `@ai-sdk/openai` with `createOpenAI`, and `@ai-sdk/gateway` for gateway-specific functionality. ([Helicone Vercel AI SDK integration](https://docs.helicone.ai/gateway/integrations/vercel-ai-sdk))",
      "fit_vercel_ai_sdk_package": "@helicone/ai-sdk-provider"
    },
    {
      "slug": "kong-ai-gateway",
      "name": "Kong AI Gateway",
      "company": "Kong Inc.",
      "tagline": "AI plugins on the Kong API gateway, adding LLM routing, guardrails and token limits to existing API infrastructure.",
      "category": "managed_gateway",
      "deployment": "both",
      "license": "Open core",
      "founded_year": 2009,
      "hq_country": "United States",
      "url_home": "https://konghq.com/products/kong-ai-gateway",
      "url_docs": "https://developer.konghq.com/ai-gateway/",
      "url_pricing": "https://konghq.com/pricing",
      "url_status": "https://status.konghq.com",
      "url_github": "https://github.com/Kong/kong",
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": 44100,
      "model_count": null,
      "model_count_note": "No total model count is published; the Konnect Plus plan caps the AI Gateway at 5 unique LLM models, with more available on Enterprise.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "You supply your own upstream LLM credentials, stored centrally in Kong's config store; Kong charges for gateways, not tokens.",
      "seat_fee_usd_month": null,
      "free_tier": "30-day Konnect free trial with enterprise functionality; separately, Kong Gateway core is Apache-2.0 and free to self-host.",
      "enterprise_from_usd_year": null,
      "extra_fees": [],
      "self_host_cost_note": "Kong Gateway core is Apache-2.0 and self-hostable at infra cost only. The Plus plan is billed per gateway per month with the amount not published; fully self-hosted gateways plus audit logs and SSO require the custom-priced, annually billed Enterprise plan. Third-party reviews note the OSS build lacks semantic caching, analytics and compliance features found in Enterprise.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenAI Embeddings",
        "MCP",
        "A2A"
      ],
      "sdk_note": "No LLM SDK of its own: traffic goes through Kong routes using the AI Proxy / AI Proxy Advanced plugins, exposing a universal OpenAI-style API.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": null,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": null,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": 12,
      "latency_source_note": "Vendor-run: Kong's own benchmark (Claudio Acquaviva, Kong, 2025-07-07) compared Kong AI Gateway with Portkey OSS 1.9.19 and LiteLLM 1.63.7 on EKS c5.4xlarge nodes, 12 CPUs each, K6 with 400 VUs against a WireMock LLM, claiming Kong 228% faster than Portkey and 859% faster than LiteLLM with 65%/86% lower latency. A third-party guide summarizes that benchmark as 12 ms p50 gateway overhead and 8,200 RPS, and cautions that independent benchmarks show much smaller gaps (50-100%, not 228%) and that Kong's figures are best-case.",
      "throughput_rps": 8200,
      "soc2": true,
      "hipaa_baa": null,
      "gdpr": true,
      "eu_residency": null,
      "eu_residency_note": "A third-party comparison notes Kong offers an EU control-plane option and that prompt-carrying data planes can stay in customer infrastructure; Kong's own trust center does not state an EU residency guarantee.",
      "zdr": null,
      "zdr_note": "No ZDR commitment is published. The practical equivalent comes from the defaults: payload logging is off, and self-hosted logs go to storage you control.",
      "air_gapped": null,
      "best_for": "Enterprises already standardized on Kong for API management that want AI traffic governed by the same gateway, plugins and ops tooling.",
      "strengths": [
        "Battle-tested NGINX/OpenResty core with 44k GitHub stars and a decade of production plugin patterns",
        "Deep AI feature set: semantic caching and routing, RAG injector, prompt templates, MCP and A2A gateways",
        "Strong data-protection tooling: PII sanitizer across 20 categories and 9 languages, plus Azure/AWS/GCP/Lakera guardrail integrations",
        "Clear compliance posture: SOC 2 Type 2, GDPR, CCPA, PCI DSS, CSA STAR Level 1, NIST 800-218",
        "Reuses existing Kong auth, rate limiting, logging and OpenTelemetry pipelines for LLM traffic"
      ],
      "weaknesses": [
        "No pricing transparency: Plus is quoted per gateway per month with no published amount and Enterprise is custom",
        "Plus caps the AI Gateway at 5 unique LLM models, pushing broader model access to Enterprise",
        "Its headline performance advantage comes from a Kong-run benchmark against mock LLMs; a third-party guide says independent tests show far smaller gaps",
        "Third-party reviews report the free OSS build lacks semantic caching, analytics and compliance features"
      ],
      "avoid_if": "You want a lightweight LLM-first proxy with published prices and hundreds of models available out of the box without an enterprise contract.",
      "reviews": [
        {
          "url": "https://siliconangle.com/2025/04/02/kongs-updated-ai-gateway-helps-secure-ai-model-production-deployments/",
          "why": "Independent coverage of AI Gateway 3.10's RAG injector and PII-sanitization plugins and what they mean for teams putting model traffic through Kong in production.",
          "date": "2025-04-02",
          "type": "written",
          "title": "Kong's updated AI Gateway helps to secure AI model production deployments",
          "publisher": "SiliconANGLE"
        },
        {
          "url": "https://techcrunch.com/2024/02/15/kongs-new-open-source-ai-gateway-makes-building-multi-llm-apps-easier/",
          "why": "Trade-press analysis of the AI Gateway launch, explaining how Kong bolted multi-LLM routing, credential management and request/response transformers onto its existing gateway.",
          "date": "2024-02-15",
          "type": "written",
          "title": "Kong's new open source AI Gateway makes building multi-LLM apps easier",
          "publisher": "TechCrunch"
        },
        {
          "url": "https://www.youtube.com/watch?v=sWI4drhs0lc",
          "why": "Independent engineer configures Kong AI Gateway from scratch, wires up OpenAI/Anthropic/Azure providers, and shows load balancing, the prompt-decorator plugin and analytics.",
          "date": "2025-12-14",
          "type": "video",
          "title": "KONG AI GATEWAY | MODEL ABSTRACTION",
          "publisher": "YouTube - Rajib Deb"
        },
        {
          "url": "https://www.reddit.com/r/LLMDevs/comments/1v2mbno/litellm_alternatives_after_prod_outage_anyone/",
          "why": "9-comment thread from teams evaluating Kong as a LiteLLM replacement after a multi-hour routing outage, with operators reporting real cluster sizes and upgrade pain.",
          "date": "2026-07-21",
          "type": "community",
          "title": "LiteLLM alternatives after prod outage, anyone running TrueFoundry or Kong for 3+ months?",
          "publisher": "Reddit - r/LLMDevs"
        },
        {
          "url": "https://www.reddit.com/r/kubernetes/comments/1themkz/migrating_away_from_the_kong_enterprise_stack/",
          "why": "15-comment postmortem of a Kong Enterprise/Kong Mesh exit: shockingly high CPU/memory under API-key rate limiting and ~$70k/year saved after a month-long migration to Envoy Gateway.",
          "date": "2026-05-19",
          "type": "community",
          "title": "Migrating Away from the Kong Enterprise Stack",
          "publisher": "Reddit - r/kubernetes"
        }
      ],
      "sources": {
        "gdpr": "https://trust.konghq.com/",
        "soc2": "https://trust.konghq.com/",
        "cap_mcp": "https://konghq.com/pricing",
        "company": "https://konghq.com/about",
        "license": "https://github.com/Kong/kong",
        "sdk_note": "https://developer.konghq.com/ai-gateway/",
        "free_tier": "https://konghq.com/pricing",
        "hq_country": "https://konghq.com/about",
        "url_status": "https://status.konghq.com",
        "weaknesses": "https://dev.to/pranay_batta/best-open-source-ai-gateway-in-2026-2flb",
        "api_surfaces": "https://developer.konghq.com/ai-gateway/",
        "cap_failover": "https://developer.konghq.com/ai-gateway/",
        "founded_year": "https://konghq.com/about",
        "github_stars": "https://github.com/Kong/kong",
        "kong_pricing": "https://konghq.com/pricing",
        "product_fate": "https://developer.konghq.com/gateway/changelog/",
        "regions_note": "https://developer.konghq.com/konnect-platform/geos/",
        "vendor_status": "https://konghq.com/company/press-room/press-release/kong-to-unveil-ai-connectivity-vision-and-roadmap",
        "byok_supported": "https://developer.konghq.com/ai-gateway/",
        "cap_embeddings": "https://developer.konghq.com/ai-gateway/",
        "cap_guardrails": "https://developer.konghq.com/ai-gateway/",
        "certifications": "https://trust.konghq.com/",
        "config_as_code": "https://developer.konghq.com/deck/file/format/",
        "retention_days": "https://konghq.com/pricing",
        "sla_uptime_pct": "https://konghq.com/pricing",
        "throughput_rps": "https://apiscout.dev/guides/portkey-vs-kong-ai-gateway-llm-routing-2026",
        "cap_rate_limits": "https://konghq.com/pricing",
        "logs_by_default": "https://docs.jp.konghq.com/hub/kong-inc/ai-proxy/configuration/",
        "model_count_note": "https://konghq.com/pricing",
        "cap_observability": "https://developer.konghq.com/ai-gateway/",
        "cap_pii_redaction": "https://developer.konghq.com/ai-gateway/",
        "eu_residency_note": "https://opper.ai/blog/best-european-ai-gateways",
        "last_release_date": "https://developer.konghq.com/gateway/changelog/",
        "openai_compatible": "https://developer.konghq.com/ai-gateway/",
        "subprocessors_url": "https://konghq.com/legal/subprocessors",
        "cap_cache_semantic": "https://developer.konghq.com/ai-gateway/",
        "cap_load_balancing": "https://developer.konghq.com/ai-gateway/",
        "guardrails_location": "https://developer.konghq.com/plugins/ai-sanitizer/",
        "latency_overhead_ms": "https://apiscout.dev/guides/portkey-vs-kong-ai-gateway-llm-routing-2026",
        "latency_source_note": "https://konghq.com/blog/engineering/ai-gateway-benchmark-kong-ai-gateway-portkey-litellm",
        "requires_vendor_sdk": "https://developer.konghq.com/plugins/ai-proxy/",
        "self_host_cost_note": "https://konghq.com/pricing",
        "cap_prompt_management": "https://developer.konghq.com/ai-gateway/",
        "data_export_available": "https://developer.konghq.com/plugins/http-log/",
        "prompt_transits_vendor": "https://developer.konghq.com/gateway/deployment-topologies/",
        "cap_conditional_routing": "https://developer.konghq.com/ai-gateway/"
      },
      "verified_at": {
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "cap_mcp": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "free_tier": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "weaknesses": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "throughput_rps": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:42.049547+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed, with weekly gateway releases and new MCP governance features. Kong opened Paris and Milan offices in July 2026.",
      "last_release_date": "2026-08-26",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://konghq.com/legal/service-level-agreement",
      "claim_basis": {},
      "last_release_note": "Kong Gateway 3.14.0.14. The open-source Kong/kong repository moves more slowly: latest tag 3.9.3 on 17 June 2026.",
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "The data plane is yours to run, or Kong-hosted as serverless or dedicated cloud gateways. Konnect's control plane is Kong-hosted in a geography you pick.",
      "logs_by_default": "none",
      "logs_note": "The AI Proxy plugin ships with statistics and payload logging both set to false, so nothing about your prompts is recorded until you turn it on and point it at a sink you own.",
      "retention_days": 30,
      "retention_note": "Thirty days for Konnect analytics and seven for audit logs. Prompt bodies are not retained at all unless you enable payload logging.",
      "zdr_status": "not_published",
      "trains_on_customer_data": "not_published",
      "training_note": "Nothing on the trust center or subprocessor page addresses training.",
      "subprocessors_url": "https://konghq.com/legal/subprocessors",
      "regions_note": "Konnect control-plane geographies in Australia, the EU, the Middle East, the US, India and Singapore, with dedicated data planes across roughly 17 AWS, 13 Azure and 11 Google regions. Self-hosted and on-premises data planes are supported.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type 2, via trust center",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "listed on the trust center",
          "status": "certified"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "self_hosted",
      "guardrails_note": "The PII sanitizer calls an anonymizer service you run yourself, typically in a Docker container, so redaction happens inside your boundary.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "platform_fee",
      "pricing_model_note": "Flat per-gateway platform fee (Konnect Plus, no dollar figure published) + enterprise-only annual quote. Kong does not resell inference. Plans billed on a calendar month basis in arrears.",
      "min_commitment": "Enterprise 'billed annually' with duration set out in the individual Order Form; no minimum amount stated. Plus is monthly.",
      "overage_note": "No metered overage billing published. Plus enforces soft limits via email + in-app notifications prompting you to reduce usage or upgrade.",
      "cache_support": "semantic",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Semantic caching included in Plus under 'Cost Control'. Exact-match and provider passthrough not stated. Kong prices no tokens, so semantic cache hits save on the customer's provider bill; cached requests may still count toward Plus request/analytics limits.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Not stated (only resource-limit email/in-app notices).",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Token-based rate limiting, enforced pre-request, included in Plus.",
          "tier": "plus",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated. Nearest published control is 'LLM access control and auth' plus a 5-unique-model Plus ceiling.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Plus includes 'AI Observability: token-level tracking and real-time cost analytics'. Per key/user/team/tag/customer splits not stated.",
      "cost_export": "None stated on the pricing page.",
      "product_kind_note": "A routing/governance proxy layered on an API gateway: \"Connectivity and governance layer for modern AI-native applications built on top of Kong Gateway\" that \"routes AI requests to various providers through a provider-agnostic API\"; the AI Proxy plugin \"lets you transform and proxy requests to a number of AI providers and models\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "deployment_modes": [
        "saas",
        "self_host",
        "hybrid_vpc",
        "on_prem"
      ],
      "deployment_note": "Konnect provides \"a unified control plane\" while customers \"Decide where you want to host your Data Plane nodes, and whether you want Kong to host them or host them yourself\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)) — i.e. customer-cloud data planes with a SaaS control plane, plus fully self-managed Kong Gateway. Feature gating is by Kong Gateway version rather than deployment (retries/fallback in load balancing require v3.10+; REST-style responses/files/assistants/batches require v3.11+; native `llm_format` pass-through requires v3.10+) ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Provider availability also varies: \"Some providers may not be available depending on your Kong Gateway version\" ([AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Not a base-URL-swap SaaS: you configure the plugin on your own Kong routes, then call your gateway with an OpenAI-compatible SDK (\"You can combine these parameters with an OpenAI-compatible SDK…\") or a provider-native SDK in native `llm_format` mode ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Native formats supported: `anthropic`, `bedrock`, `cohere`, `gemini`, `huggingface` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Caveats: \"some providers don't support all route types\"; provider-specific params must go in `extra_body`; streaming is listed as a supported capability area without further detail ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).",
      "model_count_low": null,
      "model_count_high": null,
      "model_count_source_note": "n.a.: \"Number of models: not mentioned\" on the plugin page ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)); Kong ships no model catalog of its own on pages fetched.",
      "provider_count_low": 17,
      "provider_count_high": 17,
      "provider_count_source_note": "No vendor total stated; the AI Proxy plugin page lists 17 providers (OpenAI, Azure OpenAI, Amazon Bedrock, Anthropic, Gemini, Vertex AI, Cohere, Mistral, Hugging Face, Llama, xAI, Alibaba Cloud DashScope, Cerebras, DeepSeek, Ollama, Databricks, vLLM) and the providers index lists the same 17 names; both pages undated ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)).",
      "first_party_vs_partner": "Entirely third-party/self-hosted upstreams: Kong routes to \"various providers\" and \"upstream LLMs\", including self-hosted models fulfilled \"using select supported format transformations\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). No Kong-operated models.",
      "custom_endpoint_support": "yes in practice: Ollama, vLLM, Llama and Hugging Face are first-class provider options and the plugin explicitly fulfils requests to \"self-hosted models\" ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). A generic \"register any private URL\" flow is not spelled out on the pages fetched.",
      "byok_mode": "byok_only",
      "merchant_of_record": "The upstream provider or cloud, since the customer's own keys/IAM are used (Azure Managed Identity, AWS \"IAM credentials or assumed roles\") ([AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). Kong itself is positioned to help you bill your own consumers: \"Meter, bill, and monetize the entire AI connectivity data path… automate invoicing with Stripe or ERP integrations\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).",
      "key_management_note": "Keys are stored/referenced through Konnect Config Store, or replaced by cloud identity (Azure `azure_use_managed_identity`, `azure_client_id`; AWS IAM credentials/assumed roles); request-level API keys or bearer tokens can override static plugin values when `config.auth.allow_override` is true ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/), [AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)). Because the data plane can be customer-hosted — \"Running privately and self-hosted for full control and compliance\" — request content need not leave the customer's infrastructure ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).",
      "api_openai_chat_completions_note": "yes in effect: the plugin accepts requests in \"defined and standardized OpenAI formats\", uses \"the OpenAI format by default\", and exposes route type `llm/v1/chat` mapped to OpenAI Chat completions; the literal `/v1/chat/completions` path is the customer's own Kong route, not a fixed vendor path ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_anthropic_messages_note": "yes as native pass-through: setting `config.llm_format` to `anthropic` proxies requests upstream \"without payload format conversion\"; the literal `/v1/messages` path is not stated ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_openai_responses_note": "yes: route type `llm/v1/responses`, mapped to OpenAI \"Responses\" (v3.11+ for the REST-based full-text responses set) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_embeddings_note": "yes: route type `llm/v1/embeddings` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_images_note": "yes: `image/v1/images/generations` and `image/v1/images/edits` (plus `video/v1/videos/generations`) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_audio_note": "yes, both: `audio/v1/audio/speech` (TTS), `audio/v1/audio/transcriptions` (STT), `audio/v1/audio/translations` ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "api_batch_note": "yes: `llm/v1/batches` and `llm/v1/files` support \"asynchronous bulk LLM requests\" with CRUD via POST/GET/DELETE (v3.11+) ([AI Proxy plugin](https://developer.konghq.com/plugins/ai-proxy/)).",
      "deployment_modes_note": "Hosted control plane (Konnect SaaS), self-hosted traditional, hybrid, DB-less, and Kubernetes via Kong Ingress Controller — \"AI plugins are supported in all deployment modes, including Konnect, self-hosted traditional, hybrid, and DB-less, and on Kubernetes via the Kong Ingress Controller\"; an on-prem demo instance is launched with `curl -Ls https://get.konghq.com/ai/v1 | bash` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)). Air-gapped: not documented on pages fetched.",
      "byok_mode_note": "byok_only: Kong sells software/subscription, not model tokens; credentials are the customer's own provider keys, managed centrally (\"Use Konnect Config Store to store and reference your LLM provider API keys\", \"Centralized AI provider credential management\") ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Gateway providers](https://developer.konghq.com/ai-gateway/ai-providers/)). The term \"BYOK\" itself is not used.",
      "access_meta": {
        "qualified": [
          "api_openai_chat_completions",
          "api_anthropic_messages",
          "api_audio"
        ],
        "model_count_kind": null,
        "provider_count_kind": "enumerated"
      },
      "rel_timeout_config": "config_file",
      "rel_timeout_note": "Upstream/service timeouts `connect_timeout`, `read_timeout`, `write_timeout` are configurable, and timeout is a first-class failover condition ([Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).",
      "rel_retry_config": "config_file",
      "rel_retry_note": "Key `retries`, which retries on both error and timeout. Default retry count and backoff strategy are not stated on the fetched pages: `n.a.` ([Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "ORDERED and WEIGHTED are both available — the AI Proxy Advanced plugin's target list supports `weight` (documented example 70/25/5), and failover is controlled by `failover_criteria`, which defaults to error and timeout and can add `http_429`, `http_500` and `non_idempotent` ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/), [Kong AI Proxy Advanced](https://developer.konghq.com/plugins/ai-proxy-advanced/)).",
      "rel_fallback_shape": "weighted",
      "rel_health_check_config": "config_file",
      "rel_health_check_note": "`config_file`, and it is a real circuit breaker: from v3.13+ `config.balancer.max_fails` with `config.balancer.fail_timeout` ejects a failing upstream for a period ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)). Kong Gateway itself lists \"health checking\" as configurable through the admin API or declarative config ([Kong/kong](https://github.com/Kong/kong)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "`not_documented` as a named feature. You self-host the data plane wherever you like (Linux, Docker, Kubernetes, or Konnect's managed control plane), but no cross-region failover config is documented ([Kong Gateway install](https://developer.konghq.com/gateway/install/)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Weights supported (`weight` on upstream targets). Strategies include lowest-latency with `latency_strategy` (`tpot` default, or `e2e`) and peak-EWMA, plus consistent hashing via `hash_on_header` (default header `X-Kong-LLM-Request-ID`) ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "Repo language breakdown: Lua 89.2%, Perl 5.2%, Raku 3.1% ([Kong/kong](https://github.com/Kong/kong)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "Prebuilt `deb` and `rpm` packages, official Docker Hub images including a distroless image (AMD64 and ARM64), and Helm charts for Kong Gateway and Kong Ingress Controller ([Kong Gateway install](https://developer.konghq.com/gateway/install/)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported, including WebSocket realtime traffic. Documented caveat: the lowest-latency balancing algorithm is \"less suitable for long-lived connections like WebSockets\" ([Kong AI Gateway load balancing](https://developer.konghq.com/ai-gateway/load-balancing/)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "The AI PII Sanitizer plugin sends payloads to a customer-run `ai-pii-service` anonymiser covering ~20 categories, replaces matches with placeholders or synthetic values, and can restore originals on the response path; audit logs record `ai.sanitizer.pii_identified`, `pii_sanitized`, and per-entity original/redacted pairs ([AI Sanitizer plugin](https://developer.konghq.com/plugins/ai-sanitizer/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Kong itself ships regex prompt guarding; injection/jailbreak classification comes from plugged-in services such as Lakera Guard, whose `input_block_reason` / `output_block_reason` audit fields record blocked requests ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "guard_content": "sync_block",
      "guard_content_note": "The AI Azure Content Safety plugin \"analyzes the request against configured moderation categories and allows or blocks the request\", and does the same for responses; GCP Model Armor input/output block reasons are also logged ([AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))",
      "guard_custom": "sync_block",
      "guard_custom_note": "`ai-prompt-guard` takes PCRE allow and deny pattern lists and returns HTTP 400 on a deny match; Azure Content Safety blocklist IDs and per-category thresholds are also configurable ([AI Prompt Guard](https://developer.konghq.com/plugins/ai-prompt-guard/), [AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages — routing and plugin scoping are per route/service/consumer, but an explicit model allow/deny list feature is not documented in what was fetched ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI Proxy reference](https://developer.konghq.com/plugins/ai-proxy/reference/))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "AWS Bedrock Guardrails",
        "Azure Content Safety",
        "Google Model Armor",
        "Lakera"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` — no fail-open/fail-closed statement on the fetched guardrail plugin pages ([AI Azure Content Safety](https://developer.konghq.com/plugins/ai-azure-content-safety/), [AI AWS Guardrails reference](https://developer.konghq.com/plugins/ai-aws-guardrails/reference/))",
      "log_content": "configurable",
      "log_content_note": "AI plugin logging of statistics and payloads is controlled per plugin; Kong's AI audit log records guardrail verdicts and, for the sanitizer, the original and redacted entity values ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Payload logging is a plugin config toggle, so metrics can be kept without bodies ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry is native to Kong: the `opentelemetry` plugin exports OTLP-over-HTTP spans (with AI span attributes and metrics in the AI Gateway) and supports full request-lifecycle tracing ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "export_destinations": [
        "OpenTelemetry",
        "Prometheus",
        "Datadog",
        "StatsD",
        "HTTP log",
        "Kafka",
        "Loki"
      ],
      "export_destinations_note": "OpenTelemetry collectors/OTLP-HTTP endpoints for traces and metrics; Kong's AI audit log is emitted through Kong's logging plugins ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/)). A full enumerated destination list was `n.a.` on the fetched pages.",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — No feedback/score API documented ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` — No eval/online-scoring feature documented ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "observability_requires_saas": "No — Kong runs in the customer's infrastructure and exports telemetry to the customer's own OTel/metrics stack ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "log_retention_note": "`n.a.` — Retention is a property of whatever sink the customer ships to; not stated ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` No gateway latency figure appears on the fetched Kong pages ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)).",
        "benchmark_independence": "No figures published ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))."
      },
      "guard_pii_default": "sync_block",
      "guard_injection_default": "sync_block",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Kong is the only product here where the LLM-specific reliability config inherits a mature general-purpose proxy stack (upstream entities, retries, circuit breaker, consistent hashing), and the hybrid control-plane/data-plane deployment model is documented ([Kong/kong](https://github.com/Kong/kong)).",
      "guard_notes": "Kong is the clearest example of guardrails as first-class request-path plugins with an audit trail designed for them: every guardrail writes structured block-reason fields into the AI audit log, which is exactly what an auditor wants ([AI audit log reference](https://developer.konghq.com/ai-gateway/ai-audit-log-reference/)).",
      "fit_agent_features_note": "Kong documents function calling and custom tool use, including Gemini function declarations, OpenAI `llm/v1/chat` function calling, and LangChain `bind_tools` usage. AI Gateway also provides an A2A traffic gateway for securing, governing, and observing agent-to-agent traffic. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))",
      "fit_byok_required": "yes",
      "fit_byok_required_note": "Yes for the documented walkthrough: prerequisites include \"OpenAI account and API key\" ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/)); keys can be centralised via \"Konnect Config Store to store and reference your LLM provider API keys\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "fit_client_sdk_langs": [],
      "fit_client_sdks_note": "The documentation explicitly names the **OpenAI SDK** and **Gemini SDK**. The LangChain guide documents Python usage of `langchain-openai` and `ChatOpenAI`; official Kong client-library names and a broader supported-language list are not documented. ([Kong AI provider documentation](https://developer.konghq.com/ai-gateway/ai-providers/))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Kong documents Azure authentication through an Azure Compute resource’s Managed Identity or User-Assigned Identity, configured with `config.auth.azure_use_managed_identity` and optionally `config.auth.azure_client_id`. Amazon Bedrock authentication can use AWS IAM credentials or assumed roles. ([Kong AI provider documentation](https://developer.konghq.com/ai-gateway/ai-providers/))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented). No credit-card, free-tier or trial wording appears on the AI Gateway landing page ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/)); the get-started page refers to \"conserve your free trial credits or avoid unnecessary charges\" without stating a payment requirement ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/))",
      "fit_ecosystem_notes": "Kong AI Gateway supports Konnect, self-hosted traditional, hybrid, and DB-less deployment modes, and Kubernetes deployment through the Kong Ingress Controller. Kong also documents fully self-hosted deployments on cloud, bare metal, containers, or Kubernetes, alongside Konnect’s managed SaaS control plane. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))",
      "fit_eval_prereqs": "Konnect personal access token; Kong Gateway Enterprise; decK v1.65.2+; an OpenAI account and API key; a licence for Kong Gateway Enterprise (for the self-managed path). No credit card, cloud account or cluster stated ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/))",
      "fit_eval_shape": "cli_or_container",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes on the get-started page: `curl -X POST \"$KONNECT_PROXY_URL/chat\" ... --json '{\"model\":\"gpt-5-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Say this is a test!\"}]}'` ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/)). The AI Gateway landing page itself shows no completion request, only `curl -Ls https://get.konghq.com/ai/v1 | bash` ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "fit_gateway_platform": "is_itself_gateway",
      "fit_gateway_platform_note": "Kong AI Gateway is implemented through Kong Gateway plugins, including **AI Proxy** and **AI Proxy Advanced**, with a minimum Kong Gateway version of `3.6`. The **AI MCP Proxy** is an additional Kong Gateway plugin for connecting Kong-managed services to MCP, available from Kong Gateway `3.12` and supporting hybrid, DB-less, and traditional topologies. ([Kong AI Gateway overview](https://developer.konghq.com/ai-gateway/))",
      "fit_helm_chart": "kong/ingress and kong/kong from https://charts.konghq.com",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Kong provides official Kubernetes Helm charts from `https://charts.konghq.com`, including `kong/ingress` and `kong/kong`. Kong’s Kubernetes integration uses the Kong Ingress Controller; `kong/ingress` is recommended for new installations, while `kong/kong` supports hybrid and unmanaged Kong instances. ([Kong Helm charts](https://charts.konghq.com/))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "Kong’s AI MCP Proxy bridges MCP and HTTP, supports MCP tool discovery and invocation, proxies MCP requests, aggregates tools, and can convert REST API paths into MCP tools. Kong also provides `mcp-konnect`, an MCP server with tools for querying analytics and managing Konnect resources, usable with MCP clients such as Claude Desktop and Cursor. ([Kong AI MCP Proxy plugin](https://developer.konghq.com/plugins/ai-mcp-proxy/))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_credit_card",
          "fit_model_gate",
          "fit_vercel_ai_sdk"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented). No model approval, enablement, quota, or waitlist step appears ([How to: Get started with AI Gateway](https://developer.konghq.com/ai-gateway/get-started/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "fit_onboarding_notes": "AI features are Kong Gateway plugins: \"You can enable the AI Gateway features through a set of modern and specialized plugins,\" and the landing page instructs \"Run the Kong Gateway quickstart and enable the AI Proxy plugin.\" Configuration is applied with decK state piped to `deck gateway apply`; Kong Gateway version shown is 3.15.0.5. Config tooling spans AI Gateway editor, decK, Terraform, KIC, Admin API, and Control Plane Config API ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/), [How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/))",
      "fit_openai_dropin": "partial",
      "fit_openai_dropin_note": "Kong documents connecting the LangChain OpenAI adapter without code changes. The documented setting is `base_url=f'{kong_url}/{kong_route}'`, with `kong_url=\"http://127.0.0.1:8000\"`, `kong_route=\"gpt4o\"`, and `api_key=\"NONE\"` or a Kong consumer key. ([Kong LangChain integration guide](https://docs.jp.konghq.com/gateway/latest/ai-gateway/llm-library-integration-guides/langchain/))",
      "fit_prod_prereqs": "A Konnect personal access token and a Konnect Control Plane plus local Data Plane for the hybrid path, or Kong Gateway Enterprise with a licence exported as `KONG_LICENSE_DATA` for self-managed; decK v1.65.2+ and cURL. The page refers production users to the Install page and does not itself specify database/Redis/cluster/load-balancer requirements ([How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/)). Planning topics listed include \"AI Gateway resource sizing guidelines,\" \"Deployment topologies,\" and \"Hosting options\" ([Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "infra_rollout",
      "fit_python_frameworks": [
        "LangChain"
      ],
      "fit_python_frameworks_note": "Kong documents LangChain integration using the `langchain-openai` package and the `ChatOpenAI` class. Custom tools use `langchain_core.tools` and its `tool` decorator; LlamaIndex is not documented. ([Kong LangChain integration guide](https://docs.jp.konghq.com/gateway/latest/ai-gateway/llm-library-integration-guides/langchain/))",
      "fit_quickstart_steps": 3,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://developer.konghq.com/ai-gateway/get-started/",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`curl -Ls https://get.konghq.com/quickstart | bash -s -- -e KONG_LICENSE_DATA` (self-managed Enterprise); destroy with `curl -Ls https://get.konghq.com/quickstart | bash -s -- -d`; the AI-specific demo is `curl -Ls https://get.konghq.com/ai/v1 | bash` ([How to: Get started with Kong Gateway](https://developer.konghq.com/gateway/get-started/), [Kong AI Gateway](https://developer.konghq.com/ai-gateway/))",
      "fit_terraform": "official_provider",
      "fit_terraform_note": "Kong documents the official Terraform providers `konnect`, `konnect-beta`, `kong-gateway`, and `kong-mesh`. The Terraform Registry provider source for the Kong Gateway provider is `Kong/kong-gateway`; the registry documentation states that Terraform 0.13+ is required. ([Kong Terraform documentation](https://developer.konghq.com/terraform/))",
      "fit_terraform_ref": "Kong/kong-gateway, konnect, konnect-beta and kong-mesh",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "not_documented",
      "fit_vercel_ai_sdk_note": "n.a. (not documented)",
      "fit_vercel_ai_sdk_package": null
    },
    {
      "slug": "orq-ai-router",
      "name": "Orq.ai Router",
      "company": "Orq.ai",
      "tagline": "Managed EU-hosted AI gateway and router bundled with evaluation, observability and governance tooling.",
      "category": "managed_gateway",
      "deployment": "both",
      "license": "Proprietary",
      "founded_year": 2022,
      "hq_country": "Netherlands",
      "url_home": "https://orq.ai/",
      "url_docs": "https://docs.orq.ai/docs/ai-gateway",
      "url_pricing": "https://orq.ai/pricing",
      "url_status": "https://status.orq.ai",
      "url_github": null,
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": null,
      "model_count": 500,
      "model_count_note": "Vendor-stated 500+ models from 30+ providers on the pricing page; the docs cite 300+ models and a competitor comparison cites 400+ from 28+ providers, so the number moves by page.",
      "provider_count": 30,
      "token_markup_pct": 0,
      "credit_fee_pct": 4.5,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "BYOK free for the first 1M requests/month, then 4%; no markup on model traffic itself.",
      "seat_fee_usd_month": null,
      "free_tier": "Pay-as-you-go starts free: 100k spans, 1 GB processed data, 500 agent runs, unlimited AI Gateway seats, 100 req/min limit, 30-day retention.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Orq-managed model credits",
          "amount": "4.5% fee on credits"
        },
        {
          "label": "BYOK above 1M requests/month",
          "amount": "4%"
        },
        {
          "label": "AI Studio seats",
          "amount": "EUR 35 per seat per month"
        },
        {
          "label": "Knowledge Bases and Agent Memory",
          "amount": "EUR 500 per month"
        },
        {
          "label": "Teams",
          "amount": "EUR 300 per month"
        },
        {
          "label": "Spans beyond 100k",
          "amount": "EUR 7 per 100k"
        },
        {
          "label": "Processed data beyond 1 GB",
          "amount": "EUR 3 per GB"
        },
        {
          "label": "Agent runs beyond 500",
          "amount": "EUR 0.01 per run"
        }
      ],
      "self_host_cost_note": "No open-source or free self-host option. Enterprise-only sovereign deployments: customer-owned VPC on AWS or Azure via marketplace, or fully on-prem/air-gapped via a single Helm chart, priced custom and billed annually.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": false,
      "portability_note": "Telemetry query API is in private preview.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenAI Responses",
        "Anthropic Messages",
        "MCP"
      ],
      "sdk_note": "OpenAI- and Anthropic-compatible endpoints plus first-party SDKs (orq-ai-sdk for Python, @orq-ai/node for JavaScript).",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": true,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": true,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No gateway proxy-overhead benchmark is published; the vendor states only that its Smart Router classifies each request in under 40 ms before model selection.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": true,
      "eu_residency": true,
      "eu_residency_note": "Vendor states all data is processed and stored in EU data centers on Google Cloud, with EU-based sovereign VPC and on-prem options on Enterprise. A third-party review of European gateways cautions that Orq's privacy policy describes storage on cloud database servers in the United States and that EU-only residency looks like a configuration or enterprise discussion rather than the default.",
      "zdr": true,
      "zdr_note": "Implemented as a routing filter rather than a storage setting: switch it on and the model pool narrows to providers that guarantee no request retention. Included on every plan.",
      "air_gapped": true,
      "best_for": "European teams that want an EU-resident managed gateway with governance, evaluations and observability in one platform rather than a bare proxy.",
      "strengths": [
        "Strongest EU/sovereignty story here: EU data centers, sovereign VPC, on-prem and air-gapped Helm deployment",
        "Zero markup on model traffic, with free BYOK up to 1M requests/month",
        "Zero-data-retention routing filter and PII redaction built into the gateway",
        "SOC 2 Type II, GDPR, HIPAA BAA availability and detailed audit logs with SIEM export",
        "Broad multimodal reach (image, video, audio, embeddings, rerank) plus 40+ evaluators and prompt versioning"
      ],
      "weaknesses": [
        "Closed source with no free self-host path; sovereign deployment is enterprise-only at unpublished prices",
        "Line-item pricing in euros across spans, processed data, agent runs, seats, teams and knowledge bases is hard to forecast",
        "Free tier is rate-limited to 100 requests/minute with only 30-day retention",
        "An independent European-gateway review disputes how EU-only the default data storage really is, and notes ISO 27001 is still in progress"
      ],
      "avoid_if": "You just need a thin high-throughput proxy, or you want a self-hostable open-source gateway without an enterprise contract.",
      "reviews": [
        {
          "url": "https://woutersligter.medium.com/scaling-ai-with-confidence-our-real-world-experience-with-orq-ai-9ce394d32c70",
          "why": "Consultant's hands-on account of running Orq.ai's model garden, routing, deployments, RAG and evaluators on client projects, with specific module-level criticism.",
          "date": "2025-04-24",
          "type": "written",
          "title": "Scaling AI with Confidence: Our Real-World Experience with Orq.ai",
          "publisher": "Medium - Wouter Sligter (KODIFY)"
        },
        {
          "url": "https://www.reddit.com/r/devops/comments/1ue3pzq/compared_openrouter_portkey_and_orqs_gateway_for/",
          "why": "Operator who ran all three gateways in production reports where Orq's routing and per-team cost breakdowns fall short (tier-gated), with commenters adding dashboard critiques.",
          "date": "2026-06-24",
          "type": "community",
          "title": "Compared OpenRouter, Portkey, and Orq's gateway for routing across providers",
          "publisher": "Reddit - r/devops"
        }
      ],
      "sources": {
        "zdr": "https://orq.ai/security",
        "gdpr": "https://orq.ai/security",
        "soc2": "https://orq.ai/security",
        "cap_mcp": "https://docs.orq.ai/docs/ai-gateway",
        "sdk_note": "https://docs.orq.ai/docs/ai-gateway",
        "url_docs": "https://docs.orq.ai/docs/ai-gateway",
        "zdr_note": "https://orq.ai/security",
        "cap_audio": "https://docs.orq.ai/docs/ai-gateway",
        "cap_evals": "https://orq.ai/",
        "cap_video": "https://docs.orq.ai/docs/ai-gateway",
        "free_tier": "https://orq.ai/pricing",
        "hipaa_baa": "https://docs.orq.ai/docs/enterprise/sovereign-ai",
        "air_gapped": "https://docs.orq.ai/docs/enterprise/sovereign-ai",
        "extra_fees": "https://orq.ai/pricing",
        "hq_country": "https://orq.ai/about-us",
        "url_status": "https://status.orq.ai",
        "zdr_status": "https://docs.orq.ai/docs/enterprise/eu-regions-faq",
        "cap_budgets": "https://orq.ai/pricing",
        "model_count": "https://orq.ai/pricing",
        "orq_budgets": "https://docs.orq.ai/docs/ai-gateway/budgets",
        "orq_pricing": "https://orq.ai/pricing",
        "api_surfaces": "https://docs.orq.ai/docs/ai-gateway",
        "cap_failover": "https://docs.orq.ai/docs/ai-gateway",
        "eu_residency": "https://orq.ai/security",
        "founded_year": "https://docs.orq.ai/docs/enterprise/sovereign-ai",
        "product_fate": "https://docs.orq.ai/changelog",
        "regions_note": "https://docs.orq.ai/docs/enterprise/eu-regions-faq",
        "byok_fee_note": "https://orq.ai/pricing",
        "cap_image_gen": "https://docs.orq.ai/docs/ai-gateway",
        "vendor_status": "https://orq.ai/press-releases",
        "byok_supported": "https://orq.ai/pricing",
        "cap_embeddings": "https://docs.orq.ai/docs/ai-gateway",
        "cap_guardrails": "https://orq.ai/",
        "certifications": "https://docs.orq.ai/docs/enterprise/deployment-options",
        "config_as_code": "https://docs.orq.ai/docs/deployments/routing",
        "credit_fee_pct": "https://orq.ai/pricing",
        "provider_count": "https://orq.ai/pricing",
        "retention_days": "https://orq.ai/legal/security",
        "cap_rate_limits": "https://orq.ai/pricing",
        "logs_by_default": "https://docs.orq.ai/docs/administer/data-compliance",
        "cap_cache_simple": "https://docs.orq.ai/docs/ai-gateway",
        "model_count_note": "https://opper.ai/blog/best-european-ai-gateways",
        "token_markup_pct": "https://orq.ai/pricing",
        "cap_observability": "https://orq.ai/",
        "cap_pii_redaction": "https://orq.ai/security",
        "eu_residency_note": "https://opper.ai/blog/best-european-ai-gateways",
        "last_release_date": "https://docs.orq.ai/changelog",
        "openai_compatible": "https://docs.orq.ai/docs/ai-gateway",
        "cap_load_balancing": "https://docs.orq.ai/docs/ai-gateway",
        "guardrails_location": "https://docs.orq.ai/docs/enterprise/deployment-options",
        "latency_source_note": "https://docs.orq.ai/docs/ai-gateway",
        "requires_vendor_sdk": "https://docs.orq.ai/docs/ai-gateway/features/openai-compatible-api",
        "self_host_cost_note": "https://docs.orq.ai/docs/enterprise/sovereign-ai",
        "cap_prompt_management": "https://orq.ai/",
        "data_export_available": "https://docs.orq.ai/docs/ai-studio/observability/telemetry-query-api",
        "prompt_transits_vendor": "https://docs.orq.ai/docs/enterprise/deployment-options",
        "cap_conditional_routing": "https://docs.orq.ai/docs/ai-gateway",
        "trains_on_customer_data": "https://docs.orq.ai/docs/administer/data-compliance"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "cap_mcp": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_evals": "2026-08-29",
        "cap_video": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "air_gapped": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "eu_residency": "2026-08-29",
        "founded_year": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:45.482322+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed with no sunset notice. Release 4.14 added an MCP gateway plus caching and tracing in routing rules. Raised a EUR 5M seed in December 2025.",
      "last_release_date": "2026-08-26",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": "Orq.ai prints no dates on its changelog, so this is the last-modified timestamp of the Release 4.14 page rather than a published release date.",
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "EU multi-tenant cloud on GCP, your own AWS or Azure VPC where platform data never leaves it, or an on-premises Helm install with every component inside your environment.",
      "logs_by_default": "full_content",
      "logs_note": "Hosted deployments log by default for the plan's retention period. Variables flagged as PII and masked responses are the exception: they reach the model but are not stored or shown in logs.",
      "retention_days": null,
      "retention_note": "No number is published. Versions, evaluation logs and audit logs are deleted according to your plan's retention policy, which is not stated publicly.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "No data flowing through Orq.ai is ever used to train or fine-tune any model by Orq.ai.",
      "subprocessors_url": null,
      "regions_note": "EU only for the hosted platform — there is no US region, and EU residency is included on every plan. Customer AWS or Azure VPC and on-premises or air-gapped installs are offered.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type 2, monitored via Vanta",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPA on request",
          "status": "available"
        },
        "iso27001": {
          "detail": "proceeding towards formal certification",
          "status": "in_progress"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "both",
      "guardrails_note": "PII masking is available at platform level on all tiers, and VPC or on-premises deployment puts the whole platform — masking included — in your environment.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "hybrid",
      "pricing_model_note": "Models billed at provider list prices with 0% Orq markup. Routing on Orq credits adds a 4.5% top-up fee; BYOK is free up to 1M req/mo then 4%. Seat, module and data overages layer on top. Enterprise annual.",
      "min_commitment": "None stated for Pay-as-you-go. Enterprise agreements run on annual terms.",
      "overage_note": "Spans 100k/mo included then €7 per 100k. Processed data 1 GB/mo then €3/GB. Agent runs 500/mo then €0.01/run. Knowledge Bases include 2.5 GB document processing then €0.10/MB. Retention 30 days, no published price for extending. Usage beyond allowance bills as separate line items — nothing blocks.",
      "cache_support": "none",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Pricing page lists caching in the platform comparison but does not name a mechanism; AI Gateway docs page timed out on fetch. Because model traffic is billed at provider rates with no Orq markup, any provider-side cache discount flows to the customer, but Orq publishes no cached-token pricing of its own.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Documented example of a $100 budget with an alert configured at 90% ($90).",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Not stated as virtual keys, but budgets attach to API keys.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Budgets include a per-minute rate cap; plan-level 100 req/min.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated as webhooks.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Budgets can be scoped per model; a policy-style allowlist is not stated separately.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_key": {
          "note": "Budgets scoped to workspace, project, identity, API key, provider or model. Pre-request hard cap — requests are blocked with HTTP 429 until the period resets.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Workspace / project / identity budgets supported; a 'team' scope not stated but workspace covers most team use.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "Attribution by workspace, project, identity, API key, provider and model. Tag/customer splits not stated.",
      "cost_export": "CSV/JSON of traces, experiments and datasets; Dashboards & Reporting API; SIEM export to Splunk/Datadog — all with no stated additional cost. S3/warehouse export not stated.",
      "product_kind_note": "Vendor calls it the \"AI Gateway\" / Router inside a broader generative-AI collaboration platform, positioned as a \"Sovereign AI Gateway and Platform\"; requests go to a router endpoint such as `POST https://my.orq.ai/v3/router/responses` ([Orq AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction), page dated 2026-08-19).",
      "deployment_modes": [
        "saas",
        "self_host",
        "hybrid_vpc",
        "on_prem"
      ],
      "deployment_note": "Three documented options — Cloud, VPC, On-Premise — all with \"sovereign AI controls\"; on-premise supports model endpoints reachable from inside the environment ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)). Clients point at a custom host via `server_url` / `base_url` / `ORQ_BASE_URL` ([Base URLs](https://docs.orq.ai/reference/base-urls)). Tier gating: not stated (deployment options live under \"enterprise\" docs) ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "OpenAI-shaped paths but under an Orq prefix (`/v3/router/...`), so it is a base-URL swap plus path-prefix change; SDKs allow overriding the base URL ([Base URLs](https://docs.orq.ai/reference/base-urls)). Models are addressed provider-qualified, e.g. `\"model\": \"openai/gpt-5.4\"` ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Streaming caveats: n.a.",
      "model_count_low": 500,
      "model_count_high": 500,
      "model_count_source_note": "\"500+ models\" ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)), page undated.",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "n.a. as a number on pages fetched; private-model sources number 3 (Azure AI Foundry, Google Vertex AI, LiteLLM) ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). Pages undated.",
      "first_party_vs_partner": "Third-party routing: the customer connects providers such as OpenAI via BYOK and Orq routes to them ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); private/fine-tuned/self-hosted models are imported from Azure AI Foundry, Vertex AI or LiteLLM ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). No Orq-owned inference hardware claimed.",
      "custom_endpoint_support": "yes: \"Bring Your Own Model\" supports fine-tuned, self-hosted and privately deployed models — choose Azure AI Foundry, Vertex AI or LiteLLM, enter the endpoint and credentials, then import and enable the deployment ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)).",
      "byok_mode": "byok_only",
      "merchant_of_record": "Not stated on pages fetched; because provider keys are the customer's own (BYOK), the upstream provider bills for tokens, while Orq shows \"latency, token usage, and cost\" in Traces ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Formal invoicing language: n.a.",
      "key_management_note": "Provider keys are connected per provider in the BYOK section of the workspace ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); private-model credentials are entered with the endpoint at import time ([Bring Your Own Model](https://docs.orq.ai/docs/ai-gateway/private-models)). Encryption/KMS details and vendor visibility of request content: n.a. on pages fetched; VPC and on-premise deployments keep the router in the customer's environment ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options)).",
      "api_openai_chat_completions_note": "yes: `/v3/router/chat/completions` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models), [Base URLs](https://docs.orq.ai/reference/base-urls)). Path prefix is `/v3/router`, not `/v1`.",
      "api_anthropic_messages_note": "n.a. (not documented on the router pages fetched: [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models), [Base URLs](https://docs.orq.ai/reference/base-urls)).",
      "api_openai_responses_note": "yes: `/v3/router/responses`, the endpoint used in the quickstart ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction), [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).",
      "api_embeddings_note": "yes: `/v3/router/embeddings` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).",
      "api_images_note": "yes: `/v3/router/images/generations`, `/images/edits`, `/images/variations` ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).",
      "api_audio_note": "yes, both: `/v3/router/audio/transcriptions` (STT) and `/v3/router/audio/speech` (TTS) ([Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).",
      "api_batch_note": "n.a. (not documented on pages fetched: [Supported models](https://docs.orq.ai/docs/ai-gateway/supported-models)).",
      "deployment_modes_note": "Hosted SaaS (\"managed cloud\"), VPC, and on-premise; self-hosted and on-premise deployments \"serve `/v3/router` under their own hostname\" ([Deployment options](https://docs.orq.ai/docs/enterprise/deployment-options), [Base URLs](https://docs.orq.ai/reference/base-urls), [AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)). Air-gapped: not documented.",
      "byok_mode_note": "byok_only on the evidence fetched: the quickstart requires \"Open BYOK, choose OpenAI, select Connect, and add your provider API key\" ([AI Gateway intro](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)); no Orq-sold inference credits appear on the pages fetched.",
      "access_meta": {
        "qualified": [
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "per_request",
      "rel_timeout_note": "Request-body key `timeout.call_timeout` in milliseconds (documented examples `10000` and `15000`) ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).",
      "rel_retry_config": "per_request",
      "rel_retry_note": "`retry.count` accepts 1–5 with `retry.on_codes` defaulting to `[429]`. Backoff is exponential with jitter: 1s / 2s / 4s / 8s / 16s ±25%, for a documented maximum of roughly 31s of retry delay ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).",
      "rel_retry_default_note": "`retry.count` accepts 1–5 with `retry.on_codes` defaulting to `[429]`.",
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "ORDERED: `fallbacks[].model`, tried in order, with a documented recommendation of at most 3 entries ([Orq.ai fallbacks](https://docs.orq.ai/docs/ai-gateway/features/fallbacks)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "`not_documented` as an explicit health check or circuit breaker. What exists is latency-history-driven selection: an in-memory latency store with 5-minute freshness, 10% exploration traffic and a 0.5 ms near-tie threshold ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "Self-hosted/on-prem deployments serve `/v3/router`, but no cross-region failover config is documented ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).",
      "rel_load_balancing_config": "per_request",
      "rel_load_balancing_note": "Weights supported: `load_balancer.type` of `weight_based`, `round_robin` or `latency_based`, with `models[].weight` in the range 0.001–1.0, normalized ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "Runtime is never disclosed on the fetched pages; only the hosted/self-hosted `/v3/router` endpoint is documented ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "`yes`, per the docs' statement that self-hosted and on-prem deployments serve `/v3/router`; no specific artifact (image, chart, binary) is named on the fetched pages ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported; no streaming-specific caveat is stated on the fetched pages ([Orq.ai introduction](https://docs.orq.ai/docs/ai-gateway/get-started/introduction)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Guardrail rules include PII detection, and non-compliant generations are blocked; however \"If the timeout is reached, the guardrail is skipped\" ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules), [Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails)). Separately, input variables flagged as PII are sent to the model but never stored or shown in logs ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))",
      "guard_injection": "sync_block",
      "guard_injection_note": "A jailbreak evaluator is available as a guardrail rule; detection is evaluator/model-based ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules))",
      "guard_content": "sync_block",
      "guard_content_note": "Guardrails are built on evaluators including LLM-as-judge, which can block non-compliant generations ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Custom policies take the form of Python code guardrails, regex, JSON-schema validation, and HTTP calls, all able to block ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` ([Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "fail_open",
      "guard_failure_mode_note": "**Fail open, documented**: \"If the timeout is reached, the guardrail is skipped\" ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules))",
      "log_content": "configurable",
      "log_content_note": "Logs and traces retain input/output for observability, but PII-flagged variables and masked outputs are excluded from storage ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance), [Observability overview](https://docs.orq.ai/docs/observability/overview))",
      "log_opt_out": "yes",
      "log_opt_out_note": "PII flags on input variables and full output masking prevent storage while requests still run; a global body-logging switch is not documented ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry natively: OTLP endpoint `https://api.orq.ai/v2/otel`, with agent traces and conversation threads represented ([Observability overview](https://docs.orq.ai/docs/observability/overview))",
      "export_destinations": [
        "Webhook"
      ],
      "export_destinations_note": "`n.a.` — Orq documents OTel ingest into Orq, not outbound shipping ([Observability overview](https://docs.orq.ai/docs/observability/overview))",
      "feedback_capture": "yes",
      "feedback_capture_note": "Yes — `orq.feedback.create(field=..., value=..., trace_id=...)` submits feedback for an LLM transaction via the API, keyed on the trace ID returned by invoke/get_config ([Feedback SDK reference](https://github.com/orq-ai/orq-python/blob/main/docs/sdks/feedback/README.md))",
      "eval_hooks": "yes",
      "eval_hooks_note": "Yes, on Orq's SaaS — the same evaluator library (LLM-as-judge, Python, HTTP) that powers guardrails is used for scoring, and guardrail rules carry sample rates for partial online evaluation ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules), [Guardrails](https://docs.orq.ai/docs/ai-gateway/configuration/guardrails))",
      "observability_requires_saas": null,
      "log_retention_note": "Configurable retention period, after which data is automatically deleted; the default duration is not stated ([Data compliance](https://docs.orq.ai/docs/administer/data-compliance))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` for benchmark figures. The only numbers are routing-internal thresholds: 5-minute latency freshness window and the 0.5 ms near-tie band ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing)).",
        "benchmark_independence": "No figures published ([Orq.ai load balancing](https://docs.orq.ai/docs/ai-gateway/features/load-balancing))."
      },
      "guard_pii_default": "sync_block",
      "guard_injection_default": "sync_block",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Orq publishes unusually specific retry math (jitter percentage and cumulative worst-case delay), which matters because a 31s retry envelope plus a 15s `call_timeout` can exceed most client-side timeouts ([Orq.ai retries](https://docs.orq.ai/docs/ai-gateway/features/retries)).",
      "guard_notes": "Sample rates mean a guardrail can be configured to evaluate only a fraction of traffic, which converts an apparent blocking control into a partial one; combined with skip-on-timeout, Orq's guardrails are best treated as best-effort enforcement ([Guardrail rules](https://docs.orq.ai/docs/router/guardrail-rules)).",
      "fit_agent_features_note": "Orq.ai supports function, HTTP, MCP, and built-in tools, tool-choice modes, conversation continuation with `previous_response_id`, and multi-agent orchestration using sub-agents and `team_of_agents`. Its MCP integration also supports tool filtering with `allowed_tools`; A2A is not documented. ([Orq.ai run agents](https://docs.orq.ai/docs/ai-studio/ai-engineering/run-agents))",
      "fit_byok_required": "yes",
      "fit_byok_required_note": "Yes in this quickstart: it instructs adding an OpenAI API key through AI Gateway > BYOK before enabling `gpt-4o`, creating the agent, and calling it ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))",
      "fit_client_sdk_langs": [
        "Python",
        "Node.js"
      ],
      "fit_client_sdks_note": "Official SDKs are documented for Python and Node.js: Python package `orq-ai-sdk` with class `Orq`, and Node.js package `@orq-ai/node` with class `Orq`. The `orq` CLI command is also documented. ([Orq.ai client libraries reference](https://docs.orq.ai/reference/client-libraries))",
      "fit_cloud_iam": "upstream_credentials_only",
      "fit_cloud_iam_note": "Partially: for Google Vertex AI, Orq.ai documents pasting a GCP service-account key JSON and granting the Service Account Token Creator and Vertex AI User roles; AWS SigV4/IAM roles and Azure Entra/managed identity are not documented. ([Orq.ai Vertex AI provider integration docs](https://docs.orq.ai/docs/ai-studio/integrations/providers/vertex-ai))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated; the account is described as free ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent)). No credit-card or credits wording on the deployment page ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))",
      "fit_ecosystem_notes": "Orq.ai documents managed cloud, self-hosted, VPC, and on-premise deployment options. Managed Cloud is hosted in EU data centers on Google Cloud Platform; VPC runs in a customer AWS or Azure account and is available through those marketplaces; Enterprise on-premise runs in Kubernetes on customer or self-managed infrastructure and can operate air-gapped after images are cached locally, provided configured model endpoints are reachable. ([Orq.ai deployment options](https://docs.orq.ai/docs/enterprise/deployment-options))",
      "fit_eval_prereqs": "None stated before step 1; step 1 itself is signing up for a free orq.ai account and creating a workspace. However the flow requires adding an OpenAI API key under AI Gateway > BYOK. No credit card, cloud account, or cluster mentioned ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `curl --request POST --url 'https://my.orq.ai/v3/router/responses' --header \"Authorization: Bearer $ORQ_API_KEY\" --data '{\"model\":\"agent/my-assistant\",\"input\":\"What is the capital of France?\"}'` ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "single Orq chart, Kubernetes 1.28+",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Enterprise on-premise deployment is supported as a single Helm chart in a Kubernetes cluster; Kubernetes 1.28 or later is required, and the chart uses Helm hooks for database and data migrations. The optional agent sandbox uses the upstream `kubernetes-sigs/agent-sandbox` controller version `v0.4.6` with its own Helm chart. ([Orq.ai deployment options](https://docs.orq.ai/docs/enterprise/deployment-options))",
      "fit_mcp": "hosted_mcp_server",
      "fit_mcp_note": "Yes. Orq.ai documents hosted MCP servers at `https://my.orq.ai/v2/mcp` for workspace access with 38 specialized tools, and supports MCP-compatible servers in agents through inline `server_url` or saved tools, with Streamable HTTP and SSE. A separate documentation MCP server is hosted at `https://docs.orq.ai/mcp`. ([Orq.ai MCP server integration](https://docs.orq.ai/docs/ai-studio/code-assistants/orq-mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_gateway_platform"
        ]
      },
      "fit_model_gate": "enablement_step",
      "fit_model_gate_note": "No approval gate, but a per-model enablement action inside the product: the quickstart requires enabling `gpt-4o` after adding a BYOK key ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))",
      "fit_onboarding_notes": "The path is agent-centric rather than a raw model call: create an agent, enable `gpt-4o`, configure Web Search and Web Scraper tools, publish, call, then view traces; the guide is billed as \"Beginner-friendly, no AI experience needed.\" Optional SDKs: `pip install orq-ai-sdk`, `npm install @orq-ai/node`. Every agent call is automatically traced ([Quick Start | Build - Orq.ai](https://docs.orq.ai/docs/quickstarts/build-your-first-agent))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. OpenAI-compatible clients can use the exact base URL `https://my.orq.ai/v3/router` with an Orq.ai API key in `Authorization: Bearer $ORQ_API_KEY`; another proxy page documents `https://api.orq.ai/v2/router`. ([Orq.ai base URLs reference](https://docs.orq.ai/reference/base-urls))",
      "fit_prod_prereqs": "VPC: the customer's own AWS or Azure account, deployed through the AWS or Azure marketplaces. On-premise: Kubernetes 1.28+, at least 3 worker nodes with 4 vCPU and 16 GB RAM each, PostgreSQL 14–18, a MongoDB-compatible database, and S3-protocol object storage; \"All components run inside the customer environment.\" ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))",
      "fit_prod_selfhost_shape": "infra_rollout",
      "fit_prod_shape": "cloud_console",
      "fit_python_frameworks": [
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LlamaIndex integration is documented through the packages `llama-index` and `llama-index-llms-openai-like`, using the `OpenAILike` class. LlamaIndex Agents examples use `ReActAgent` and `FunctionTool`; LangChain is not documented. ([Orq.ai LlamaIndex Agents integration](https://docs.orq.ai/docs/ai-studio/integrations/frameworks/llamaindex-agents))",
      "fit_quickstart_steps": 7,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.orq.ai/docs/quickstarts/build-your-first-agent",
      "fit_self_host_docs": "documented_no_command",
      "fit_self_host_install": "No literal install command is published; the deployment-options page states the requirements and that \"container images must be downloaded or cached in a local registry\" but gives no docker/helm command ([Deployment Options - Orq.ai](https://docs.orq.ai/docs/enterprise/deployment-options))",
      "fit_terraform": "official_provider",
      "fit_terraform_note": "An official Terraform provider is documented as `orq-ai/orq`, version `0.1.0`. ([Orq.ai Terraform provider guardrail rule](https://registry.terraform.io/providers/orq-ai/orq/latest/docs/resources/guardrail_rule))",
      "fit_terraform_ref": "orq-ai/orq 0.1.0",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Yes. Orq.ai documents the official TypeScript package `@orq-ai/vercel-provider`, installed with `npm install @orq-ai/vercel-provider ai`, and configured with `createOrqAiProvider({ apiKey: process.env.ORQ_API_KEY })`. ([Orq.ai Vercel AI SDK integration](https://docs.orq.ai/docs/ai-studio/integrations/frameworks/vercel-ai))",
      "fit_vercel_ai_sdk_package": "@orq-ai/vercel-provider"
    },
    {
      "slug": "portkey",
      "name": "Portkey",
      "company": "Portkey, Inc. (Palo Alto Networks)",
      "tagline": "Open-core AI gateway with a hosted control plane for observability, prompt management and governance.",
      "category": "managed_gateway",
      "deployment": "both",
      "license": "Open core",
      "founded_year": 2023,
      "hq_country": "United States",
      "url_home": "https://portkey.ai",
      "url_docs": "https://portkey.ai/docs",
      "url_pricing": "https://portkey.ai/pricing",
      "url_status": "https://status.portkey.ai",
      "url_github": "https://github.com/Portkey-AI/gateway",
      "url_changelog": "https://portkey.ai/docs/changelog",
      "url_models_api": null,
      "github_stars": 12848,
      "model_count": 250,
      "model_count_note": "Docs and the Gateway 2.0 announcement state a unified interface to 250+ models; the gateway's own /v1/models endpoint requires an API key, so no public catalog endpoint exists.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Requests use your own provider credentials via provider integrations; enterprise deployments can also bring their own KMS encryption keys.",
      "seat_fee_usd_month": null,
      "free_tier": "Developer plan free: 10,000 recorded logs/month, 3-day log and 30-day metric retention, universal API, fallbacks, load balancing, retries, 3 prompt templates, simple caching, deterministic guardrails.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Production plan platform fee",
          "amount": "$49/month (100k recorded logs, 30-day log retention)"
        },
        {
          "label": "Production log/request overage",
          "amount": "+$9 per additional 100k requests"
        }
      ],
      "self_host_cost_note": "The gateway core is MIT-licensed and free to self-host (you pay only your own infrastructure and model providers); the Gateway 2.0 release moved circuit breakers, semantic cache, budget limits, model catalog, metadata governance and config management into open source. Self-hosting drops the hosted dashboards/control plane, and an On-Prem Enterprise Gateway (gRPC, SSO, SCIM, AWS KMS, RBAC, JWT, audit logs, multi-workspace) is sold separately.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Portkey unified API",
        "Admin/Control-plane API"
      ],
      "sdk_note": "Three-line integration via Portkey SDKs or by pointing the OpenAI SDK at https://api.portkey.ai/v1.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": true,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": null,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "Portkey docs state benchmarks estimate a total added latency of 20-40 ms versus direct API calls, along with 99.99% uptime and 25M+ requests served daily; no single figure is published. A third-party comparison describes Portkey's SaaS hop as similar to OpenRouter's without a number.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": true,
      "eu_residency": null,
      "eu_residency_note": null,
      "zdr": true,
      "zdr_note": "Zero retention, PII scrubbing and a metrics-only mode are all offered, but enabled on request rather than self-service.",
      "air_gapped": null,
      "best_for": "Teams that want governance, guardrails and prompt management in one control plane, with the option to self-host the routing engine.",
      "strengths": [
        "MIT-licensed gateway core covering routing, fallbacks, conditional routing, semantic cache, budgets and guardrails",
        "Full governance stack: RBAC, SSO/SCIM, audit logs, budget and rate limits, PII-redaction guardrails",
        "SOC 2, HIPAA, GDPR and ISO 27001 compliance claims plus custom BAAs",
        "MCP registry with OAuth 2.1 (PKCE) in the open-source gateway",
        "Cheap entry point: free Developer tier, $49/month Production"
      ],
      "weaknesses": [
        "Pricing is metered on recorded logs, not tokens, so high-request workloads hit $9 per extra 100k requests",
        "No public model-catalog endpoint - the /v1/models call requires an API key",
        "Self-hosting loses the hosted dashboards, prompt management and guardrails UI",
        "Third-party reviews note slower development cadence and a smaller self-hosting community than LiteLLM"
      ],
      "avoid_if": "You mainly need a thin, zero-config marketplace router and do not want to run or pay for a control plane.",
      "reviews": [
        {
          "url": "https://www.infoworld.com/article/3835182/portkey-an-open-source-ai-gateway-for-easy-llm-orchestration.html",
          "why": "InfoWorld's hands-on review of the open-source gateway, exercising Portkey's unified API across OpenAI, Anthropic, Azure, Groq, Fireworks and Ollama plus its hosted tier.",
          "date": "2025-03-06",
          "type": "written",
          "title": "Portkey: An open-source AI gateway for easy LLM orchestration",
          "publisher": "InfoWorld"
        },
        {
          "url": "https://www.youtube.com/watch?v=-rSbvS0qLqY",
          "why": "Latent Space's hosts interrogate Portkey's CEO on gateway design tradeoffs - routing, guardrails, logging overhead and observability - on an established AI-engineering podcast.",
          "date": "2025-02-05",
          "type": "video",
          "title": "Why every AI Engineer needs an AI Gateway (ft Portkey.ai CEO)",
          "publisher": "YouTube - Latent Space"
        },
        {
          "url": "https://www.youtube.com/watch?v=TpUwSmGfMrQ",
          "why": "Independent hands-on build using Portkey's gateway: virtual keys, configs, a Bun/Node project querying OpenAI and Perplexity, then inspecting logs and cache hits.",
          "date": "2024-01-18",
          "type": "video",
          "title": "AI Gateway: Enhancing LLM Integrations for Application Development",
          "publisher": "YouTube - Developers Digest"
        },
        {
          "url": "https://www.reddit.com/r/devops/comments/1ue3pzq/compared_openrouter_portkey_and_orqs_gateway_for/",
          "why": "Same operator thread covers Portkey's routing behaviour and cost-attribution dashboards after running it in production alongside two rivals.",
          "date": "2026-06-24",
          "type": "community",
          "title": "Compared OpenRouter, Portkey, and Orq's gateway for routing across providers",
          "publisher": "Reddit - r/devops"
        }
      ],
      "sources": {
        "zdr": "https://portkey.ai/docs",
        "gdpr": "https://portkey.ai/docs",
        "soc2": "https://portkey.ai/docs",
        "cap_mcp": "https://portkey.ai/blog/gateway-2-0",
        "license": "https://github.com/Portkey-AI/gateway",
        "sdk_note": "https://portkey.ai/docs",
        "url_docs": "https://portkey.ai/docs",
        "zdr_note": "https://portkey.ai/docs",
        "cap_evals": "https://portkey.ai/pricing",
        "free_tier": "https://portkey.ai/pricing",
        "hipaa_baa": "https://portkey.ai/docs",
        "deployment": "https://portkey.ai/docs",
        "extra_fees": "https://portkey.ai/pricing",
        "hq_country": "https://www.linkedin.com/company/portkey-ai",
        "url_github": "https://github.com/Portkey-AI/gateway",
        "url_status": "https://status.portkey.ai",
        "zdr_status": "https://portkey.ai/docs/enterprise/security",
        "acquired_by": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents",
        "cap_budgets": "https://docs.portkey.ai/docs/product/enterprise-offering",
        "model_count": "https://portkey.ai/docs",
        "url_pricing": "https://portkey.ai/pricing",
        "api_surfaces": "https://portkey.ai/docs",
        "cap_failover": "https://portkey.ai/pricing",
        "github_stars": "https://api.github.com/repos/Portkey-AI/gateway",
        "product_fate": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents",
        "regions_note": "https://portkey.ai/docs/enterprise/security",
        "acquired_date": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents",
        "byok_fee_note": "https://portkey.ai/blog/gateway-2-0",
        "url_changelog": "https://portkey.ai/docs/changelog",
        "vendor_status": "https://www.paloaltonetworks.com/company/press/2026/palo-alto-networks-completes-acquisition-of-portkey-to-secure-ai-agents",
        "byok_supported": "https://portkey.ai/docs",
        "cap_guardrails": "https://portkey.ai/pricing",
        "certifications": "https://docs.portkey.ai/docs/product/enterprise-offering/security-portkey",
        "config_as_code": "https://docs.portkey.ai/docs/api-reference/config-object",
        "retention_days": "https://portkey.ai/docs/enterprise/security",
        "sla_uptime_pct": "https://portkey.ai/docs/product/product-feature-comparison",
        "cap_rate_limits": "https://docs.portkey.ai/docs/product/enterprise-offering",
        "logs_by_default": "https://docs.portkey.ai/docs/introduction/what-is-portkey",
        "portkey_pricing": "https://portkey.ai/pricing",
        "cap_cache_simple": "https://portkey.ai/pricing",
        "model_count_note": "https://portkey.ai/blog/gateway-2-0",
        "acquisition_terms": "https://economictimes.com/tech/startups/palo-alto-networks-to-acquire-elevation-capital-backed-ai-app-infra-startup-portkey/articleshow/130668555.cms",
        "cap_observability": "https://portkey.ai/pricing",
        "cap_pii_redaction": "https://docs.portkey.ai/docs/product/enterprise-offering",
        "last_release_date": "https://portkey.ai/docs/changelog/enterprise",
        "openai_compatible": "https://portkey.ai/docs",
        "cap_cache_semantic": "https://portkey.ai/pricing",
        "cap_load_balancing": "https://portkey.ai/pricing",
        "guardrails_location": "https://portkey.ai/docs/enterprise/security",
        "latency_source_note": "https://portkey.ai/docs",
        "requires_vendor_sdk": "https://docs.portkey.ai/docs/integrations/llms/openai",
        "self_host_cost_note": "https://portkey.ai/blog/gateway-2-0",
        "cap_prompt_management": "https://portkey.ai/pricing",
        "data_export_available": "https://docs.portkey.ai/docs/product/observability/logs-export",
        "portkey_budget_limits": "https://docs.portkey.ai/docs/product/ai-gateway/virtual-keys/budget-limits",
        "prompt_transits_vendor": "https://portkey.ai/docs/enterprise/security",
        "cap_conditional_routing": "https://portkey.ai/blog/gateway-2-0",
        "portkey_feature_comparison": "https://portkey.ai/docs/product/product-feature-comparison"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "cap_mcp": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_evals": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "url_github": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:45.482322+00:00",
      "vendor_status": "acquired",
      "acquired_by": "Palo Alto Networks, Inc.",
      "acquired_date": "2026-05-29",
      "acquisition_terms": "Terms undisclosed; The Economic Times reported $120-140M citing sources",
      "product_fate": "Palo Alto Networks closed its acquisition on 29 May 2026 and is establishing Portkey as the core AI Gateway for its Prisma AIRS platform. Palo Alto Networks said it will continue to support existing and new Portkey customers, and the commercial Enterprise Gateway is still shipping. The open-source gateway repository, however, has had no tagged release since 12 January 2026, so if you were planning to self-host the free version, check that cadence before committing.",
      "last_release_date": "2026-08-21",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://portkey.ai/docs/product/product-feature-comparison",
      "claim_basis": {},
      "last_release_note": "Enterprise Gateway v2.19.0. The open-source repository is far staler: last tag v1.15.2 on 12 January 2026.",
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "Requests hit the nearest regional edge gateway on the hosted product, or you run the gateway in your own VPC or air-gapped, where prompts, responses and logs never leave. One caveat holds in every mode: operational metrics are always sent to Portkey's control plane.",
      "logs_by_default": "full_content",
      "logs_note": "Hosted logging is on by default. Suppressing body storage is not a setting you can flip — it is enabled on request.",
      "retention_days": 90,
      "retention_note": "Ninety days for logs and 365 for metrics by default. Self-hosted puts logs in your own S3-compatible store under your lifecycle policy.",
      "zdr_status": "yes",
      "trains_on_customer_data": "not_published",
      "training_note": "Nothing in the privacy policy, terms, enterprise security page or security overview addresses training.",
      "subprocessors_url": null,
      "regions_note": "Region pinning with data stored in your selected region and requests served in-region across a network of about 310 data centres, plus customer-VPC and fully disconnected air-gapped deployment.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II said to be available via an unnamed trust portal",
          "status": "claimed"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "described as GDPR-compliant",
          "status": "claimed"
        },
        "iso27001": {
          "detail": "asserted in docs; no report or portal named",
          "status": "claimed"
        },
        "hipaa_baa": {
          "detail": "described as HIPAA-compliant; no BAA offer found",
          "status": "claimed"
        }
      },
      "guardrails_location": "both",
      "guardrails_note": "PII scrubbing is a gateway feature, and the gateway runs either at Portkey's edge or inside your VPC or air-gapped environment.",
      "acquired_date_basis": "closed",
      "security_incidents": [],
      "pricing_model": "hybrid",
      "pricing_model_note": "Free OSS self-host + free Dev tier + flat monthly platform fee ($49/mo Pro per docs) + request overage + Enterprise quote. Public pricing page shows tiers without dollar amounts and describes the free allowance as '10k recorded logs' per month; docs comparison lists 'Pro $49/Month' with '10k requests/month'. These two framings contradict.",
      "min_commitment": "Not stated.",
      "overage_note": "Dev 10K requests/mo with 'No Overage Allowed' (docs) — but pricing page says exceeding the log limit only stops recording, not requests. Pro 100K then $9/mo per additional 100K up to 3M requests. Enterprise custom. Retention-overage price and export cost not stated.",
      "cache_support": "hybrid",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact/'simple' caching on all tiers; semantic caching from the paid tier (Production/Pro). No cached-token pricing published. Semantic cache savings are paid-tier-only.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "'Alerts' in Production observability; spend-specific alerts not stated.",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Virtual keys are the object budgets attach to.",
          "tier": null,
          "supported": true
        },
        "rate_limiting": {
          "note": "'Granular Budget & Rate Limits' is an Enterprise bullet.",
          "tier": "enterprise",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Docs contradict on availability: 'currently only available to Enterprise' vs 'Available on Enterprise plan and select Pro customers'. Pricing page lists 'Granular Budget & Rate Limits' as Enterprise-only. Minimum $1 cost limit or 100 tokens. Pre-request; key auto-expires on exhaustion. No automatic reset by default.",
          "tier": "enterprise",
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Per-workspace limits supported. Team-level not stated.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "Custom metadata and filters on all tiers. Per key/user/team/tag/customer cost splits not explicitly stated.",
      "cost_export": "'Data Export to Data Lakes' on Enterprise only. CSV/API/webhook/S3 not stated; price not stated.",
      "product_kind_note": "Portkey calls itself a \"unified interface for interacting with over 250 AI models\" and an \"AI Gateway\" that sits in front of provider APIs, adding routing, fallbacks, guardrails and observability ([Portkey: what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). It is a routing/observability proxy in front of many providers, available as an open-source gateway and as managed SaaS ([portkey-ai/gateway on GitHub](https://github.com/portkey-ai/gateway)).",
      "deployment_modes": [
        "saas",
        "self_host",
        "hybrid_vpc"
      ],
      "deployment_note": "The OSS gateway runs locally with `npx @portkey-ai/gateway` exposing `http://localhost:8787/v1` ([portkey-ai/gateway](https://github.com/portkey-ai/gateway)). Enterprise hybrid uses the `portkey/gateway-enterprise:latest` Docker image, Helm v3.0+ on Kubernetes v1.24+, deployed into the customer's own VPC: \"Deploy Portkey's data plane in your VPC… All prompts and responses stay in your VPC. Only metrics leave (no sensitive data)\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01). Hybrid/VPC is presented as the enterprise offering ([Portkey hybrid on GCP](https://portkey.ai/docs/self-hosting/hybrid-deployments/gcp)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap: point an existing OpenAI-compatible client at `https://api.portkey.ai/v1` and pass `x-portkey-api-key`; a first-party `portkey_ai` SDK also exists but is not required ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Requests select the upstream through a virtual key / provider slug, and sensitive headers can be forwarded \"without processing\" ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Streaming caveats: n.a.",
      "model_count_low": 250,
      "model_count_high": 2300,
      "model_count_source_note": "Vendor numbers disagree: \"over 250 AI models\" ([Portkey docs](https://portkey.ai/docs/introduction/what-is-portkey), undated), \"1,600+ LLMs\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01), \"2,300+ models\" ([portkey-ai/gateway](https://github.com/portkey-ai/gateway), undated).",
      "provider_count_low": 40,
      "provider_count_high": 48,
      "provider_count_source_note": "Vendor numbers disagree: \"45+ providers\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01) and \"2,300+ models across 40+ providers\" ([portkey-ai/gateway](https://github.com/portkey-ai/gateway), undated); the integrations index lists 48 provider cards ([Portkey LLM integrations](https://portkey.ai/docs/integrations/llms), undated).",
      "first_party_vs_partner": "All third-party routed: Portkey owns no models and proxies to provider APIs plus privately hosted models ([Portkey LLM integrations](https://portkey.ai/docs/integrations/llms), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).",
      "custom_endpoint_support": "Yes: add a \"Local/Privately hosted provider\" in Model Catalog → Add Provider, pick a compatible provider API specification and enter a \"Custom Host\", or pass `customHost` per request ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Named support for vLLM/Ollama/SageMaker specifically: n.a.",
      "byok_mode": "byok_only",
      "merchant_of_record": "n.a. Portkey's pricing page does not state who invoices model usage, and lists no token markup or credit top-up fee ([Portkey pricing](https://portkey.ai/pricing)); with BYOK the provider key is the customer's own ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys)).",
      "key_management_note": "Upstream credentials live in Portkey's \"secure vault\" as virtual keys, \"stored securely, never exposed in code\", with organization-level management and sharing across workspaces ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)). Data is AES-256 encrypted in transit and at rest, storage of request/response bodies can be disabled, and Portkey holds ISO 27001 and SOC 2 ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). In hybrid/VPC mode \"all prompts and responses stay in your VPC. Only metrics leave (no sensitive data)\" ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), 2024-10-01).",
      "api_openai_chat_completions_note": "Yes: `POST https://api.portkey.ai/v1/chat/completions`, usable by pointing the OpenAI SDK's base URL at Portkey ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)).",
      "api_anthropic_messages_note": "Yes: `/messages` is listed among the endpoints Portkey exposes for provider integrations ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).",
      "api_openai_responses_note": "n.a. (not stated on the Portkey pages fetched in this session).",
      "api_embeddings_note": "Yes: `/embeddings` documented alongside `/chat/completions` and `/completions` ([Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)).",
      "api_images_note": "Yes: image generation is listed among the modalities the gateway supports ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Endpoint path: n.a.",
      "api_audio_note": "Yes, both directions: text-to-speech and speech-to-text (plus realtime) are listed as supported modalities ([Portkey AI Gateway docs](https://portkey.ai/docs/product/ai-gateway)). Endpoint paths: n.a.",
      "api_batch_note": "n.a. (no batch endpoint stated on the pages fetched).",
      "deployment_modes_note": "hosted SaaS ([Portkey docs](https://portkey.ai/docs/introduction/what-is-portkey)); self-host / open-source gateway via npx or Docker ([portkey-ai/gateway](https://github.com/portkey-ai/gateway)); hybrid / customer-VPC data plane with Docker + Helm on AWS, Azure, GCP and OpenShift ([Portkey hybrid deployment](https://portkey.ai/docs/enterprise/hybrid2), page dated 2024-10-01; [Portkey hybrid on GCP](https://portkey.ai/docs/self-hosting/hybrid-deployments/gcp)). On-prem and air-gapped: n.a. (not documented on the pages fetched).",
      "byok_mode_note": "byok_only on the evidence fetched: Portkey requests run on the customer's own provider credentials held as virtual keys ([Portkey virtual keys](https://portkey.ai/docs/product/ai-gateway/virtual-keys), [Portkey BYO-LLM docs](https://portkey.ai/docs/integrations/llms/byollm)); the pricing page states no token markup or credit purchase model ([Portkey pricing](https://portkey.ai/pricing)).",
      "access_meta": {
        "qualified": [
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "per_request",
      "rel_timeout_note": "`per_request` and `config_file`. Header `x-portkey-request-timeout`, SDK parameter `requestTimeout`, and config key `request_timeout` in milliseconds, settable at strategy level or per target with nested inheritance ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts)).",
      "rel_retry_config": "config_file",
      "rel_retry_note": "`retry.attempts` up to `5`, default retried status codes `[429, 500, 502, 503, 504]` overridable via `on_status_codes`; exponential backoff 1s / 2s / 4s / 8s / 16s ([Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).",
      "rel_retry_default_note": "`retry.attempts` up to `5`, default retried status codes `[429, 500, 502, 503, 504]` overridable via `on_status_codes`; exponential backoff 1s / 2s / 4s / 8s / 16s ([Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).",
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "ORDERED: `strategy.mode: fallback` with an ordered `targets` array; default trigger is any non-2xx response ([Portkey fallbacks](https://docs.portkey.ai/docs/product/ai-gateway/fallbacks)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "No upstream health check, circuit breaker or provider-ejection key appears on the fetched configs/fallback/load-balancing pages ([Portkey configs](https://docs.portkey.ai/docs/product/ai-gateway/configs)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "You can list targets that happen to be in different regions, but no cross-region failover feature is documented ([Portkey fallbacks](https://docs.portkey.ai/docs/product/ai-gateway/fallbacks)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Weights supported: `strategy.mode: loadbalance` with per-target `weight` (default `1`, `0` disables a target, weights are normalized) ([Portkey load balancing](https://docs.portkey.ai/docs/product/ai-gateway/load-balancing)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "`interpreted_proxy` (JS runtime). Repo is TypeScript 96.0% and ships as an npm-launched service ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "`npx @portkey-ai/gateway` (npm package `@portkey-ai/gateway`), with documented Docker, Cloudflare, AWS, Azure, GCP and Kubernetes deployments ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).",
      "latency_claims": [
        {
          "url": "https://github.com/Portkey-AI/gateway",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "<1",
          "metric": "not stated",
          "publisher": "vendor",
          "conditions": "README headline. No percentile, no RPS, no payload, no hardware, no cache state."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://github.com/Portkey-AI/gateway",
          "unit": "tokens/day",
          "scope": "marketing",
          "value": "over 10B",
          "metric": "fleet total",
          "publisher": "vendor",
          "conditions": "Aggregate traffic across all customers, not a measured per-instance capacity."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported, with an important documented caveat: the request timeout does not trigger if at least one chunk has already arrived, so a stalled mid-stream response is not cut off by `request_timeout` ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "\"Detect PII\" is one of 20+ deterministic checks; guardrail actions default to `async: TRUE` (log only, no added latency), and only when a check is set synchronous **and** `Deny: TRUE` does Portkey return HTTP 446 and block. With `Deny: FALSE` the request passes through with a 246 status and the verdict recorded ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Injection/jailbreak detection comes from partner guardrail vendors rather than a built-in classifier, and inherits the same async/sync + Deny semantics ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))",
      "guard_content": "sync_block",
      "guard_content_note": "\"Moderate Content\" is a built-in check subject to the same async default ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policies take the form of Regex Match, Contains / word lists, JSON Schema and JSON Keys validation, Model Whitelist, and Webhook (custom classifier hook) ([Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Model allowlists and denylists are configured per Integration and cascade to AI Providers, with workspace provisioning deciding which teams reach which provider; a Model Whitelist guardrail check exists as well ([Model Catalog](https://portkey.ai/docs/product/model-catalog), [Guardrail checks](https://portkey.ai/docs/product/guardrails/list-of-guardrail-checks))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "Acuvity",
        "Aporia",
        "AWS Bedrock Guardrails",
        "Azure Content Safety",
        "Javelin",
        "Lasso",
        "Mistral",
        "Pangea",
        "Palo Alto Prisma AIRS",
        "Patronus AI",
        "Pillar Security",
        "Qualifire"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "The docs describe deny/allow verdict semantics but not behaviour when the guardrail service itself errors or times out ([Guardrails](https://docs.portkey.ai/docs/product/guardrails))",
      "log_content": "configurable",
      "log_content_note": "Full Logging stores \"Complete request payloads\", \"Full response content\", and all metrics; Metrics Only (Privacy Mode) stores usage, metadata, and errors \"without sensitive content\" ([Request logging settings](https://portkey.ai/docs/product/administration/configuring-request-logging)). Clicking a log entry reveals \"the entire raw data with the request and response objects\" ([Logs](https://docs.portkey.ai/docs/product/observability/logs))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Org/workspace-level Metrics Only mode ([Request logging settings](https://portkey.ai/docs/product/administration/configuring-request-logging)), or per request the `DO NOT TRACK` control: set `debug: false` on the client or send header `x-portkey-debug: false`, keeping tokens, cost, and latency ([Logs](https://docs.portkey.ai/docs/product/observability/logs))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry export is supported in two modes: analytics-only spans without prompts, and complete logs including full prompt/completion (marked experimental); trace IDs group multi-step requests ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel), [Feedback](https://docs.portkey.ai/docs/product/observability/feedback))",
      "export_destinations": [
        "OpenTelemetry",
        "Datadog",
        "Grafana",
        "LangSmith"
      ],
      "export_destinations_note": "Any OTLP-compatible backend, with Datadog, Grafana, and LangSmith named explicitly ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel))",
      "feedback_capture": "yes",
      "feedback_capture_note": "Yes — `POST https://api.portkey.ai/v1/feedback` with `trace_id`, `value` (−10…10), optional `weight` and `metadata`; the trace ID is returned in the `x-portkey-trace-id` response header ([Feedback](https://docs.portkey.ai/docs/product/observability/feedback))",
      "eval_hooks": "partial",
      "eval_hooks_note": "Feedback data feeds Analytics and a Prompt Eval tab inside Portkey; the same guardrail checks can run asynchronously over live traffic as scoring. Both require Portkey's platform ([Feedback](https://docs.portkey.ai/docs/product/observability/feedback), [Guardrails](https://docs.portkey.ai/docs/product/guardrails))",
      "observability_requires_saas": "Enterprise deployments can run the gateway in customer infrastructure and export telemetry via OTel, but the log/analytics UI is Portkey's control plane ([Portkey OTel](https://portkey.ai/docs/product/enterprise-offering/otel/otel))",
      "log_retention_note": "Developer 3 days (10k logs/month), Production 30 days (100k logs/month, $9 per additional 100k), Enterprise unlimited logs with retention unspecified ([Logs](https://docs.portkey.ai/docs/product/observability/logs))",
      "production_meta": {
        "sources": {},
        "qualified": [
          "guard_pii",
          "guard_injection",
          "guard_content",
          "guard_custom"
        ],
        "latency_prose": "README claims **`<1ms` latency** and a `122kb` footprint. Metric type (p50/p99/mean) is not stated, what was measured is not stated beyond \"latency\", and no RPS, payload, hardware or cache conditions are given — conditions not stated ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).",
        "multi_surface": {
          "rel_timeout": [
            "per_request",
            "config_file"
          ]
        },
        "throughput_prose": "README claims \"over 10B tokens processed everyday\" — this is a fleet-usage figure, not a measured per-instance throughput; conditions not stated ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway)).",
        "benchmark_independence": "Both figures are vendor self-published with no methodology; neither is a third-party benchmark and neither is a claim about a competitor ([Portkey-AI/gateway](https://github.com/Portkey-AI/gateway))."
      },
      "guard_pii_default": "sync_observe",
      "guard_injection_default": "sync_observe",
      "guard_content_default": "sync_observe",
      "guard_custom_default": "sync_observe",
      "rel_notes": "Nested timeout inheritance (strategy level vs target level) is the most granular timeout model in this set; combined with `retry.attempts: 5` and exponential backoff, worst-case wall-clock time should be computed before setting client timeouts ([Portkey request timeouts](https://docs.portkey.ai/docs/product/ai-gateway/request-timeouts), [Portkey automatic retries](https://docs.portkey.ai/docs/product/ai-gateway/automatic-retries)).",
      "guard_notes": "**The single most important default in this report**: Portkey guardrails ship asynchronous, meaning a freshly configured \"PII guardrail\" annotates the log and lets the request through. Enforcement requires both switching the check to synchronous and setting Deny ([Guardrails](https://docs.portkey.ai/docs/product/guardrails)).",
      "fit_agent_features_note": "Portkey supports function calling and tool use. Its Responses API documents built-in tool use and reasoning, including web search, file search, Computer Use Assistant, and Remote MCP; its Assistants API documents persistent threads. ([Portkey universal API](https://docs.portkey.ai/docs/product/ai-gateway/universal-api))",
      "fit_byok_required": "yes",
      "fit_byok_required_note": "Not explicitly stated, but the documented snippets reference a provider: `provider=\"@YOUR_PROVIDER\"` and examples using `YOUR_OPENAI_API_KEY`; the self-hosted validation step creates \"a virtual key\" before the first call ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))",
      "fit_client_sdk_langs": [
        "Python",
        "JavaScript"
      ],
      "fit_client_sdks_note": "Official examples use the OpenAI SDK and Portkey SDKs: Python packages `openai` and `portkey-ai`/`portkey_ai`, JavaScript packages `openai` and `portkey-ai`, and the Python `Portkey` class. Portkey also documents the Anthropic package `anthropic` for the Messages API. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "vendor_runtime_only",
      "fit_cloudflare_workers_note": "The Portkey Gateway runs on Cloudflare Workers and uses Cloudflare’s global edge network and data centers, according to a Portkey blog post. ([Why Portkey chose TypeScript for its AI Gateway](https://portkey.ai/blog/why-we-chose-ts-over-python-to-build-potkeys-ai-gateway))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated; \"Portkey's Gateway is open source and free to use. On managed version, Portkey offers a free plan with 10k requests per month.\" ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey))",
      "fit_ecosystem_notes": "Portkey documents both the hosted gateway URL `https://api.portkey.ai/v1` and local/self-hosted use at `http://localhost:8787/v1`; `PORTKEY_API_KEY` can be skipped when self-hosting. It states that the gateway supports 30+ providers and all models within those providers. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))",
      "fit_eval_prereqs": "A Portkey account and an API key from the Settings page, with permissions ticked on the API-key modal. No credit card, cloud account, cluster, or provider key stated ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))",
      "fit_eval_shape": "sdk_install",
      "fit_eval_snippet": "partial",
      "fit_eval_snippet_note": "Yes, on the overview page: `Portkey(api_key=\"YOUR_PORTKEY_API_KEY\", provider=\"@YOUR_PROVIDER\")` then `portkey.chat.completions.create(model=\"gpt-3.5-turbo\", messages=[...])` and `print(chat_complete.choices[0].message.content)` ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey)). The make-your-first-request page names the integration options but rendered no code block in the fetched content ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "charts/portkey-gateway and charts/portkey-app in Portkey-AI/helm",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Portkey provides official Kubernetes Helm charts in the `Portkey-AI/helm` repository: `charts/portkey-gateway` for Portkey Hybrid (Gateway) and `charts/portkey-app` for Portkey Full (Full App). ([Portkey official Helm charts](https://github.com/Portkey-AI/helm))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "Portkey documents an MCP Gateway that centralizes authentication and team/user access control for MCP servers and tools, logs tool calls, and supports identity forwarding. It works with Claude Desktop, Cursor, VS Code, and other MCP-compatible clients. ([Portkey Gateway GitHub repository](https://github.com/portkey-ai/gateway))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_gateway_platform",
          "fit_model_gate"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented). The only permissions wording is on the API-key modal: \"Based on your access level, you might see the relevant permissions on the API key modal\" ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request)); no approval gate in the private-deployment guide either ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))",
      "fit_onboarding_notes": "Self-hosted validation ends with a health check, creating a virtual key, calling `https://gateway.internal.mycompany.com/v1/chat/completions` with `x-portkey-api-key`, and viewing logs/analytics; port-forward for testing via `kubectl -n portkey port-forward svc/portkey-gateway 8787:8787`. Portkey adds \"20-40ms\" latency per its own docs. Native integrations exist for OpenAI, Azure OpenAI, Anthropic, Langchain, LlamaIndex and Ollama ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2), [Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey), [Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Portkey documents direct OpenAI SDK compatibility using the exact hosted base URL `https://api.portkey.ai/v1`; local deployments use `http://localhost:8787/v1`. ([Portkey AI Gateway getting started](https://docs.portkey.ai/docs/guides/getting-started/getting-started-with-ai-gateway))",
      "fit_prod_prereqs": "Kubernetes v1.24+, Helm v3.0+, outbound internet access returning `HTTP/2 200` from `https://control.portkey.ai`, a storage bucket (`s3`, `gcs`, `mongo`, or `wasabi`), an auto-deployed Redis cache, Portkey control-plane access with 1Password credentials, Docker registry username/password plus a gateway JWT, an ingress with class `nginx`, and a Portkey dashboard account at app.portkey.ai ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))",
      "fit_prod_selfhost_shape": "infra_rollout",
      "fit_prod_shape": "sdk_install",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Portkey documents integrations with LangChain and LlamaIndex. LangChain examples use `langchain`, `langchain-openai`, `ChatOpenAI`, `createHeaders`, and `LangchainCallbackHandler`; LlamaIndex examples use `llama_index.llms.openai.OpenAI`, `ChatMessage`, and `portkey-ai`. ([Portkey LangChain agents integration](https://portkey.ai/docs/integrations/agents/langchain-agents))",
      "fit_quickstart_steps": 3,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://portkey.ai/docs/introduction/make-your-first-request",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`docker pull portkey/gateway-enterprise:latest`; `helm repo add portkey https://portkey-ai.github.io/helm`; `helm repo update`; `helm upgrade --install portkey-gateway portkey/gateway -n portkey -f values.yaml` (plus `kubectl create namespace portkey` and an image-pull secret) ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))",
      "fit_terraform": "official_provider",
      "fit_terraform_note": "Portkey provides an official Terraform provider named `Portkey-AI/portkey` on the Terraform Registry. ([Portkey Terraform provider](https://registry.terraform.io/providers/Portkey-AI/portkey/latest/docs))",
      "fit_terraform_ref": "Portkey-AI/portkey",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": "Vendor claims, verbatim: \"It takes 2 mins to integrate\" ([Docs - what is Portkey](https://portkey.ai/docs/introduction/what-is-portkey)); page description \"Integrate Portkey and analyze your first LLM call in 2 minutes!\" ([Make Your First Request - Portkey Docs](https://portkey.ai/docs/introduction/make-your-first-request)); and for private deployment \"Get your AI gateway running in 4 simple steps. Most teams finish in under 2 hours,\" with a breakdown of Total 2 hours / Prerequisites 5 min / Credentials 10 min / Deployment 30 min / Validation 15 min ([Deploy Portkey in Your Infrastructure](https://portkey.ai/docs/enterprise/hybrid2))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Portkey provides a Vercel AI SDK integration. The documentation names the provider package `@portkey-ai/vercel-provider` and uses `createPortkey`. ([Portkey Vercel AI SDK integration](https://docs.portkey.ai/docs/integrations/libraries/vercel))",
      "fit_vercel_ai_sdk_package": "@portkey-ai/vercel-provider"
    },
    {
      "slug": "vercel-ai-gateway",
      "name": "Vercel AI Gateway",
      "company": "Vercel",
      "tagline": "Vercel-operated gateway that routes AI SDK and OpenAI-format requests to many providers with zero token markup.",
      "category": "managed_gateway",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2015,
      "hq_country": "United States",
      "url_home": "https://vercel.com/ai-gateway",
      "url_docs": "https://vercel.com/docs/ai-gateway",
      "url_pricing": "https://vercel.com/docs/ai-gateway/pricing",
      "url_status": "https://www.vercel-status.com",
      "url_github": null,
      "url_changelog": "https://vercel.com/changelog",
      "url_models_api": "https://ai-gateway.vercel.sh/v1/models",
      "github_stars": null,
      "model_count": 360,
      "model_count_note": "Counted from the public, unauthenticated https://ai-gateway.vercel.sh/v1/models endpoint on 2026-08-29 (360 entries across 35 model owners); Vercel docs describe the catalog as 200+ models, and an independent August 2026 count found 313 entries split across language, embedding, reranking, image, video, realtime, speech and transcription models.",
      "provider_count": null,
      "token_markup_pct": 0,
      "credit_fee_pct": 0,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "No markup or fee on BYOK, but BYOK requires purchased AI Gateway Credits (paid tier); failed BYOK requests fall back to Vercel system credentials and are billed against credits, and BYOK spend is not covered by budgets.",
      "seat_fee_usd_month": 20,
      "free_tier": "Every team gets free AI Gateway credits on a subset of models with lower per-model rate limits; the amount is not published. Pro plans include $20 of credit across resources.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Team-wide zero data retention",
          "amount": "$0.10 per 1,000 requests (Pro and Enterprise)"
        },
        {
          "label": "Team-wide provider allowlist",
          "amount": "$0.10 per 1,000 successful requests (Pro and Enterprise)"
        },
        {
          "label": "Custom Reporting writes",
          "amount": "$0.075 per 1,000 tag / user ID / quota entity writes"
        },
        {
          "label": "Custom Reporting queries",
          "amount": "$5 per 1,000 queries to the reporting endpoint"
        },
        {
          "label": "Vercel Pro developer seat",
          "amount": "$20 per month"
        }
      ],
      "self_host_cost_note": null,
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": null,
      "portability_note": null,
      "api_surfaces": [
        "Vercel AI SDK",
        "OpenAI Chat Completions",
        "OpenAI Responses"
      ],
      "sdk_note": "Native to the Vercel AI SDK via providerOptions.gateway (order, only, sort, caching, byok); OIDC-based auth for apps deployed on Vercel.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": false,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": false,
      "cap_pii_redaction": null,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": true,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": true,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No published gateway-overhead figure. An independent hands-on comparison reported 'consistent time to first token' against direct providers without publishing millisecond numbers; the LLM Gateway TTFT benchmark did not test Vercel AI Gateway.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": true,
      "eu_residency": null,
      "eu_residency_note": "Vercel states no data is stored permanently inside EU regions on the platform (EU caching is ephemeral); no AI Gateway-specific EU residency guarantee was found.",
      "zdr": true,
      "zdr_note": "Available on Pro and Enterprise only. Per-request ZDR costs nothing; team-wide ZDR is billed at $0.10 per 1,000 requests.",
      "air_gapped": false,
      "best_for": "Teams already shipping on Vercel with the AI SDK who want zero token markup and no credit-purchase fee.",
      "strengths": [
        "Zero markup on tokens, including with BYOK, and no credit-purchase fee",
        "Public unauthenticated models endpoint with pricing and per-provider endpoint data",
        "Per-request ZDR and provider allowlists at no extra cost",
        "Deep AI SDK integration plus provider ordering, sorting, timeouts and model fallbacks",
        "Strong platform compliance posture: SOC 2 Type 2, ISO 27001, HIPAA on enterprise, GDPR"
      ],
      "weaknesses": [
        "Narrower long-tail catalog and fewer provider deployments per open-weight model than OpenRouter",
        "Team-wide ZDR and team-wide provider allowlists are metered at $0.10 per 1,000 requests",
        "BYOK requires purchased credits, can silently fall back to Vercel credentials, and is excluded from budgets",
        "No semantic cache and no native content guardrails"
      ],
      "avoid_if": "You need the widest possible provider pool per model, self-hosting, or you are not otherwise on Vercel.",
      "reviews": [
        {
          "url": "https://zackproser.com/blog/vercel-ai-gateway-review",
          "why": "Independent engineer benchmarks the gateway's latency and pricing against direct provider calls while using it for real Claude Code work.",
          "date": null,
          "type": "written",
          "title": "Vercel AI Gateway Review: Measured Speed & Pricing",
          "publisher": "zackproser.com"
        },
        {
          "url": "https://www.infoq.com/news/2025/09/vercel-ai-gateway/",
          "why": "InfoQ's Daniel Dominguez covers the GA launch and collects developer pushback on failover behaviour and how pricing compares with OpenRouter.",
          "date": "2025-09-12",
          "type": "written",
          "title": "Vercel Introduces AI Gateway for Multi-Model Integration",
          "publisher": "InfoQ"
        },
        {
          "url": "https://www.youtube.com/watch?v=pql8C8elbnk",
          "why": "Hands-on build of a multi-model chatbot through the gateway with the AI SDK, covering key setup, model switching and what the Vercel dashboard reports.",
          "date": "2025-08-24",
          "type": "video",
          "title": "New Vercel AI Gateway is Live! Here's How to Use It",
          "publisher": "YouTube - OrcDev"
        },
        {
          "url": "https://www.reddit.com/r/LangChain/comments/1u37foh/vercel_ai_gateway_is_it_ready_for_production/",
          "why": "Enterprise-readiness thread where practitioners weigh gateway lock-in, auditability of agent actions and keeping an abstracted model-access layer behind it.",
          "date": "2026-06-11",
          "type": "community",
          "title": "Vercel AI Gateway, is it ready for production enterprise use?",
          "publisher": "Reddit - r/LangChain"
        },
        {
          "url": "https://news.ycombinator.com/item?id=47889982",
          "why": "Practitioner digs into the gateway's cache-read pricing ($0.01/M flash vs $0.14/M pro) and the 64% discount versus direct and OpenRouter pricing.",
          "date": "2026-04-24",
          "type": "community",
          "title": "Why is cache for DeepSeek-v4 cheapest on Vercel AI Gateway?",
          "publisher": "Hacker News"
        }
      ],
      "sources": {
        "zdr": "https://vercel.com/docs/ai-gateway/pricing",
        "gdpr": "https://vercel.com/security",
        "soc2": "https://vercel.com/security",
        "license": "https://mcp.directory/blog/vercel-ai-gateway-vs-portkey-vs-openrouter-vs-litellm-2026",
        "sdk_note": "https://vercel.com/docs/ai-gateway/provider-options",
        "url_docs": "https://vercel.com/docs/ai-gateway",
        "url_home": "https://vercel.com/docs/ai-gateway/pricing",
        "zdr_note": "https://vercel.com/docs/ai-gateway/pricing",
        "cap_audio": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway",
        "cap_video": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway",
        "free_tier": "https://vercel.com/docs/ai-gateway/pricing",
        "hipaa_baa": "https://vercel.com/security",
        "air_gapped": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "deployment": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "extra_fees": "https://vercel.com/docs/ai-gateway/pricing",
        "hq_country": "https://en.wikipedia.org/wiki/Vercel",
        "url_status": "https://www.vercel-status.com",
        "zdr_status": "https://vercel.com/docs/ai-gateway/security-and-compliance/zdr",
        "cap_budgets": "https://vercel.com/docs/ai-gateway/byok",
        "model_count": "https://ai-gateway.vercel.sh/v1/models",
        "url_pricing": "https://vercel.com/docs/ai-gateway/pricing",
        "api_surfaces": "https://vercel.com/docs/ai-gateway/provider-options",
        "cap_failover": "https://vercel.com/docs/ai-gateway",
        "founded_year": "https://en.wikipedia.org/wiki/Vercel",
        "product_fate": "https://vercel.com/changelog",
        "regions_note": "https://vercel.com/legal/dpa",
        "byok_fee_note": "https://vercel.com/docs/ai-gateway/byok",
        "cap_image_gen": "https://vercel.com/pricing",
        "url_changelog": "https://vercel.com/changelog",
        "vendor_status": "https://vercel.com/blog/series-f",
        "byok_supported": "https://vercel.com/docs/ai-gateway/byok",
        "cap_embeddings": "https://vercel.com/pricing",
        "cap_guardrails": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "certifications": "https://security.vercel.com/",
        "credit_fee_pct": "https://vercel.com/docs/ai-gateway/pricing",
        "retention_days": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs",
        "sla_uptime_pct": "https://vercel.com/legal/sla",
        "url_models_api": "https://vercel.com/docs/ai-gateway/models-and-providers",
        "cap_rate_limits": "https://vercel.com/docs/ai-gateway/pricing",
        "logs_by_default": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs",
        "cap_cache_simple": "https://vercel.com/docs/ai-gateway/provider-options",
        "cap_virtual_keys": "https://vercel.com/docs/ai-gateway/byok",
        "model_count_note": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway",
        "token_markup_pct": "https://vercel.com/docs/ai-gateway/pricing",
        "cap_observability": "https://vercel.com/pricing",
        "eu_residency_note": "https://vercel.com/security",
        "last_release_date": "https://vercel.com/changelog/hy4-preview-now-available-on-ai-gateway",
        "openai_compatible": "https://vercel.com/docs/ai-gateway/byok",
        "subprocessors_url": "https://security.vercel.com/",
        "cap_cache_semantic": "https://api7.ai/openrouter-vs-vercel-ai-gateway",
        "cap_load_balancing": "https://vercel.com/pricing",
        "seat_fee_usd_month": "https://vercel.com/pricing",
        "guardrails_location": "https://vercel.com/docs/ai-gateway/security-and-compliance",
        "latency_source_note": "https://zackproser.com/blog/openrouter-vs-vercel-ai-gateway",
        "requires_vendor_sdk": "https://vercel.com/docs/ai-gateway/sdks-and-apis/openai-chat-completions",
        "data_export_available": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs",
        "prompt_transits_vendor": "https://vercel.com/legal/dpa",
        "vercel_ai_gateway_byok": "https://vercel.com/docs/ai-gateway/byok",
        "cap_conditional_routing": "https://vercel.com/docs/ai-gateway/provider-options",
        "trains_on_customer_data": "https://vercel.com/docs/ai-gateway/security-and-compliance/disallow-prompt-training",
        "vercel_ai_gateway_budgets": "https://vercel.com/docs/ai-gateway/observability-and-spend/budgets",
        "vercel_ai_gateway_pricing": "https://vercel.com/docs/ai-gateway/pricing"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_video": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "air_gapped": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "url_models_api": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "cap_virtual_keys": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:47.93379+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed, with AI Gateway models and features shipping continuously. Vercel raised a Series F in September 2025 at a $9.3B valuation.",
      "last_release_date": "2026-08-28",
      "sla_uptime_pct": 99.99,
      "sla_url": "https://vercel.com/legal/sla",
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Hosted only, with primary processing facilities in the United States.",
      "logs_by_default": "full_content",
      "logs_note": "Vercel's own documentation contradicts itself here. The logs page shows captured inputs and outputs in a transcript view; the security overview states the gateway uses zero data retention by default; the ZDR page scopes ZDR to paid tiers. Get the behaviour for your specific tier in writing before you rely on it.",
      "retention_days": 30,
      "retention_note": "Routing attempt details are kept for 30 days. Prompts and outputs are kept for zero days where ZDR actually applies — which is the point the documentation disagrees on.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "Vercel does not use your prompts or responses for training. Separately, it does not by default avoid providers that do — you must set disallowPromptTraining to filter those out.",
      "subprocessors_url": "https://security.vercel.com/",
      "regions_note": "US primary, and the data-processing agreement permits transfer anywhere in the world. No customer-VPC option and no documented gateway region pinning.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type 2, third-party audited",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPA published",
          "status": "certified"
        },
        "iso27001": {
          "detail": "listed on the trust center",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "listed on the trust center",
          "status": "certified"
        }
      },
      "guardrails_location": "none",
      "guardrails_note": "No guardrails or PII redaction is documented. The security surface is ZDR plus routing filters on provider training policy.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "markup_only",
      "pricing_model_note": "Credit-based pay-as-you-go with 0% markup and no platform fee on tokens. Monetizes gateway features à la carte: provider allowlists, ZDR, custom reporting writes/queries, Trace Drains. Enterprise can pay by invoice with no processing fees. Pricing page last updated 2026-08-23.",
      "min_commitment": "None. Credits purchasable at any time with no obligation to renew; custom volume discounts on Enterprise.",
      "overage_note": "Trace Drains meter trace events delivered and trace data transferred; Pro plans include no allowance for either. Team-wide provider allowlist: $0.10 per 1,000 successful requests. Team-wide zero data retention: $0.10 per 1,000 requests. Custom Reporting: $0.075 per 1,000 tag/user/quota-entity writes and $5 per 1,000 reporting-endpoint queries. All billed outside credits.",
      "cache_support": "none",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Pricing page does not state whether the gateway caches or passes through provider caching, and no cached-token pricing is published. Because Vercel charges provider list price with 0% markup, any provider cache discount would reach the customer unchanged.",
      "cost_controls": {
        "enforcement": "mixed",
        "spend_alerts": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated as virtual keys; budgets attach to keys.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Platform-set per model, higher on paid tier; not user-configurable.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Per-request `only` filter free on all plans. Team-wide provider allowlist is Pro/Enterprise metered at $0.10 per 1,000 successful requests.",
          "tier": "paid",
          "supported": true
        },
        "budget_caps_per_key": {
          "note": "Budgets at four scopes — team, project, API key, team member. Minimum $1, default monthly refresh. Enforcement is soft: the request that crosses the limit still completes, then HTTP 402. BYOK spend excluded from budgets.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Team-scope budgets supported (see per-key).",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "By tags, user IDs and quota entity IDs via Custom Reporting (queried through the reporting endpoint at $5/1k queries). Per team/key/customer not explicitly stated.",
      "cost_export": "Reporting API plus Trace Drains (Pro/Enterprise, metered). CSV/S3/warehouse not stated.",
      "product_kind_note": "Vercel describes AI Gateway as \"a unified API to access hundreds of AI models through a single endpoint\", handling key management, budgets, retries and provider failover ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), updated 2026-08-27). It is a hosted routing proxy in front of third-party providers, not an inference platform.",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "Fully managed service reached at the `https://ai-gateway.vercel.sh/v1` base URL; no customer-installed component is documented ([Vercel AI Gateway getting started](https://vercel.com/docs/ai-gateway/getting-started), updated 2026-08-22). Tier gating applies to features rather than deployment: BYOK is \"available on the paid tier\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)) and zero-data-retention options are priced for Pro/Enterprise ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), updated 2026-08-23).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap: set the OpenAI (or Anthropic) SDK base URL to `https://ai-gateway.vercel.sh/v1` and use a Vercel AI Gateway API key; the AI SDK integration is optional, not required ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), 2026-05-11; [getting started](https://vercel.com/docs/ai-gateway/getting-started), 2026-08-22). Model discovery uses `GET /v1/models`, `GET /v1/models/{creator}/{model}/endpoints` ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), updated 2026-08-24).",
      "model_count_low": 200,
      "model_count_high": 350,
      "model_count_source_note": "Vendor wording varies: \"hundreds of AI models\" ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27) and \"350+ more models\" ([getting started](https://vercel.com/docs/ai-gateway/getting-started), 2026-08-22); a related link on the catalog page says \"200+ models\" ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "n.a. Vercel's models-and-providers page states no provider total ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24); the BYOK credential table lists 5 providers whose keys can be supplied (Anthropic, OpenAI, Azure, Google Vertex AI, Amazon Bedrock) ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)).",
      "first_party_vs_partner": "All third-party routed; Vercel operates no model hardware and brokers access to provider endpoints, exposing per-model provider endpoints for routing ([Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24; [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).",
      "custom_endpoint_support": "n.a. Registering a customer-hosted endpoint (vLLM, Ollama, SageMaker, private URL) is not documented; BYOK is limited to \"any provider listed in our catalog\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok); [Vercel models and providers](https://vercel.com/docs/ai-gateway/models-and-providers), 2026-08-24).",
      "byok_mode": "both",
      "merchant_of_record": "Vercel invoices credit-mode usage, with \"no markup and no platform fee on tokens\", and Enterprise can pay by invoice ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23). In BYOK mode the upstream provider bills the customer's own account, but failed BYOK requests fall back to Vercel's system credentials and are billed to credits ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)).",
      "key_management_note": "Customers add provider credentials in the AI Gateway dashboard for the five supported providers, and gateway keys authenticate the caller; storage mechanism (vault/KMS) is n.a. ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok)). Because it is hosted-only, requests transit Vercel; zero-data-retention options are offered at $0.10 per 1,000 requests on Pro/Enterprise ([Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23).",
      "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` on `https://ai-gateway.vercel.sh/v1`, alongside `GET /v1/models` ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), updated 2026-05-11).",
      "api_anthropic_messages_note": "Yes: `POST /v1/messages` plus `POST /v1/messages/count_tokens` ([Vercel AI Gateway Anthropic Messages API](https://vercel.com/docs/ai-gateway/sdks-and-apis/anthropic-messages-api), updated 2026-08-11).",
      "api_openai_responses_note": "Yes: a dedicated Responses API surface is documented ([Vercel AI Gateway Responses API](https://vercel.com/docs/ai-gateway/sdks-and-apis/responses)).",
      "api_embeddings_note": "Yes: `POST /v1/embeddings` ([Vercel AI Gateway OpenAI compatibility](https://vercel.com/docs/ai-gateway/openai-compat), 2026-05-11).",
      "api_images_note": "Yes: image generation (and video generation) are documented gateway capabilities ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).",
      "api_audio_note": "Yes, both directions: text-to-speech (\"Generate spoken audio from text\"), speech-to-text transcription, and realtime audio over WebSockets ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).",
      "api_batch_note": "n.a. as an OpenAI-style Batch API; video generation is documented as running through background jobs ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27).",
      "deployment_modes_note": "hosted SaaS only ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway), 2026-08-27). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. (not documented).",
      "byok_mode_note": "both: usage can run on Vercel AI Gateway Credits, or on the customer's own provider keys, and BYOK still \"requires purchased credits\" and is \"available on the paid tier\" ([Vercel AI Gateway BYOK](https://vercel.com/docs/ai-gateway/byok); [Vercel AI Gateway pricing](https://vercel.com/docs/ai-gateway/pricing), 2026-08-23).",
      "access_meta": {
        "qualified": [
          "api_audio",
          "api_batch"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "per_request",
      "rel_timeout_note": "`providerOptions.gateway.providerTimeouts` in milliseconds, minimum `1000`, maximum `789000`. It measures time until the provider starts streaming and is cleared on the first token. **BYOK only.** Default value not stated ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "No retry count or backoff strategy on the fetched pages; the documented recovery path is the fallback model list. Default: `n.a.` ([Vercel model fallbacks](https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "ORDERED: a `models` array tried in order (with a top-level `models` shorthand on the Chat Completions surface) ([Vercel model fallbacks](https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "It is automatic provider selection rather than a configurable health check or breaker: default provider choice is based on \"recent uptime and latency\" ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "`not_documented` ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).",
      "rel_load_balancing_config": "per_request",
      "rel_load_balancing_note": "`per_request`, without weights. Controls are `order`, `only` and `sort` (`'cost'`, `'ttft'`, `'tps'`); no weight key is documented ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "No runtime is disclosed anywhere on the fetched Vercel AI Gateway docs, so it is classified `vendor_saas` on that basis ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "No self-host artifact is offered on the fetched docs ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported. Documented caveats: \"Some providers don't support stream cancellation\" and \"You may still be charged for timed-out requests\"; the provider timeout only covers time-to-first-token and is cleared once streaming starts ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "The security and compliance section documents allowlists and ZDR, and states nothing about PII detection ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))",
      "guard_injection": "not_documented",
      "guard_injection_note": "`not_documented` ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))",
      "guard_content": "not_documented",
      "guard_content_note": "`not_documented` ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))",
      "guard_custom": "not_documented",
      "guard_custom_note": "No regex/word-list/JSON-schema policy engine ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Provider Allowlist and Model Allowlist restrict what a team can call; \"A request only returns 403 when no allowed provider can serve it\", opt-in on Pro and Enterprise ([Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "There is no guardrail service; the analogous documented failure is ZDR routing, which fails closed with a `no_providers_available` 400 error when no ZDR provider can serve the model ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr))",
      "log_content": "configurable",
      "log_content_note": "The Logs view shows a transcript of request inputs and outputs, i.e. prompt/completion text is visible, yet the same docs say \"AI Gateway uses zero data retention by default\" and \"permanently deletes your prompts and responses after requests complete\" ([Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs), [Security and compliance](https://vercel.com/docs/ai-gateway/security-and-compliance)) — see Contradictions",
      "log_opt_out": "yes",
      "log_opt_out_note": "Team-wide ZDR from the AI Gateway Settings tab, or per request `providerOptions.gateway.zeroDataRetention: true`; team-wide ZDR costs $0.10 per 1,000 requests while per-request ZDR is free on Pro/Enterprise ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr))",
      "tracing_support": "not_documented",
      "tracing_note": "`n.a.` — No OpenTelemetry or agent tracing documented for the gateway ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability))",
      "export_destinations": [
        "CSV export"
      ],
      "export_destinations_note": "CSV and JSON export from the logs view; no streaming destinations documented ([Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability))",
      "observability_requires_saas": null,
      "log_retention_note": "Routing/observability details are kept 30 days by default, with Observability Plus extending the window ([Observability](https://vercel.com/docs/ai-gateway/observability-and-spend/observability), [Logs](https://vercel.com/docs/ai-gateway/observability-and-spend/logs))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` No numeric latency benchmark is published. Per-request observability is provided instead: response metadata `responseTimeMs`, plus `providerTimeout` and `configuredTimeoutMs`, and error code `PROVIDER_TIMEOUT` ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` `sort: 'tps'` exists as a routing preference but no measured figure is published ([Vercel provider options](https://vercel.com/docs/ai-gateway/models-and-providers/provider-options)).",
        "benchmark_independence": "No figures published ([Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "not_documented",
      "guard_custom_default": "not_documented",
      "rel_notes": "The timeout control being BYOK-only is the sharpest gotcha here: on Vercel-managed credentials there is no documented way to bound a slow provider ([Vercel provider timeouts](https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts)).",
      "guard_notes": "Governance here is routing-level (which provider/model, and whether the provider retains data), not content-level. BYOK keys are skipped under ZDR unless explicitly marked ZDR-compliant ([ZDR](https://vercel.com/docs/ai-gateway/security-and-compliance/zdr)).",
      "fit_agent_features_note": "Tool calling is documented through OpenAI Chat Completions and the AI SDK, including `tools`, `tool(...)`, `inputSchema`, `execute`, and `stopWhen: stepCountIs(5)` for continued processing after tool execution. The AI SDK Gateway provider also documents provider-executed tools and built-in tools including `perplexity_search`, `exa_search`, `tako_search`, and `parallel_search`; sessions, threads, and A2A are not stated. ([AI SDK integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/ai-sdk))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. \"AI Gateway also supports Bring Your Own Key (BYOK) for any provider listed in our catalog,\" i.e. optional ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing)); the quickstart uses an AI Gateway key or OIDC ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))",
      "fit_client_sdk_langs": [
        "TypeScript",
        "Python"
      ],
      "fit_client_sdks_note": "Documented clients include OpenAI client libraries with TypeScript and Python examples, the TypeScript `ai` package, and official Python SDKs for OpenAI (`pip install openai`) and Anthropic (`pip install anthropic`); both Python SDKs provide async clients. The ecosystem documentation also lists LangChain, LlamaIndex, Mastra, Pydantic AI, LiteLLM, and Langfuse integrations. ([AI Gateway Python SDKs](https://vercel.com/docs/ai-gateway/sdks-and-apis/python))",
      "fit_cloud_iam": "oidc_only",
      "fit_cloud_iam_note": "No AWS/Azure/GCP IAM reuse is documented; instead Vercel documents OIDC-based auth with `VERCEL_OIDC_TOKEN` so deployments can call the gateway without a separate API key. ([Vercel AI Gateway authentication docs](https://vercel.com/docs/ai-gateway/authentication))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "No for the first call: \"A payment method is not required for the first API call; a free tier is available… Your free credits start when you make your first AI Gateway request.\" ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))",
      "fit_ecosystem_notes": "AI Gateway provides unified access to hundreds of models through a single endpoint, with integrations for the AI SDK, OpenAI Chat Completions, OpenAI Responses, Anthropic Messages, and other frameworks. The Vercel documentation describes API-key and Vercel OIDC authentication, dashboard-based usage management and observability, provider BYOK, and availability through AWS Marketplace private offers; self-hosting and single-cloud requirements are not stated. ([Vercel AI Gateway overview](https://vercel.com/ai-gateway))",
      "fit_eval_prereqs": "None stated as prerequisites; within the steps you create `.env.local` with `AI_GATEWAY_API_KEY` (an AI Gateway API key), or use OIDC tokens instead. No credit card, cloud account, or cluster stated ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))",
      "fit_eval_shape": "sdk_install",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `streamText({ model: 'openai/gpt-5.6-sol', prompt: 'Invent a new holiday and describe its traditions.' })`, streaming the response and printing token usage and finish reason ([Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": null,
      "fit_kubernetes": "not_documented",
      "fit_kubernetes_note": "n.a. (not documented)",
      "fit_mcp": "client_tooling",
      "fit_mcp_note": "Per Vercel's AI Gateway versus MCP comparison page, the Vercel AI SDK provides `createMCPClient` for converting MCP tools into AI SDK tools, while `mcp-handler` supports building and hosting MCP servers. The page also identifies `mcp.vercel.com` as an official remote MCP server with OAuth. ([AI Gateway vs MCP](https://vercel.com/i/ai-gateway-vs-mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_gateway_platform",
          "fit_kubernetes",
          "fit_terraform"
        ]
      },
      "fit_model_gate": "tier_limited",
      "fit_model_gate_note": "No approval gate, but a catalogue limit on the free tier: it includes \"a subset of models, not the full catalog,\" and other models require purchasing AI Gateway Credits ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))",
      "fit_onboarding_notes": "The hub page splits quickstarts by modality (text, image, video, realtime voice agents, speech, audio transcription) and advertises \"350+ more models\" for text plus \"unified access to 200+ models through the AI SDK for Python,\" so the effort is a Node/TS project scaffold rather than a bare curl ([Getting Started with AI Gateway](https://vercel.com/docs/ai-gateway/getting-started), [Text Generation Quickstart](https://vercel.com/docs/ai-gateway/getting-started/text))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Existing OpenAI client libraries can be used by setting the base URL to `https://ai-gateway.vercel.sh/v1` and authenticating with an AI Gateway API key or Vercel OIDC token. ([OpenAI Chat Completions integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/openai-chat-completions))",
      "fit_prod_prereqs": "A Vercel team account and AI Gateway credits beyond the free tier: \"Every Vercel team account gets access to both a free tier and a paid tier for AI Gateway Credits,\" and buying credits requires \"Choose your payment method and click on Confirm and Pay\" ([AI Gateway Pricing](https://vercel.com/docs/ai-gateway/pricing))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "sdk_install",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Official documentation covers LangChain through the Chat Completions endpoint and LlamaIndex through `llama-index-llms-vercel-ai-gateway`; the LlamaIndex class is `VercelAIGateway` from `llama_index.llms.vercel_ai_gateway`. ([AI Gateway Python SDKs](https://vercel.com/docs/ai-gateway/sdks-and-apis/python))",
      "fit_quickstart_steps": 5,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://vercel.com/docs/ai-gateway/getting-started",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "not_documented",
      "fit_terraform_note": "n.a. (not documented)",
      "fit_terraform_ref": null,
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Vercel AI Gateway works with the Vercel AI SDK. The documented npm package is `ai` (`npm install ai@latest`), and the AI Gateway provider is available through `@ai-sdk/gateway`; OpenAI-compatible access can also use `@ai-sdk/openai-compatible`. ([AI SDK integration](https://vercel.com/docs/ai-gateway/sdks-and-apis/ai-sdk))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/gateway"
    },
    {
      "slug": "apache-apisix-ai-gateway",
      "name": "Apache APISIX AI Gateway",
      "company": "Apache Software Foundation",
      "tagline": "AI proxy plugins for the Apache APISIX API gateway, adding LLM routing and token limits to an OpenResty data plane.",
      "category": "open_source",
      "deployment": "self_host_only",
      "license": "Apache-2.0",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://apisix.apache.org/",
      "url_docs": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
      "url_pricing": null,
      "url_status": null,
      "url_github": "https://github.com/apache/apisix",
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": 16800,
      "model_count": null,
      "model_count_note": "No model count is published; models are whatever the configured upstream provider exposes.",
      "provider_count": 10,
      "token_markup_pct": 0,
      "credit_fee_pct": 0,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Provider credentials are configured in the plugin; the project charges nothing.",
      "seat_fee_usd_month": 0,
      "free_tier": "Entire project is free under Apache-2.0, including the ai-proxy and ai-proxy-multi AI gateway plugins.",
      "enterprise_from_usd_year": null,
      "extra_fees": [],
      "self_host_cost_note": "No license cost at all; you run APISIX and its etcd control store on your own infrastructure. The docs are co-branded with API7, a commercial vendor offering a supported distribution, whose pricing is not published on these pages.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenAI Embeddings",
        "Anthropic Messages"
      ],
      "sdk_note": "No SDK: you declare an APISIX route with the ai-proxy or ai-proxy-multi plugin, which normalizes requests and can convert to the Anthropic /v1/messages format.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": null,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": 0.2,
      "latency_source_note": "Project-published (Apache APISIX, not AI-specific): under 0.2 ms average added latency, ~18,000 QPS on a single core and 140,000 QPS on an 8-core AWS instance at 0.2 ms. These are general gateway figures rather than AI-plugin measurements; a third-party 2026 comparison estimates roughly 1-2 ms overhead on the AI path.",
      "throughput_rps": 18000,
      "soc2": null,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": null,
      "zdr": null,
      "zdr_note": "Traffic never leaves your infrastructure, so there is no vendor retention to negotiate away.",
      "air_gapped": null,
      "best_for": "Teams already running APISIX that want basic multi-provider LLM proxying, retries and token-based rate limiting without adding another gateway.",
      "strengths": [
        "Fastest published data-plane numbers here: ~18k QPS per core at under 0.2 ms added latency",
        "Fully Apache-2.0 with no paid tier, gated features or vendor account required",
        "Mature routing primitives: round-robin/consistent-hash/semantic load balancing, retries with time windows, active health checks and fallback strategies",
        "Token-level logging variables and token-based rate limiting for LLM traffic"
      ],
      "weaknesses": [
        "Not AI-first: no response or semantic caching, budgets, virtual keys or prompt management in the AI plugins",
        "Only about ten provider integrations documented, versus 20-140 for LLM-native gateways",
        "Plugin configuration is Lua/OpenResty-shaped and the published performance numbers are for generic proxying, not LLM paths",
        "No compliance certifications, support SLA or hosted control plane from the project itself"
      ],
      "avoid_if": "You want LLM-native features like semantic caching, spend budgets, virtual keys or a managed control plane out of the box.",
      "reviews": [
        {
          "url": "https://medium.com/syntonize/enterprise-ready-llm-gateway-with-apache-apisix-part-2-bd796db06340",
          "why": "Consultancy engineer builds an enterprise LLM gateway with APISIX's ai-proxy, key-auth, consumer-restriction and OIDC plugins wired to Entra ID, with config shown.",
          "date": "2025-05-21",
          "type": "written",
          "title": "Enterprise-ready LLM Gateway with Apache APISIX (Part 2)",
          "publisher": "Medium - Syntonize"
        },
        {
          "url": "https://www.youtube.com/watch?v=hKB9JFrtttA",
          "why": "Third-party case study of an enterprise agent platform fronted by the APISIX AI gateway, routing between private-datacenter and public models plus MCP/A2A traffic.",
          "date": "2026-02-01",
          "type": "video",
          "title": "Building an Enterprise AI Agent Platform: APISIX AI Gateway in Practice",
          "publisher": "YouTube - Let's Talk Tech"
        }
      ],
      "sources": {
        "cap_mcp": "https://github.com/apache/apisix",
        "company": "https://github.com/apache/apisix",
        "license": "https://github.com/apache/apisix",
        "sdk_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "free_tier": "https://github.com/apache/apisix",
        "hq_country": "https://en.wikipedia.org/wiki/The_Apache_Software_Foundation",
        "weaknesses": "https://dev.to/pranay_batta/best-open-source-ai-gateway-in-2026-2flb",
        "api7_pricing": "https://api7.ai/pricing",
        "api_surfaces": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "cap_failover": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/",
        "github_stars": "https://github.com/apache/apisix",
        "product_fate": "https://github.com/apache/apisix/releases",
        "regions_note": "https://apisix.apache.org/ai-gateway/",
        "apisix_github": "https://github.com/apache/apisix",
        "vendor_status": "https://apisix.apache.org/",
        "byok_supported": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "cap_embeddings": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "cap_guardrails": "https://github.com/apache/apisix",
        "certifications": "https://apisix.apache.org/ai-gateway/",
        "config_as_code": "https://apisix.apache.org/docs/apisix/deployment-modes/",
        "credit_fee_pct": "https://github.com/apache/apisix",
        "provider_count": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "throughput_rps": "https://github.com/apache/apisix",
        "apisix_ai_proxy": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "cap_rate_limits": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "logs_by_default": "https://apisix.apache.org/ai-gateway/",
        "model_count_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "token_markup_pct": "https://github.com/apache/apisix",
        "apisix_ai_gateway": "https://apisix.apache.org/ai-gateway/",
        "cap_observability": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "last_release_date": "https://github.com/apache/apisix/releases/tag/3.18.0",
        "openai_compatible": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "cap_load_balancing": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/",
        "seat_fee_usd_month": "https://github.com/apache/apisix",
        "guardrails_location": "https://apisix.apache.org/docs/apisix/plugins/ai-prompt-guard/",
        "latency_overhead_ms": "https://github.com/apache/apisix",
        "latency_source_note": "https://apisix.apache.org/",
        "requires_vendor_sdk": "https://apisix.apache.org/docs/apisix/3.17/plugins/ai-proxy/",
        "self_host_cost_note": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/",
        "data_export_available": "https://apisix.apache.org/docs/apisix/plugins/http-logger/",
        "prompt_transits_vendor": "https://apisix.apache.org/ai-gateway/",
        "apisix_ai_rate_limiting": "https://apisix.apache.org/docs/apisix/plugins/ai-rate-limiting/",
        "cap_conditional_routing": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy-multi/"
      },
      "verified_at": {
        "cap_mcp": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "free_tier": "2026-08-29",
        "weaknesses": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "throughput_rps": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:32.97827+00:00",
      "vendor_status": "foundation",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "An Apache Software Foundation top-level project on a regular release train, 3.16 through 3.18 between April and August 2026. There is no vendor behind it, which cuts both ways: nobody can be acquired or change the licence, and nobody owes you a support contract or an SLA.",
      "last_release_date": "2026-08-20",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "no",
      "prompt_path_note": "Apache-licensed software you deploy yourself. There is no foundation-operated hosted gateway, so no third party receives your prompts.",
      "logs_by_default": "not_applicable",
      "logs_note": "Token usage goes to your own access logs and observability stack. Whatever it keeps is your decision.",
      "retention_days": null,
      "retention_note": "Determined entirely by the log store you point it at.",
      "zdr_status": "not_applicable",
      "trains_on_customer_data": "not_applicable",
      "training_note": "The project ships software rather than a data-processing service; no vendor receives prompts.",
      "subprocessors_url": null,
      "regions_note": "Anywhere you run it. There are no vendor-managed regions.",
      "certifications": {
        "itar": {
          "status": "not_applicable"
        },
        "soc2": {
          "status": "not_applicable"
        },
        "fedramp": {
          "status": "not_applicable"
        },
        "gdpr_dpa": {
          "status": "not_applicable"
        },
        "iso27001": {
          "status": "not_applicable"
        },
        "hipaa_baa": {
          "status": "not_applicable"
        }
      },
      "guardrails_location": "self_hosted",
      "guardrails_note": "Prompt guards, content moderation and auditing run in your own gateway before traffic reaches a model, checking content against allow and deny patterns you define.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "oss_no_vendor_price",
      "pricing_model_note": "Apache 2.0 OSS project — no ASF pricing page, tier list or paid SKU exists. Commercial support is sold by a separate vendor, API7.ai: API7 Cloud Standard is $2 per 1M API calls (floored to $0.01) + $250/gateway group/month + $10/service/month; API7 Enterprise is annual licensing by API gateway CPU core with a custom quote.",
      "min_commitment": "None for OSS. API7 Enterprise is annual per-CPU-core licensing.",
      "overage_note": "None for OSS. API7 Cloud is pay-as-you-go 'with no usage limitation', so cost grows with volume rather than being an overage penalty.",
      "cache_support": "none",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "The AI Gateway overview and ai-rate-limiting plugin docs do not document a response cache or provider-cache passthrough — usage fields like cached_tokens are surfaced but no gateway-owned cache is described. APISIX never prices tokens, so any caching saving is 100% the customer's.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Not supported / not stated.",
          "tier": null,
          "supported": false
        },
        "virtual_keys": {
          "note": "Supported in effect via Consumer and Consumer Group objects, though not branded as virtual keys.",
          "tier": null,
          "supported": true
        },
        "rate_limiting": {
          "note": "Free, pre-request.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not supported / not stated.",
          "tier": null,
          "supported": false
        },
        "model_allowlists": {
          "note": "Not stated (routing/fallback across providers is; model allowlisting is not).",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Token-quota caps rather than dollar budgets. Pre-request. ai-rate-limiting plugin controls tokens consumed within a time frame with limit_strategy of total_tokens/prompt_tokens/completion_tokens or an expression. Default local policy keeps counters per node — effective quota scales with node count.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Scoped by Route, Service, Consumer, Consumer Group or custom attributes.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "Token consumption trackable by Route, Service, Consumer, Consumer Group or custom attributes. Dollar cost attribution not documented.",
      "cost_export": "None stated. Observability comes from the plugin ecosystem.",
      "product_kind_note": "Apache calls it \"APISIX AI Gateway\", \"a complete AI gateway product\" built on \"a modern API gateway\", with `ai-proxy` / `ai-proxy-multi` plugins that \"simplify access to large language models (LLMs) and embedding models\" ([Introducing APISIX AI Gateway](https://apisix.apache.org/blog/2025/04/08/introducing-apisix-ai-gateway/), 2025-04-08). Its own AI page calls it \"an open-source LLM gateway and proxy\" ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)). It is a self-hosted routing proxy, not a model platform.",
      "deployment_modes": [
        "saas",
        "self_host",
        "on_prem"
      ],
      "deployment_note": "Configuration is applied to a locally run gateway through the APISIX Admin API (`http://127.0.0.1:9180/apisix/admin/routes`) and a `config.yaml`, reloaded after changes ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). Commercial on-prem licensing is per gateway CPU core, annually: \"Our on-premises API gateway pricing is based on API Gateway CPU cores\" ([API7 pricing](https://api7.ai/pricing)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "partial",
      "api_embeddings": "yes",
      "api_images": "not_documented",
      "api_audio": "not_documented",
      "api_batch": "not_documented",
      "api_surface_note": "Not a base-URL-swap SaaS: you configure APISIX routes whose `override.endpoint` points at a provider, so clients call your gateway's route paths ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). No proprietary SDK. Streaming caveats are explicit: `max_stream_duration_ms` can truncate an SSE stream without a protocol terminator such as `[DONE]`, `message_stop` or `response.completed`, and Bedrock streaming returns AWS EventStream binary framing (`Content-Type: application/vnd.amazon.eventstream`) that clients must parse themselves; Bedrock routes must end in `/converse` (`/model/<model>/converse-stream` for streaming) ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "model_count_low": null,
      "model_count_high": null,
      "model_count_source_note": "n.a.; no model total is stated, only examples such as `gpt-4`, `deepseek-chat`, `text-embedding-3-small` ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/), undated).",
      "provider_count_low": 10,
      "provider_count_high": 20,
      "provider_count_source_note": "\"20+ model providers\" on the AI gateway page ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/), undated). The plugin reference enumerates 10 provider values — `openai`, `deepseek`, `azure-openai`, `aimlapi`, `anthropic`, `openrouter`, `gemini`, `vertex-ai`, `bedrock`, `openai-compatible` — plus \"other OpenAI-compatible APIs\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), undated).",
      "first_party_vs_partner": "All third-party: the gateway forwards to upstream provider APIs (OpenAI, DeepSeek, Azure OpenAI, Anthropic, OpenRouter, Gemini, Vertex AI, Bedrock, AIMLAPI) and ships no models of its own ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "custom_endpoint_support": "Yes: any endpoint can be set with `override.endpoint`, and it is required for the `openai-compatible` provider — \"Custom LLM provider endpoint, required when `provider` is `openai-compatible`\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)). vLLM/Ollama/SageMaker by name: n.a.",
      "byok_mode": "byok_only",
      "merchant_of_record": "The upstream provider, implicitly: the gateway is software the customer runs with its own provider keys, and no APISIX-side billing for model usage exists ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)). API7 invoices only the gateway license, per CPU core ([API7 pricing](https://api7.ai/pricing)). An explicit statement of who invoices model usage: n.a.",
      "key_management_note": "Keys live in the customer's own APISIX configuration, per provider instance via `instances.auth.header` / `auth.query`, with GCP service-account JSON in `auth.gcp.service_account_json` or the `GCP_SERVICE_ACCOUNT` environment variable; AWS `secret_access_key` and `session_token` are \"stored encrypted\" ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)). There is no vendor plane: request/response payload logging is opt-in via `logging.payloads` (with `logging.summaries` for model, duration and token counts) and goes to the operator's own `http-logger`/`kafka-logger` sinks; incoming client headers including `Authorization` and `Cookie` are forwarded upstream by default unless stripped ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "api_openai_chat_completions_note": "Yes: routes proxy to the upstream `/v1/chat/completions` path and requests use the OpenAI `messages` array format ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).",
      "api_anthropic_messages_note": "Yes: setting the route URI to `/v1/messages` triggers automatic conversion between Anthropic Messages format and OpenAI-compatible format, including SSE streaming ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "api_openai_responses_note": "Partially documented: `openai-responses` appears as a target protocol and in the `max_tokens` mapping table as \"OpenAI Responses API\", but no `/v1/responses` path is given ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "api_embeddings_note": "Yes: documented with an override to `https://api.openai.com/v1/embeddings` using `text-embedding-3-small` ([ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).",
      "api_images_note": "n.a. (no image endpoint documented on the pages fetched).",
      "api_audio_note": "n.a. (no STT/TTS endpoint documented on the pages fetched).",
      "api_batch_note": "n.a.",
      "deployment_modes_note": "self-host (open-source data plane) ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)); on-prem commercial edition via API7 ([API7 pricing](https://api7.ai/pricing)). API7 also markets a managed option — \"the open-source & managed AI gateway\" ([API7 pricing](https://api7.ai/pricing)). Hybrid/VPC and air-gapped: n.a. — The managed/SaaS shape is offered by API7, the commercial steward, not by the Apache APISIX project itself.",
      "byok_mode_note": "byok_only: every example configures the customer's own provider credentials (`OPENAI_API_KEY`, `DEEPSEEK_API_KEY`, GCP service-account JSON, AWS keys) in the plugin's `auth` block ([ai-proxy plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/), [ai-proxy-multi plugin](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy-multi/)).",
      "access_meta": {
        "qualified": [],
        "model_count_kind": null,
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "config_file",
      "rel_timeout_note": "`ai-proxy` / `ai-proxy-multi` key `timeout`, integer milliseconds, range `1`–`60000`, **default `30000`**. Connection reuse is tuned with `keepalive`, `keepalive_timeout` (`60000`) and `keepalive_pool` (`30`) ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/), [APISIX ai-proxy](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy/)).",
      "rel_retry_config": "config_file",
      "rel_retry_note": "Retry/fallback ordering is driven by `provider.priority`; no retry-count key, default count or backoff strategy is documented on the fetched plugin pages: `n.a.` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "ORDERED via `provider.priority` (higher-priority providers are tried first, lower priorities act as fallbacks); WEIGHTED distribution is separately available via `provider.weight` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)).",
      "rel_fallback_shape": "weighted",
      "rel_health_check_config": "config_file",
      "rel_health_check_note": "`config_file`, and it is both an upstream health check and unhealthy-node ejection: APISIX will \"Enable health check on the upstream node and will automatically filter unhealthy nodes during load balancing\", plus \"Intelligent tracking of unhealthy upstream services\" (circuit breaking) ([apache/apisix](https://github.com/apache/apisix)). The AI Gateway landing page also advertises \"health checks and weighted balancing\" without naming keys ([APISIX AI Gateway](https://apisix.apache.org/ai-gateway/)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "`not_documented` for provider failover. The only HA statement is about the control store: \"Support to configure multiple etcd addresses in the same cluster\" ([apache/apisix](https://github.com/apache/apisix)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Weights supported: `provider.weight`, `balancer.algorithm` (`roundrobin` default, or `chash`) with `balancer.hash_on` / `balancer.key` ([APISIX ai-proxy-multi](https://apisix.apache.org/docs/apisix/3.12/plugins/ai-proxy-multi/)). The repo describes round-robin \"with weight\", consistent-hash balancing, and custom algorithms in the `balancer` phase ([apache/apisix](https://github.com/apache/apisix)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "Repo language breakdown Lua 81.6% (OpenResty-style plugin runtime; Java/Go/Python/Node.js plugins run out-of-process via RPC, and Wasm is experimental) ([apache/apisix](https://github.com/apache/apisix)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "Runs \"from bare-metal to Kubernetes\", official Helm charts installed with `helm repo add apisix https://apache.github.io/apisix-helm-chart` then `helm install apisix apisix/apisix` ([apache/apisix](https://github.com/apache/apisix), [APISIX Helm chart](https://apisix.apache.org/docs/helm-chart/apisix/)).",
      "latency_claims": [
        {
          "url": "https://github.com/apache/apisix",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "0.2",
          "metric": "mean",
          "publisher": "vendor",
          "conditions": "18k QPS on a single core. General API-gateway benchmark, NOT measured on LLM proxying; payload and cache state not stated."
        },
        {
          "url": "https://github.com/apache/apisix",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "0.2",
          "metric": "mean",
          "publisher": "vendor",
          "conditions": "140,000 QPS on an eight-core AWS server. Again general gateway traffic, not LLM proxying."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://github.com/apache/apisix",
          "unit": "QPS",
          "scope": "capacity",
          "value": "18,000",
          "metric": "sustained",
          "publisher": "vendor",
          "conditions": "Single core, general gateway traffic rather than LLM proxying."
        },
        {
          "url": "https://github.com/apache/apisix",
          "unit": "QPS",
          "scope": "capacity",
          "value": "140,000",
          "metric": "sustained",
          "publisher": "vendor",
          "conditions": "Eight-core AWS server, general gateway traffic."
        }
      ],
      "streaming_support": "partial",
      "streaming_note": "SSE handling is documented for the `mcp-bridge` plugin, which converts stdio MCP servers into \"scalable HTTP SSE services\"; no LLM-streaming caveat is stated on the fetched AI plugin pages ([apache/apisix](https://github.com/apache/apisix)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "No PII/DLP plugin on the fetched AI plugin pages ([AI Gateway](https://apisix.apache.org/ai-gateway/), [ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "guard_injection": "sync_block",
      "guard_injection_note": "`ai-prompt-guard` matches allow/deny regex patterns against prompts and rejects with HTTP 400; detection is regex, not a classifier ([ai-prompt-guard](https://apisix.apache.org/docs/apisix/next/plugins/ai-prompt-guard/))",
      "guard_content": "sync_block",
      "guard_content_note": "`ai-aws-content-moderation` sends the request body to AWS Comprehend and rejects when a category or the overall toxicity threshold is exceeded, returning `400 Bad Request` with e.g. \"request body exceeds PROFANITY threshold\" ([ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policy form is PCRE-style allow/deny pattern lists in `ai-prompt-guard`, plus per-category numeric moderation thresholds ([ai-prompt-guard](https://apisix.apache.org/docs/apisix/next/plugins/ai-prompt-guard/), [ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages — model selection is per route via `ai-proxy` configuration rather than an administrative allow/deny list ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "AWS Comprehend"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([ai-aws-content-moderation](https://docs.api7.ai/hub/ai-aws-content-moderation))",
      "log_content": "metadata_only",
      "log_content_note": "`metadata_only` — and in fact less: in `ai-proxy`, `logging.summaries` and `logging.payloads` both default to `false`, so neither token summaries nor request/response bodies are logged unless enabled ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`logging.payloads: false` (the default) disables body logging while `logging.summaries: true` can keep token statistics ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry natively via the `opentelemetry` plugin (binary OTLP over HTTP, configurable sampler, optional full request-lifecycle spans); the plugin is disabled by default. Multi-step agent traces are not modelled — spans are HTTP-request-scoped ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/))",
      "export_destinations": [
        "OpenTelemetry",
        "Prometheus",
        "Kafka",
        "HTTP logger",
        "Loki",
        "Datadog",
        "Splunk",
        "Google Cloud Logging"
      ],
      "export_destinations_note": "OTLP/HTTP collectors for traces ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/)); `http-logger` and `kafka-logger` for AI request logs, with the `llm_content_risk_level` variable available to loggers ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([AI Gateway](https://apisix.apache.org/ai-gateway/))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` ([AI Gateway](https://apisix.apache.org/ai-gateway/))",
      "observability_requires_saas": "No — everything runs and exports from the customer's own deployment ([OpenTelemetry plugin](https://apisix.apache.org/docs/apisix/plugins/opentelemetry/))",
      "log_retention_note": "`n.a.` — determined by the customer's log sink ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "Two figures, both general-gateway (not LLM-specific): \"The single-core QPS reaches 18k with an average delay of fewer than **0.2 milliseconds**\" (metric: average delay; single core; payload/cache/hardware model unspecified) and \"APISIX's QPS reaches 140,000 with a latency of only **0.2 ms**\" using \"AWS's eight-core server\" (payload and cache conditions not stated). No p50/p99 or gateway-overhead breakdown is given ([apache/apisix](https://github.com/apache/apisix)).",
        "multi_surface": {},
        "throughput_prose": "**18k QPS** single-core and **140,000 QPS** on an eight-core AWS server; payload and cache conditions not stated, and neither figure is attributed to LLM proxying ([apache/apisix](https://github.com/apache/apisix)).",
        "benchmark_independence": "Vendor/project self-published (Apache APISIX repo). No independent third-party benchmark and no claims about competitors on the fetched pages ([apache/apisix](https://github.com/apache/apisix))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "sync_block",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "The published QPS/latency numbers describe APISIX as a general API gateway, not as an AI proxy carrying token streams; do not read them as LLM gateway overhead ([apache/apisix](https://github.com/apache/apisix)).",
      "guard_notes": "APISIX is the most privacy-default of the group: with payload and summary logging both off by default, an operator must deliberately turn on observability, which is the inverse of most hosted gateways ([ai-proxy](https://apisix.apache.org/docs/apisix/next/plugins/ai-proxy/)).",
      "fit_agent_features_note": "APISIX-MCP documentation describes Cursor Agent mode and YOLO Mode automatically invoking MCP tools, including plugin analysis and route-management tools such as `get_plugins_list`, `get_plugin_schema`, `create_route`, `update_route`, and `get_route`. It also states that the workflow supports task decomposition, closed-loop validation, and iterative optimization; gateway-native sessions, threads, and A2A are not documented. ([APISIX and AI/MCP intelligent API management](https://apisix.apache.org/blog/2025/04/01/embrace-intelligent-api-management-with-ai-and-mcp/))",
      "fit_byok_required": "yes",
      "fit_byok_required_note": "Effectively yes: the ai-proxy examples configure provider API keys or AWS credentials (Bedrock \"requires AWS SigV4 credentials and a non-empty AWS region\"), though the page does not state a general first-call rule ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))",
      "fit_client_sdk_langs": [],
      "fit_client_sdks_note": "The API7 integration catalog lists OpenAI SDK and Anthropic SDK integrations. The documented TypeScript integration uses `ai`, `@ai-sdk/openai`, and `@ai-sdk/openai-compatible`; official Apache APISIX client libraries and supported client languages are not otherwise documented. ([API7 AI Gateway integrations](https://docs.api7.ai/ai-gateway/integrations))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Yes: the `ai-proxy` plugin documents Amazon Bedrock authentication with AWS IAM credentials and SigV4 signing and Vertex AI authentication with a GCP service-account JSON (`auth.gcp.service_account_json`); Azure OpenAI uses an `api-key` header (no Entra/managed identity documented). ([API7/APISIX ai-proxy plugin reference](https://docs.api7.ai/hub/ai-proxy))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented); open-source install pages state no payment requirement ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))",
      "fit_ecosystem_notes": "Apache APISIX is open source, Apache 2.0 licensed, platform agnostic, and described as having no vendor lock-in, portable configuration, and standalone, cluster, bare-metal, and Kubernetes deployment options. Its ecosystem includes an Ingress Controller, Helm Charts, Docker, Java/Go/Python plugin runners, MCP components, and integrations with OpenAI, DeepSeek, OpenAI-compatible services, AWS Comprehend, Azure OpenAI, and Azure AI Search; API7 documentation also describes open-source gateway, cloud, and hybrid-cloud access. ([Apache APISIX AI Gateway](https://apisix.apache.org/ai-gateway/))",
      "fit_eval_prereqs": "Docker version 20.10 or later and `curl`; no account, credit card, cloud account, cluster, or licence. For an AI completion you additionally need a provider key — the plugin examples use `OPENAI_API_KEY`, `DEEPSEEK_API_KEY`, `AZ_OPENAI_API_KEY`, `ANTHROPIC_API_KEY` ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))",
      "fit_eval_shape": "cli_or_container",
      "fit_eval_snippet": "partial",
      "fit_eval_snippet_note": "Yes, but in two parts: the getting-started page verifies with `curl \"http://127.0.0.1:9080/get\"` (not a completion), while the ai-proxy page shows the LLM call `curl \"http://127.0.0.1:9080/anything\" -X POST -H \"Content-Type: application/json\" -H \"Host: api.openai.com\" -d '{\"messages\":[{\"role\":\"system\",\"content\":\"You are a mathematician\"},{\"role\":\"user\",\"content\":\"What is 1+1?\"}]}'` after you configure a Route with the `ai-proxy` plugin ([Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/), [AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/))",
      "fit_gateway_platform": "is_itself_gateway",
      "fit_gateway_platform_note": "Apache APISIX is itself an open-source API gateway and AI gateway, built on NGINX and etcd, with an AI-focused plugin ecosystem. Documented AI plugins include `ai-proxy`, `ai-proxy-multi`, `ai-request-rewrite`, `ai-rate-limiting`, `ai-prompt-decorator`, `ai-prompt-template`, `ai-prompt-guard`, `ai-aws-content-moderation`, and `ai-rag`. ([Introducing Apache APISIX AI Gateway](https://apisix.apache.org/blog/2025/04/08/introducing-apisix-ai-gateway/))",
      "fit_helm_chart": "apache/apisix-helm-chart; API7 ships api7/gateway from https://charts.api7.ai",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Apache APISIX documents a Kubernetes Ingress Controller and Helm Charts. The official Apache repository is `apache/apisix-helm-chart`, containing the Apache APISIX Helm Chart and Apache APISIX Ingress Controller Helm Chart; a separate API7 chart is named `api7/gateway` in the `https://charts.api7.ai` repository. ([Apache APISIX Helm Charts repository](https://github.com/apache/apisix-helm-chart))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "Apache APISIX documents MCP support through gateway proxying and plugins: it can proxy Streamable HTTP traffic between MCP clients and backend MCP servers, and the `mcp-bridge` plugin converts stdio-based MCP servers to scalable HTTP SSE services. APISIX-MCP is also documented as an open-source npm/GitHub MCP server, available with `npx -y apisix-mcp`, for managing APISIX through its Admin API. ([Apache APISIX MCP Protocol AI Gateway guide](https://apisix.apache.org/learning-center/mcp-protocol-ai-gateway/))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_credit_card",
          "fit_model_gate",
          "fit_terraform"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented) ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/), [Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/))",
      "fit_onboarding_notes": "After install you must configure a Route plus `ai-proxy` with provider auth and model; Kubernetes examples require `kubectl apply -f ai-proxy-ic.yaml` and declarative config uses `adc sync -f adc.yaml`; logger examples require setting up Kafka. Non-container installs need `apisix init` then `apisix start`, with config in `conf/config.yaml` and optional systemd management ([AI Proxy | APISIX](https://apisix.apache.org/docs/apisix/plugins/ai-proxy/), [Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))",
      "fit_openai_dropin": "partial",
      "fit_openai_dropin_note": "An OpenAI SDK integration is listed, but generic drop-in compatibility is not explicitly documented. The Vercel AI SDK example configures a custom `baseURL` of `http://127.0.0.1:3000/v1` with `@ai-sdk/openai`. ([API7 Vercel AI SDK integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/vercel-ai-sdk))",
      "fit_prod_prereqs": "etcd as the configuration store (installed automatically by the Docker and Helm methods); Docker 20.10+ for the container path; \"Production environments should enable Admin API authentication\"; the quickstart containers use Docker host network mode. No database, Redis, ClickHouse, load balancer, control-plane account, licence, or S3 requirement is stated ([Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/), [Getting Started with Apache APISIX](https://apisix.apache.org/docs/apisix/getting-started/README/))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "infra_rollout",
      "fit_python_frameworks": [
        "LangChain",
        "LangGraph",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Python integrations are documented for LangChain through `langchain-openai` and `ChatOpenAI`, and for LangGraph through `langgraph`. LlamaIndex is documented through `llama-index-llms-openai` and the `OpenAIResponses` class; `OpenAILike` is also mentioned for Chat Completions compatibility. ([API7 LangChain integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/langchain))",
      "fit_quickstart_steps": 0,
      "fit_quickstart_steps_caveat": "The getting-started page is prose plus commands, and its verification call is not an LLM completion. The actual LLM call lives on the ai-proxy plugin reference page.",
      "fit_quickstart_url": "https://apisix.apache.org/docs/apisix/getting-started/README/",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`git clone https://github.com/apache/apisix-docker.git`, `cd apisix-docker/example`, `docker-compose -p docker-apisix up -d`; ARM: `docker-compose -p docker-apisix -f docker-compose-arm64.yml up -d`; Helm: `helm repo add apisix https://charts.apiseven.com`, `helm repo update`, `helm install apisix apisix/apisix --create-namespace --namespace apisix`; RPM: `sudo yum install apisix`; DEB: `sudo apt install -y apisix` ([Installation - Apache APISIX](https://apisix.apache.org/docs/apisix/installation-guide/))",
      "fit_terraform": "not_documented",
      "fit_terraform_note": "n.a. (not documented)",
      "fit_terraform_ref": null,
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "openai_compat_only",
      "fit_vercel_ai_sdk_note": "Vercel AI SDK integration is documented using the npm packages `ai` and `@ai-sdk/openai`; it creates a provider with `createOpenAI`. The example uses the gateway base URL `http://127.0.0.1:3000/v1`. ([API7 Vercel AI SDK integration](https://docs.api7.ai/ai-gateway/integrations/frameworks/vercel-ai-sdk))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/openai with a custom baseURL"
    },
    {
      "slug": "bifrost",
      "name": "Bifrost",
      "company": "Maxim AI",
      "tagline": "Go-based open-source AI gateway focused on low proxy overhead, with an enterprise tier for clustering and SSO.",
      "category": "open_source",
      "deployment": "self_host_only",
      "license": "Apache-2.0",
      "founded_year": 2023,
      "hq_country": "United States",
      "url_home": "https://www.getmaxim.ai/bifrost",
      "url_docs": "https://docs.getbifrost.ai/",
      "url_pricing": "https://www.getmaxim.ai/pricing",
      "url_status": null,
      "url_github": "https://github.com/maximhq/bifrost",
      "url_changelog": "https://github.com/maximhq/bifrost/releases",
      "url_models_api": null,
      "github_stars": 7600,
      "model_count": 1000,
      "model_count_note": "Vendor-stated 1,000+ models across 20+ providers; the product page headline cites 8+ providers, so counts differ by page.",
      "provider_count": 23,
      "token_markup_pct": 0,
      "credit_fee_pct": 0,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Self-hosted with your own provider keys and vault integration; no gateway token fee.",
      "seat_fee_usd_month": 0,
      "free_tier": "OSS tier is \"Free Forever\": drop-in gateway, OpenTelemetry observability, budgets, rate limits, virtual keys, custom routing, fallbacks, simple and semantic caching, MCP gateway, prompt repository.",
      "enterprise_from_usd_year": null,
      "extra_fees": [],
      "self_host_cost_note": "OSS is free (Apache-2.0); infra cost only. Enterprise adds guardrails, cluster mode, adaptive load balancing, SAML/OIDC SSO, vault integration, log exports, audit logs, RBAC and SLAs at custom pricing after a 14-day trial; VPC, on-prem and air-gapped installs are enterprise options.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Anthropic Messages",
        "OpenAI Embeddings",
        "MCP"
      ],
      "sdk_note": "Drop-in replacement: change the base URL of an existing OpenAI/Anthropic client; also ships a Go SDK and CLI.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": null,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": true,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": 0.99,
      "latency_source_note": "Vendor-run (Maxim AI): 0.99 ms mean gateway overhead vs LiteLLM's 40 ms against a 60 ms mock OpenAI upstream at 500 RPS on a t3.medium; a separate Maxim stress test reports ~11-20 microseconds internal overhead at 5,000 RPS on a t3.xlarge with 100% success. Not independently verified, and LiteLLM's own competing benchmark measured Bifrost at 4.54 ms added p99.",
      "throughput_rps": 5000,
      "soc2": null,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": null,
      "zdr": null,
      "zdr_note": "No branded ZDR commitment. Functionally reachable through disable_content_logging.",
      "air_gapped": true,
      "best_for": "Teams that want a fast Go gateway they can self-host for free and later buy clustering, SSO and guardrails from a single vendor.",
      "strengths": [
        "Go implementation with the lowest vendor-claimed proxy overhead in this set",
        "Generous OSS tier: semantic caching, MCP gateway, budgets, virtual keys and prompt repository at no cost",
        "Apache-2.0 license with VPC, on-prem and air-gapped deployment paths",
        "Ships with OpenTelemetry-native observability tied into Maxim's evaluation platform"
      ],
      "weaknesses": [
        "Headline latency and throughput figures are all vendor-run against mock upstreams; no independent benchmark found",
        "Much smaller community than LiteLLM, Kong or APISIX (7.6k vs 16.8k-57.5k stars)",
        "Guardrails, cluster mode, adaptive load balancing, SSO, RBAC and audit logs are enterprise-only with unpublished pricing",
        "Provider and model counts are inconsistent across the vendor's own pages"
      ],
      "avoid_if": "You need certified compliance artifacts, published enterprise pricing, or a large third-party operator community to lean on.",
      "reviews": [
        {
          "url": "https://kleinloog.ch/articles/the-3-llm-gateways-on-my-shortlist/",
          "why": "Arnoud Kleinloog explains why Bifrost (Maxim AI) made his personal shortlist over LiteLLM, Portkey, Helicone, BricksLLM and Martian, with deployment and maturity caveats.",
          "date": "2026-04-19",
          "type": "written",
          "title": "The 3 LLM Gateways on My Shortlist",
          "publisher": "kleinloog.ch"
        },
        {
          "url": "https://www.youtube.com/watch?v=xPdAOvvxtLs",
          "why": "Established Python-teaching channel runs Bifrost locally in Docker, configures OpenAI/Anthropic/Mistral/Ollama providers and wires it into Claude Code and an MCP server.",
          "date": "2026-02-09",
          "type": "video",
          "title": "Bifrost: High-Speed Open Source AI Gateway",
          "publisher": "YouTube - NeuralNine"
        },
        {
          "url": "https://www.youtube.com/watch?v=5hk5d_GKY_s",
          "why": "Independent tutorial that deploys Bifrost via Docker, adds providers including a custom OpenAI-compatible endpoint, then inspects its usage, cost and latency monitoring.",
          "date": "2026-01-29",
          "type": "video",
          "title": "Bifrost LLM Proxy Tutorial: Route and Monitor Requests Across Multiple Providers",
          "publisher": "YouTube - Ready Tensor"
        }
      ],
      "sources": {
        "cap_mcp": "https://www.getmaxim.ai/pricing",
        "company": "https://github.com/maximhq/bifrost",
        "license": "https://github.com/maximhq/bifrost",
        "sdk_note": "https://github.com/maximhq/bifrost",
        "cap_audio": "https://docs.getbifrost.ai/",
        "free_tier": "https://www.getmaxim.ai/pricing",
        "air_gapped": "https://www.getmaxim.ai/pricing",
        "hq_country": "https://www.cbinsights.com/company/maxim-ai",
        "zdr_status": "https://docs.getbifrost.ai/enterprise/log-exports",
        "cap_budgets": "https://www.getmaxim.ai/pricing",
        "model_count": "https://docs.getbifrost.ai/",
        "api_surfaces": "https://docs.getbifrost.ai/",
        "cap_failover": "https://www.getmaxim.ai/pricing",
        "founded_year": "https://www.cbinsights.com/company/maxim-ai",
        "github_stars": "https://github.com/maximhq/bifrost",
        "product_fate": "https://github.com/maximhq/bifrost/releases",
        "regions_note": "https://docs.getbifrost.ai/enterprise/log-exports",
        "cap_image_gen": "https://docs.getbifrost.ai/",
        "url_changelog": "https://github.com/maximhq/bifrost/releases",
        "vendor_status": "https://www.getmaxim.ai/blog/announcing-maxim-ais-general-availability-and-the-3m-funding-round-led-by-elevation-capital/",
        "byok_supported": "https://docs.getbifrost.ai/",
        "cap_embeddings": "https://docs.getbifrost.ai/",
        "cap_guardrails": "https://www.getmaxim.ai/pricing",
        "certifications": "https://www.getmaxim.ai/bifrost/docs/enterprise/governance",
        "config_as_code": "https://docs.getbifrost.ai/deployment-guides/config-json",
        "credit_fee_pct": "https://www.getmaxim.ai/pricing",
        "provider_count": "https://github.com/maximhq/bifrost",
        "retention_days": "https://docs.getbifrost.ai/enterprise/log-exports",
        "sla_uptime_pct": "https://www.getmaxim.ai/bifrost/resources/enterprise-deployment",
        "throughput_rps": "https://www.getmaxim.ai/bifrost",
        "bifrost_pricing": "https://www.getmaxim.ai/pricing",
        "cap_rate_limits": "https://www.getmaxim.ai/pricing",
        "logs_by_default": "https://docs.getbifrost.ai/enterprise/log-exports",
        "cap_cache_simple": "https://www.getmaxim.ai/pricing",
        "cap_virtual_keys": "https://www.getmaxim.ai/pricing",
        "model_count_note": "https://www.getmaxim.ai/bifrost",
        "token_markup_pct": "https://www.getmaxim.ai/pricing",
        "cap_observability": "https://www.getmaxim.ai/pricing",
        "last_release_date": "https://github.com/maximhq/bifrost/releases/tag/core%2Fv1.8.4",
        "openai_compatible": "https://github.com/maximhq/bifrost",
        "cap_cache_semantic": "https://www.getmaxim.ai/pricing",
        "cap_load_balancing": "https://www.getmaxim.ai/pricing",
        "seat_fee_usd_month": "https://www.getmaxim.ai/pricing",
        "guardrails_location": "https://docs.getbifrost.ai/enterprise/log-exports",
        "latency_overhead_ms": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
        "latency_source_note": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
        "requires_vendor_sdk": "https://www.getmaxim.ai/docs/bifrost/usage/http-transport/integrations/openai-compatible",
        "self_host_cost_note": "https://www.getmaxim.ai/pricing",
        "cap_prompt_management": "https://www.getmaxim.ai/pricing",
        "data_export_available": "https://docs.getbifrost.ai/api-reference/logging/get-logs",
        "prompt_transits_vendor": "https://www.getmaxim.ai/docs/self-hosting/overview",
        "cap_conditional_routing": "https://www.getmaxim.ai/pricing"
      },
      "verified_at": {
        "cap_mcp": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "free_tier": "2026-08-29",
        "air_gapped": "2026-08-29",
        "hq_country": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "github_stars": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "throughput_rps": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "cap_virtual_keys": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:34.758692+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed with several releases a week. Built by Maxim AI, an independent venture-backed company that raised a $3M seed led by Elevation Capital.",
      "last_release_date": "2026-08-27",
      "sla_uptime_pct": 99.999,
      "sla_url": "https://www.getmaxim.ai/bifrost/resources/enterprise-deployment",
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "A binary, Docker image or embedded Go library you run. Maxim also offers a private deployment where both data and control planes sit in your VPC, or a VPC-peered hybrid.",
      "logs_by_default": "full_content",
      "logs_note": "Every offloadable payload field is uploaded to object storage by default — your storage, but stored. Setting disable_content_logging skips content and does not persist reveal data.",
      "retention_days": 365,
      "retention_note": "Default 365 days, minimum one day, and zero disables cleanup entirely rather than disabling storage.",
      "zdr_status": "not_published",
      "trains_on_customer_data": "not_published",
      "training_note": "Nothing on the README, enterprise docs or governance page addresses training.",
      "subprocessors_url": null,
      "regions_note": "Your own infrastructure. Log offload targets S3 or GCS in a region you name. Maxim offers full VPC isolation or a VPC-peered hybrid.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type 2 for Maxim AI",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "reported compliant on the governance page",
          "status": "certified"
        },
        "iso27001": {
          "detail": "trust center says compliant, governance page says in progress",
          "status": "conflicting"
        },
        "hipaa_baa": {
          "detail": "trust center says compliant, governance page says not applicable",
          "status": "conflicting"
        }
      },
      "guardrails_location": "self_hosted",
      "guardrails_note": "Guardrail redaction happens in the Bifrost instance you run, and the reveal mapping stays with your own log row.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "oss_with_managed_tier",
      "pricing_model_note": "OSS 'Free Forever' (Docker/K8s/Go binary) for developers, small teams and self-managed deployments. Enterprise is 'Custom Pricing' for VPC/on-prem/air-gapped, quote-only.",
      "min_commitment": "None for OSS. Enterprise not stated.",
      "overage_note": "None stated. Self-hosting infrastructure cost is yours.",
      "cache_support": "hybrid",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Simple (exact) and semantic caching both in OSS. Semantic caching 'reduces costs and latency for similar queries with semantic similarity caching'. Bifrost prices no tokens, so caching saving is fully the customer's. MCP Code Mode claims to cut tokens by 50%+ separately.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "'Real-time notifications for budget limits, failures and performance issues' via email/Slack/PagerDuty/Teams/Webhook is Enterprise.",
          "tier": "enterprise",
          "supported": true
        },
        "virtual_keys": {
          "note": "Supported, free — 'control access permissions, budgets, rate limits, and routing per consumer'.",
          "tier": "oss",
          "supported": true
        },
        "rate_limiting": {
          "note": "Supported, free.",
          "tier": "oss",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Webhook notification channel is Enterprise.",
          "tier": "enterprise",
          "supported": true
        },
        "model_allowlists": {
          "note": "Not stated for models. MCP tool allow-lists are: 'strict allow-lists for which MCP tools are available per virtual key'.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Enforced through virtual keys.",
          "tier": "oss",
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Hierarchical cost control across virtual keys, teams and customers.",
          "tier": "oss",
          "supported": true
        }
      },
      "cost_attribution": "Per virtual key, consumer, team and customer. Per-user and per-tag not stated.",
      "cost_export": "OTel-compatible metrics and traces (any OTel backend can receive data). CSV, cost API, webhook-of-spend, S3 and warehouse export not stated.",
      "product_kind_note": "Maxim calls Bifrost \"a high-performance AI gateway unifying 20+ providers through a single OpenAI-compatible API\", with \"automatic failover, load balancing, semantic caching, and enterprise-grade governance\" ([Bifrost overview](https://docs.getbifrost.ai/overview)), and on GitHub \"the fastest enterprise AI gateway (50x faster than LiteLLM)\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). It is a self-hosted routing proxy (also embeddable as a Go SDK).",
      "deployment_modes": [
        "self_host",
        "hybrid_vpc",
        "on_prem",
        "air_gapped"
      ],
      "deployment_note": "`npx -y @maximhq/bifrost` or `docker run -p 8080:8080 maximhq/bifrost` (versioned tags such as `maximhq/bifrost:v1.3.9`, amd64/arm64), with a mounted data volume holding `config.json`, `config.db` and `logs.db`; config/log stores can be SQLite or PostgreSQL 16+ (UTF8) ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)). Embedded use is `go get github.com/maximhq/bifrost/core` ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). Tier gating: OSS self-managed is \"Free Forever\"; VPC/on-prem/air-gapped, clustering, guardrails and private networking are Enterprise \"Custom Pricing\" ([Maxim pricing](https://www.getmaxim.ai/pricing); [maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap is the primary pattern: \"Replace existing AI SDK connections by changing just the base URL\" with per-SDK prefixes `.../openai`, `.../anthropic`, `.../genai`, plus native AWS Bedrock SDK integration and LiteLLM/LangChain/PydanticAI compatibility ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27; [Bifrost overview](https://docs.getbifrost.ai/overview)). No proprietary SDK is required, though a Go SDK exists for embedded use ([maximhq/bifrost](https://github.com/maximhq/bifrost)). Models can be addressed as `openai/gpt-4o-mini` or bare `gpt-4o-mini` resolved through the Model Catalog ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).",
      "model_count_low": null,
      "model_count_high": null,
      "model_count_source_note": "n.a.; no total is stated, only examples (GPT-4o, o1, Claude 4/3.5/3, Gemini, Grok) ([Bifrost overview](https://docs.getbifrost.ai/overview), undated).",
      "provider_count_low": 20,
      "provider_count_high": 23,
      "provider_count_source_note": "Vendor numbers differ: \"20+ providers\" ([Bifrost overview](https://docs.getbifrost.ai/overview), undated) vs \"23+ providers\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). The overview page enumerates 22 named providers (OpenAI, Anthropic, AWS Bedrock, Google Vertex AI, Azure OpenAI, Google Gemini, Groq, Mistral, Cohere, Cerebras, DeepSeek, Ollama, Hugging Face, OpenRouter, Perplexity, ElevenLabs, Nebius, xAI, Parasail, Replicate, SGL, vLLM) ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "first_party_vs_partner": "All third-party or customer-run: Bifrost owns no models and routes to provider APIs plus self-hosted engines such as vLLM (\"self-hosted OpenAI-compatible inference\") and Ollama (\"local inference with OpenAI-compatible format\") ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "custom_endpoint_support": "Yes, indirectly: vLLM and Ollama are first-class providers for self-hosted/local inference ([Bifrost overview](https://docs.getbifrost.ai/overview)). Explicit registration of an arbitrary private URL: n.a. ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "byok_mode": "byok_only",
      "merchant_of_record": "n.a. as an explicit statement; the pages mention \"cost control\" but do not specify billing, invoicing, credits or payment responsibility ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)), and Maxim invoices only Enterprise \"Custom Pricing\" ([Maxim pricing](https://www.getmaxim.ai/pricing)).",
      "key_management_note": "Keys are held in the customer's own Config Store (SQLite or PostgreSQL) alongside provider configs and MCP settings, typically referenced from environment variables; Bifrost adds \"intelligent API key distribution with weighted load balancing, model-specific filtering, and automatic failover\" and virtual keys as \"the primary governance entity\" controlling access, budgets, rate limits and routing per consumer, hierarchically at virtual-key, team and customer levels ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Bifrost overview](https://docs.getbifrost.ai/overview)). Because deployment is customer-run, request logs stay in the customer's own logs store ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)); explicit vendor-visibility statements: n.a.",
      "api_openai_chat_completions_note": "Yes: `POST http://localhost:8080/v1/chat/completions`, working \"with any provider (OpenAI, Anthropic, Bedrock, etc.)\", and all Bifrost APIs \"follow OpenAI request/response format\" ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).",
      "api_anthropic_messages_note": "Yes, via SDK-compatible base-URL replacement: point the Anthropic SDK at `http://localhost:8080/anthropic` as a \"drop-in replacement for the Anthropic Python and TypeScript SDKs\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27; [Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "api_openai_responses_note": "n.a. (not documented on the pages fetched).",
      "api_embeddings_note": "Yes: embeddings are documented for providers including Google Gemini, Cohere, Nebius, SGL and vLLM (\"chat, embeddings, and STT\") ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "api_images_note": "Yes at the interface level: \"support for text, images, audio, and streaming, all behind a common interface\" ([maximhq/bifrost](https://github.com/maximhq/bifrost), 2026-08-27). Dedicated image endpoint path: n.a.",
      "api_audio_note": "Yes, both directions: Hugging Face \"TTS, and STT\", ElevenLabs \"text-to-speech and speech-to-text models\", vLLM \"chat, embeddings, and STT\" ([Bifrost overview](https://docs.getbifrost.ai/overview)). Endpoint paths: n.a.",
      "api_batch_note": "n.a.: no general batch or asynchronous API is described; only Replicate's \"prediction-based architecture with async modes\" ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "deployment_modes_note": "self-host via NPX binary, Docker, Kubernetes or Go binary ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up); [Maxim pricing](https://www.getmaxim.ai/pricing): \"SELF-HOSTED: DOCKER | K8S | GO BINARY\"); hybrid/VPC, on-prem and air-gapped on the Enterprise tier — \"ENTERPRISE READY: VPC | ON-PREM | AIR-GAPPED\" ([Maxim pricing](https://www.getmaxim.ai/pricing)), with \"In-VPC Deployments… Deploy within your private cloud infrastructure with VPC isolation and enhanced security controls\" ([Bifrost overview](https://docs.getbifrost.ai/overview)). Hosted SaaS: n.a. — the docs mention only \"a hosted or production deployment\" as customer-run guidance ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)).",
      "byok_mode_note": "byok_only: providers are configured with the customer's own keys, e.g. `\"value\": \"env.OPENAI_API_KEY\"`, \"Add API keys with clicks, not code\" ([Bifrost setup](https://docs.getbifrost.ai/quickstart/gateway/setting-up)); the OSS gateway is \"Free Forever\" and no model credits are sold ([Maxim pricing](https://www.getmaxim.ai/pricing)).",
      "access_meta": {
        "qualified": [
          "api_anthropic_messages",
          "api_images",
          "api_audio"
        ],
        "model_count_kind": null,
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "config_file",
      "rel_timeout_note": "A `timeout` config parameter is documented, but no default value is published: default `n.a.` ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "rel_retry_config": "config_file",
      "rel_retry_note": "A `retry` config parameter is documented; default retry count and backoff strategy are not published: `n.a.` ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "Fallback across providers/models is documented; WEIGHTED selection applies to API keys (weighted key selection, cited at ~10 ns selection cost) rather than to the fallback chain, and the docs do not state a weighted fallback split ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "rel_fallback_shape": "weighted",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "`not_documented` as an upstream provider health check or circuit breaker. What is documented is Cluster Mode: peer-to-peer HA failover between Bifrost nodes — i.e. gateway-level HA, not provider ejection ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "Cluster Mode covers node failover; no cross-region provider failover configuration is documented ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "`config_file`, with weights on API keys (weighted key selection) ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "architecture_class": "compiled_binary",
      "architecture_note": "Go, using goroutines and `fasthttp`, distributed as an ~80MB binary ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "Docker image `maximhq/bifrost`, `npx -y @maximhq/bifrost`, a `helm-charts` folder, and a Go SDK via `go get github.com/maximhq/bifrost/core` ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
      "latency_claims": [
        {
          "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "0.99",
          "metric": "median added",
          "publisher": "vendor",
          "conditions": "Derived: median end-to-end 60.99ms against a mock OpenAI holding a fixed 60ms latency."
        },
        {
          "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
          "unit": "ms",
          "scope": "end_to_end",
          "value": "804",
          "metric": "p50",
          "publisher": "vendor",
          "conditions": "500 RPS, AWS EC2 t3.medium (2 vCPU / 4 GB), us-east-1, 60s, 500 virtual users, real OpenAI Tier 5."
        },
        {
          "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
          "unit": "ms",
          "scope": "end_to_end",
          "value": "1680",
          "metric": "p99",
          "publisher": "vendor",
          "conditions": "Same 500 RPS t3.medium run; max observed 6.13s."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://www.getmaxim.ai/bifrost/resources/benchmarks",
          "unit": "req/s",
          "scope": "capacity",
          "value": "424",
          "metric": "sustained",
          "publisher": "vendor",
          "conditions": "500 RPS comparison test on t3.medium against real OpenAI. A separate 5,000 RPS stress run is Bifrost-only against a mocked provider with ~10KB payloads."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported: the docs state support for \"text, images, audio, and streaming\"; no streaming caveat is stated on the fetched pages ([Bifrost overview](https://docs.getbifrost.ai/overview)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Providers include in-process regex (RE2 patterns with `entity_type`), Gitleaks-based secrets detection, Microsoft Presidio, and Azure AI Language PII. Each supports `action: detect_only | block | redact`, `redaction_strategy: replace | mask | hash`, and `redaction_mode: runtime | logs_only | runtime_reversible`. Azure PII defaults to `detect_only` ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails), [Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Azure Content Safety jailbreak shield, Model Armor, CrowdStrike AIDR, Gray Swan, Patronus, Lakera, Repello Argus are configurable as guardrail providers ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))",
      "guard_content": "sync_block",
      "guard_content_note": "Bedrock Guardrails and Azure Content Safety are supported providers for harmful-content categories ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Custom policy form is RE2 regex patterns with per-pattern flags, entity types, and actions, plus `ignored_secret_keywords` suppression lists on the secrets provider ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Virtual keys carry `allowed_models` per provider ([Budgets and limits](https://docs.getbifrost.ai/features/governance/budget-and-limits))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "Microsoft Presidio",
        "Azure AI Language",
        "AWS Bedrock Guardrails",
        "Google Model Armor",
        "Gitleaks"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` — the fetched guardrail pages document action outcomes but no explicit timeout/fail-open or fail-closed rule ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails))",
      "log_content": "configurable",
      "log_content_note": "Bifrost records metadata plus content: \"actual prompts, completions, and tool traffic\", including chat history, tool definitions and arguments, reasoning traces, embedding inputs, and verbatim provider bodies when `x-bf-store-raw-request-response` is used ([Content logging](https://docs.getbifrost.ai/features/observability/content-logging))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`client.disable_content_logging: true` reduces log rows to metadata only; per-request override header `x-bf-disable-content-logging` works only when `client.allow_per_request_content_storage_override` is enabled; each observability connector has its own independent `disable_content_logging` defaulting to `false`; metrics never carry content ([Content logging](https://docs.getbifrost.ai/features/observability/content-logging))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry natively, with `disable_root_span_content` and `disable_content_logging` controls over span content ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))",
      "export_destinations": [
        "OpenTelemetry",
        "Prometheus",
        "Datadog",
        "Langfuse",
        "Maxim",
        "S3",
        "Webhook"
      ],
      "export_destinations_note": "OTel to Grafana Cloud, Datadog, New Relic, Honeycomb, Langfuse, and self-hosted collectors ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel)); log payload offload to Amazon S3 and Google Cloud Storage only — \"Azure Blob, local filesystem, and data warehouse destinations are not implemented\" — with the logs DB on SQLite, Postgres, or ClickHouse and a BigQuery connector gated by `disable_content_logging` ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports), [Content logging](https://docs.getbifrost.ai/features/observability/content-logging))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — no feedback endpoint documented on the fetched pages ([Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` from the fetched Bifrost pages ([Enterprise guardrails](https://docs.getbifrost.ai/enterprise/guardrails))",
      "observability_requires_saas": "No — logs live in the customer's own SQLite/Postgres/ClickHouse and object storage, and telemetry exports to the customer's collectors ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports), [Bifrost OTel](https://docs.getbifrost.ai/features/observability/otel))",
      "log_retention_note": "`client_config.log_retention_days`, default **365 days**, minimum 1 day, `0` disables cleanup ([Log exports](https://docs.getbifrost.ai/enterprise/log-exports))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "The most detailed published set in this report, all vendor-run: - 500 RPS test on AWS EC2 `t3.medium` (2 vCPU / 4 GB, `us-east-1`, 60s, 500 VUs, OpenAI Tier 5): Bifrost **P50 804 ms**, **P99 1.68 s**, **max 6.13 s** (LiteLLM comparison figures noted in field 11) ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - Median end-to-end **60.99 ms** against a mock OpenAI with 60 ms fixed latency, implying **gateway overhead 0.99 ms** ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - 5,000 RPS Bifrost-only stress test (~10KB payloads, mocked OpenAI): **gateway overhead 59 µs** on `t3.medium` and **11 µs** on `t3.xlarge` (4 vCPU / 16 GB), queue wait **47.13 µs** / **1.67 µs**, average latency including provider **2.12 s** / **1.61 s**, 100% success, peak memory 1312.79 MB / 3340.44 MB ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). - Headline claim \"**<15µs added latency per request**\", conditions not stated in the claim itself ([maximhq/bifrost](https://github.com/maximhq/bifrost)).",
        "multi_surface": {},
        "throughput_prose": "**424 req/s** sustained in the 500 RPS comparison test on `t3.medium` ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)); the 5,000 RPS stress run is Bifrost-only against a mocked provider with ~10KB payloads ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). Reproduction instructions are published ([Bifrost benchmarking guide](https://docs.getbifrost.ai/benchmarking/getting-started)).",
        "benchmark_independence": "Vendor-published, and **explicitly flagged: the same benchmark post publishes figures about a COMPETITOR, LiteLLM** — LiteLLM P50 38.65 s, P99 90.72 s, max 92.67 s, median end-to-end 100 ms against the 60 ms mock (≈40 ms overhead), 44.84 req/s ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)). Those rival numbers are not verifiable from LiteLLM's own docs and should not be used; LiteLLM's own published figure is `8ms P95 latency at 1k RPS` ([BerriAI/litellm](https://github.com/BerriAI/litellm)). No independent third-party benchmark was found."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "choose_at_setup",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Bifrost is the only product in this set that publishes hardware, VU count, duration, payload size and mock-provider baseline, which makes its own numbers auditable even where its competitor numbers are not ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)).",
      "guard_notes": "Bifrost has the most nuanced redaction model here: `logs_only` redacts what is stored while the model still sees the original, `runtime` redacts what the model sees, and `runtime_reversible` stores a reveal mapping that stays on the Bifrost log row and is never exported ([Guardrails config](https://docs.getbifrost.ai/deployment-guides/config-json/guardrails), [Log exports](https://docs.getbifrost.ai/enterprise/log-exports)).",
      "fit_agent_features_note": "Bifrost returns LLM tool calls as suggestions requiring a separate execution API call, while Agent Mode supports autonomous multi-step tool execution with configurable auto-approval. Code Mode lets the model use Python to orchestrate multiple tools in a sandbox. ([Bifrost MCP gateway](https://www.getmaxim.ai/bifrost/resources/mcp-gateway))",
      "fit_byok_required": "optional",
      "fit_byok_required_note": "Not explicitly stated; the displayed first-call snippets contain no provider key, but provider setup \"supports adding API keys through the UI\" and the last Helm step is \"Configure Providers & Plugins\" ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_client_sdk_langs": [
        "Go"
      ],
      "fit_client_sdks_note": "An official Go SDK is documented with `go get github.com/maximhq/bifrost/core`. Documented integrations also include the OpenAI, Anthropic, AWS Bedrock, Google GenAI, LiteLLM, and LangChain SDKs; the evidence does not provide a complete list of official Bifrost client-library languages. ([Bifrost GitHub repository](https://github.com/maximhq/bifrost))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Bifrost documents cloud-native authentication through AWS IRSA, GCP Workload Identity, and Azure Azure WIF, and describes this as native IAM integration. Specific AWS SigV4, Azure managed-identity, and GCP service-account authentication are not documented. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented) on either page ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_ecosystem_notes": "Bifrost is open source and supports self-hosted deployment in private VPCs, on-premises or bare-metal environments, air-gapped environments, and single-node or edge deployments. Documented deployment options include AWS, GCP, Azure, generic Kubernetes, Docker Compose, and a single Go binary; air-gapped operation is described as fully offline with no phone-home or telemetry. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))",
      "fit_eval_prereqs": "None stated: no account, credit card, cloud account, cluster, or provider API key is listed as a prerequisite; provider keys are added later through the Web UI or API ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))",
      "fit_eval_shape": "cli_or_container",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `curl -X POST http://localhost:8080/v1/chat/completions -H \"Content-Type: application/json\" -d '{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello, Bifrost!\"}]}'` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "bifrost/bifrost from https://maximhq.github.io/bifrost/helm-charts",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "Bifrost documents the official Helm chart `bifrost/bifrost` from the repository `https://maximhq.github.io/bifrost/helm-charts`. ([Bifrost Helm deployment guide](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "Bifrost acts as both an MCP client and MCP server, connecting to external MCP servers over STDIO, HTTP, and SSE, auto-discovering tools, and exposing them through a single gateway URL. It supports tool filtering, virtual MCP servers, MCP Tool Groups, explicit tool execution, and Tool Hosting. ([Bifrost MCP gateway](https://www.getmaxim.ai/bifrost/resources/mcp-gateway))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_credit_card",
          "fit_gateway_platform",
          "fit_model_gate"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or waitlist wording ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_onboarding_notes": "Unversioned `npx`/`docker` commands are explicitly \"for local evaluation,\" so the evaluate and production paths diverge deliberately; configuration is via Web UI or API; verification commands include `kubectl port-forward svc/bifrost 8080:8080`, `curl http://localhost:8080/health` and `/metrics` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Existing OpenAI SDK applications require zero code changes beyond setting `base_url` to `\"http://localhost:8080/openai\"`; the Python and JavaScript examples retain an API key such as `\"your-openai-key\"` or `process.env.OPENAI_API_KEY`. ([Bifrost OpenAI-compatible integration](https://www.getmaxim.ai/docs/bifrost/usage/http-transport/integrations/openai-compatible))",
      "fit_prod_prereqs": "Kubernetes v1.19+, configured `kubectl`, Helm 3.2.0+, a Persistent Volume provisioner (required for SQLite, optional for Postgres-only), UTF8-encoded PostgreSQL storage, a required `image.tag`, and private registry credentials for Enterprise. The quickstart also says production should \"Pin an image version, enable authentication and TLS, and choose durable storage\" ([Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm), [Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "infra_rollout",
      "fit_python_frameworks": [
        "LangChain"
      ],
      "fit_python_frameworks_note": "LangChain integration is documented, including `langchain_openai.ChatOpenAI`, `langchain_anthropic.ChatAnthropic`, and `langchain_google_genai.ChatGoogleGenerativeAI`. LlamaIndex is not documented. ([Bifrost LangChain integration](https://docs.getbifrost.ai/integrations/langchain-sdk))",
      "fit_quickstart_steps": 4,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.getbifrost.ai/quickstart/gateway/setting-up",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`npx -y @maximhq/bifrost`; `docker pull maximhq/bifrost` + `docker run -p 8080:8080 maximhq/bifrost`; Helm: `helm repo add bifrost https://maximhq.github.io/bifrost/helm-charts`, `helm repo update`, `kubectl create secret generic bifrost-encryption-key --from-literal=encryption-key=\"$(openssl rand -base64 32)\"`, `helm install bifrost bifrost/bifrost --set image.tag=v1.4.11 ...`, production `helm install bifrost bifrost/bifrost -f production.yaml` ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up), [Quick Start - Bifrost Helm](https://docs.getbifrost.ai/deployment-guides/helm))",
      "fit_terraform": "official_modules",
      "fit_terraform_note": "A Terraform module source is documented as `github.com/maximhq/bifrost//terraform/modules/bifrost?ref=terraform/v0.1.0`. A Terraform Registry provider or module name is not documented. ([Bifrost enterprise deployment](https://www.getmaxim.ai/bifrost/resources/enterprise-deployment))",
      "fit_terraform_ref": "github.com/maximhq/bifrost//terraform/modules/bifrost?ref=terraform/v0.1.0; not on the Registry",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get Bifrost running as an HTTP API gateway in 30 seconds with zero configuration.\" ([Setting Up - Bifrost](https://docs.getbifrost.ai/quickstart/gateway/setting-up))",
      "fit_vercel_ai_sdk": "compatible_no_package",
      "fit_vercel_ai_sdk_note": "Bifrost is stated to be compatible with the Vercel AI SDK. No official provider package name is documented. ([Maxim AI homepage](https://www.getmaxim.ai/))",
      "fit_vercel_ai_sdk_package": null
    },
    {
      "slug": "litellm",
      "name": "LiteLLM",
      "company": "BerriAI",
      "tagline": "Open-source AI gateway and Python SDK that puts one OpenAI-compatible API in front of many LLM providers.",
      "category": "open_source",
      "deployment": "self_host_only",
      "license": "MIT",
      "founded_year": 2023,
      "hq_country": "United States",
      "url_home": "https://www.litellm.ai/",
      "url_docs": "https://docs.litellm.ai/docs/simple_proxy",
      "url_pricing": "https://www.litellm.ai/#pricing",
      "url_status": null,
      "url_github": "https://github.com/BerriAI/litellm",
      "url_changelog": "https://docs.litellm.ai/release_notes",
      "url_models_api": null,
      "github_stars": 57500,
      "model_count": 1892,
      "model_count_note": "Vendor-stated count of unique models reachable through the gateway; the GitHub repo more conservatively says 100+ models across 140+ providers.",
      "provider_count": 140,
      "token_markup_pct": 0,
      "credit_fee_pct": 0,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "Self-hosted only: you configure your own upstream provider keys and pay providers directly; no gateway token fee.",
      "seat_fee_usd_month": 0,
      "free_tier": "Open-source gateway is $0 forever: 140+ providers, virtual keys, teams, budgets, load balancing, RPM/TPM limits, guardrails.",
      "enterprise_from_usd_year": null,
      "extra_fees": [],
      "self_host_cost_note": "OSS is free (MIT); you pay only for your own containers plus Postgres and Redis. Enterprise features (SSO, RBAC, JWT auth, SCIM, audit logs, support SLAs) require a paid LiteLLM commercial license whose price is not published.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenAI Responses",
        "OpenAI Embeddings",
        "OpenAI Images",
        "OpenAI Audio",
        "OpenAI Batches",
        "Rerank",
        "Anthropic Messages"
      ],
      "sdk_note": "Python SDK plus a self-hosted proxy server; any OpenAI-compatible client works by swapping the base URL.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": true,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": true,
      "latency_overhead_ms": 0.66,
      "latency_source_note": "Vendor-run: LiteLLM's own \"AI Gateway Bench\" reports 0.66 ms added p99 latency and 2,800+ RPS for its Rust gateway (~22 MB RSS), and in the same self-run test puts Portkey at 2.29 ms and Bifrost at 4.54 ms. The GitHub README separately cites 8 ms p95 at 1k RPS. Third-party write-ups report the Python proxy degrading well before that (see weaknesses).",
      "throughput_rps": 2800,
      "soc2": null,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": "Self-hosted, so data residency is wherever you deploy; the vendor states self-hosted deployments send no telemetry back.",
      "zdr": null,
      "zdr_note": "Nothing transits BerriAI when you self-host, and content logging is off by default, so there is no vendor retention to switch off.",
      "air_gapped": true,
      "best_for": "Platform teams that want a free, self-hosted, maximally broad provider abstraction with virtual keys and budgets, at moderate request volumes.",
      "strengths": [
        "Widest provider and model coverage of any gateway here (140+ providers)",
        "MIT-licensed with no usage fees, plus Helm/Terraform/Docker install paths and air-gapped support",
        "Virtual keys, team budgets, RPM/TPM limits and guardrails in the free tier",
        "Response and semantic caching backed by Redis, S3 or GCS",
        "Very broad API surface: chat, responses, embeddings, images, audio, batches, rerank, MCP"
      ],
      "weaknesses": [
        "Third-party operators report the Python proxy hitting a GIL bottleneck past ~300 RPS per instance with P99 blowing out to seconds and memory growth to OOM",
        "Independent 2026 comparison recommends staying under 250-300 RPS per instance and disputes vendor cache/MCP feature claims",
        "Vendor benchmark numbers are self-run and conflict sharply with rivals' self-run numbers (Kong measured LiteLLM 859% slower than Kong)",
        "SSO, RBAC, audit logs and SCIM sit behind an unpublished-price commercial license"
      ],
      "avoid_if": "You need a single instance to sustain thousands of RPS with tight tail latency, or you need certified compliance artifacts from a vendor rather than your own deployment.",
      "reviews": [
        {
          "url": "https://dev.to/polar3130/using-gemini-cli-through-litellm-proxy-1627",
          "why": "Hands-on writeup of putting Gemini CLI behind LiteLLM Proxy on Cloud Run with Cloud SQL, including the operational gotchas found in testing.",
          "date": "2025-11-25",
          "type": "written",
          "title": "Using Gemini CLI Through LiteLLM Proxy",
          "publisher": "DEV Community - polar3130"
        },
        {
          "url": "https://www.youtube.com/watch?v=RWDJKIya-WI",
          "why": "Conference talk by Alina Dallmann on a production LiteLLM deployment for a privacy-preserving university chatbot: proxy setup, Postgres budgets, Redis load balancing, key management.",
          "date": "2025-11-23",
          "type": "video",
          "title": "One API to Rule Them All? LiteLLM in Production",
          "publisher": "YouTube - PyData"
        },
        {
          "url": "https://www.youtube.com/watch?v=WyW4Ifu4rSo",
          "why": "Sudarshan Koirala's hands-on course covering the SDK, proxy server, Docker deploy, admin UI, teams and virtual keys against OpenAI, Ollama and Groq.",
          "date": "2025-06-02",
          "type": "video",
          "title": "LiteLLM Crash Course | For Complete Beginners",
          "publisher": "YouTube - Data Science Basics"
        },
        {
          "url": "https://news.ycombinator.com/item?id=47501426",
          "why": "Supply-chain incident thread with practitioner impact reports and the maintainers' response, including which deployment paths (pinned Docker image) were unaffected.",
          "date": "2026-03-24",
          "type": "community",
          "title": "Tell HN: Litellm 1.82.7 and 1.82.8 on PyPI are compromised",
          "publisher": "Hacker News"
        },
        {
          "url": "https://www.reddit.com/r/LocalLLaMA/comments/1mh99hu/litellm_started_breaking_down_for_us_past_300_rps/",
          "why": "32-comment scaling thread with concrete tuning data: failures past ~300 RPS and an 84.7% P99 reduction (4.9s to 750ms) on 4 CPUs / 4 instances without caching.",
          "date": "2025-08-04",
          "type": "community",
          "title": "LiteLLM started breaking down for us past 300 RPS, what are folks using in prod?",
          "publisher": "Reddit - r/LocalLLaMA"
        }
      ],
      "sources": {
        "cap_mcp": "https://github.com/BerriAI/litellm",
        "company": "https://www.ycombinator.com/companies/litellm",
        "license": "https://github.com/BerriAI/litellm",
        "sdk_note": "https://github.com/BerriAI/litellm",
        "url_docs": "https://docs.litellm.ai/docs/simple_proxy",
        "cap_audio": "https://github.com/BerriAI/litellm",
        "cap_batch": "https://github.com/BerriAI/litellm",
        "free_tier": "https://www.litellm.ai/#pricing",
        "air_gapped": "https://www.litellm.ai/#pricing",
        "hq_country": "https://www.ycombinator.com/companies/litellm",
        "weaknesses": "https://dev.to/debmckinney/litellm-broke-at-300-rps-in-production-heres-how-we-fixed-it-5ej",
        "cap_budgets": "https://www.litellm.ai/#pricing",
        "model_count": "https://www.litellm.ai/",
        "api_surfaces": "https://github.com/BerriAI/litellm",
        "cap_failover": "https://www.litellm.ai/#pricing",
        "founded_year": "https://www.ycombinator.com/companies/litellm",
        "github_stars": "https://github.com/BerriAI/litellm",
        "product_fate": "https://www.litellm.ai/",
        "regions_note": "https://www.litellm.ai/enterprise",
        "cap_image_gen": "https://github.com/BerriAI/litellm",
        "url_changelog": "https://docs.litellm.ai/release_notes",
        "vendor_status": "https://www.ycombinator.com/companies/litellm",
        "byok_supported": "https://www.litellm.ai/#pricing",
        "cap_embeddings": "https://github.com/BerriAI/litellm",
        "cap_guardrails": "https://www.litellm.ai/#pricing",
        "certifications": "https://docs.litellm.ai/docs/data_security",
        "config_as_code": "https://docs.litellm.ai/docs/proxy/configs",
        "credit_fee_pct": "https://www.litellm.ai/#pricing",
        "provider_count": "https://www.litellm.ai/",
        "retention_days": "https://docs.litellm.ai/docs/proxy/ui_spend_log_settings",
        "throughput_rps": "https://www.litellm.ai/",
        "cap_rate_limits": "https://www.litellm.ai/#pricing",
        "litellm_caching": "https://docs.litellm.ai/docs/proxy/caching",
        "litellm_pricing": "https://www.litellm.ai/pricing",
        "logs_by_default": "https://docs.litellm.ai/docs/proxy/ui_logs",
        "cap_cache_simple": "https://docs.litellm.ai/docs/proxy/prod",
        "cap_virtual_keys": "https://www.litellm.ai/#pricing",
        "model_count_note": "https://github.com/BerriAI/litellm",
        "token_markup_pct": "https://www.litellm.ai/#pricing",
        "cap_observability": "https://github.com/BerriAI/litellm",
        "cap_pii_redaction": "https://www.litellm.ai/#pricing",
        "eu_residency_note": "https://www.litellm.ai/#pricing",
        "last_release_date": "https://github.com/BerriAI/litellm/releases/tag/v1.98.0",
        "openai_compatible": "https://docs.litellm.ai/docs/simple_proxy",
        "cap_cache_semantic": "https://docs.litellm.ai/docs/proxy/prod",
        "cap_load_balancing": "https://www.litellm.ai/#pricing",
        "litellm_enterprise": "https://www.litellm.ai/enterprise",
        "seat_fee_usd_month": "https://www.litellm.ai/#pricing",
        "security_incidents": "https://docs.litellm.ai/blog/security-update-march-2026",
        "guardrails_location": "https://docs.litellm.ai/docs/proxy/guardrails/pii_masking_v2",
        "latency_overhead_ms": "https://www.litellm.ai/",
        "latency_source_note": "https://www.litellm.ai/",
        "requires_vendor_sdk": "https://docs.litellm.ai/docs/proxy/user_keys",
        "self_host_cost_note": "https://www.litellm.ai/#pricing",
        "litellm_virtual_keys": "https://docs.litellm.ai/docs/proxy/virtual_keys",
        "data_export_available": "https://docs.litellm.ai/docs/proxy/cost_tracking",
        "litellm_cost_tracking": "https://docs.litellm.ai/docs/proxy/cost_tracking",
        "prompt_transits_vendor": "https://docs.litellm.ai/docs/data_security",
        "cap_conditional_routing": "https://www.litellm.ai/",
        "trains_on_customer_data": "https://docs.litellm.ai/docs/data_security"
      },
      "verified_at": {
        "cap_mcp": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_batch": "2026-08-29",
        "free_tier": "2026-08-29",
        "air_gapped": "2026-08-29",
        "hq_country": "2026-08-29",
        "weaknesses": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "github_stars": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "throughput_rps": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "cap_virtual_keys": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:42.049547+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed, MIT-licensed core maintained by BerriAI with a paid enterprise tier layered on the same code rather than a separate build.",
      "last_release_date": "2026-08-23",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": "Latest stable release v1.98.0. A v1.100.0 release candidate was tagged on 30 August 2026; release candidates are excluded here.",
      "prompt_transits_vendor": "no",
      "prompt_path_note": "Self-hosted: no personal data is collected or transmitted to LiteLLM's servers, and enterprise supports air-gapped deployment so neither your data nor your keys leave your infrastructure.",
      "logs_by_default": "metadata_only",
      "logs_note": "Content logging is off by default — only metadata such as tokens, cost and model is stored. Storing request and response bodies requires opting in with store_prompts_in_spend_logs.",
      "retention_days": null,
      "retention_note": "No default deletion. Logs persist according to your configuration until you set something like maximum_spend_logs_retention_period.",
      "zdr_status": "not_applicable",
      "trains_on_customer_data": "no",
      "training_note": "True by construction rather than by policy: no data or telemetry reaches LiteLLM's servers when self-hosted. No separate training policy is published.",
      "subprocessors_url": null,
      "regions_note": "Your own infrastructure, including air-gapped. Enterprise offers a multi-region control plane.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "docs say Type II in progress with an ETA of 15 Sep 2026; the enterprise page markets it as done",
          "status": "conflicting"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "status": "not_published"
        },
        "iso27001": {
          "detail": "marketed on the enterprise page, unverified",
          "status": "claimed"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "self_hosted",
      "guardrails_note": "PII masking requires Presidio analyzer and anonymizer containers that you deploy and address locally, so redaction happens entirely inside your boundary.",
      "acquired_date_basis": null,
      "security_incidents": [
        {
          "cve": "CVE-2026-33634",
          "date": "2026-03-24",
          "what": "Two releases, 1.82.7 and 1.82.8, were published straight to PyPI carrying a credential stealer. Neither version was ever tagged on GitHub — the attacker bypassed the project's CI/CD entirely and uploaded to PyPI directly, using an API token exposed through the compromised Trivy releases in the wider TeamPCP campaign.",
          "title": "Malicious releases published to PyPI (CVE-2026-33634)",
          "action": "If you installed LiteLLM from PyPI inside the 24 March window, treat every credential reachable from that host as compromised and rotate it — model keys, cloud credentials, SSH keys and CI tokens. Check for litellm_init.pth in site-packages, a sysmon.service user unit, and outbound traffic to models.litellm[.]cloud or checkmarx[.]zone.",
          "impact": "The payload collected environment variables, model API keys, SSH keys, AWS, GCP and Azure credentials, Kubernetes tokens, database passwords, shell history and CI/CD secrets, encrypted them with AES-256 under an RSA-4096 key, and posted them to models.litellm[.]cloud — a domain registered the day before the upload and unconnected to the project. It then polled checkmarx[.]zone for follow-on code and persisted via a systemd user unit named sysmon.service.",
          "status": "resolved",
          "sources": {
            "GitHub issue #24518": "https://github.com/BerriAI/litellm/issues/24518",
            "PyPI incident report": "https://blog.pypi.org/posts/2026-04-02-incident-report-litellm-telnyx-supply-chain-attack/",
            "LiteLLM security advisory": "https://docs.litellm.ai/blog/security-update-march-2026",
            "The Hacker News follow-up": "https://thehackernews.com/2026/08/malicious-litellm-releases-tied-to.html",
            "Datadog Security Labs analysis": "https://securitylabs.datadoghq.com/articles/litellm-compromised-pypi-teampcp-supply-chain-campaign/"
          },
          "severity": "critical",
          "assessment": "This was a compromise of the distribution channel, not a flaw in the gateway's own code, and the maintainers did not write the malware. It is recorded here because the failure mode it exposes is structural: a self-hosted gateway installed from a public package index inherits that index's supply-chain risk, and this gateway is one that sits directly on your model keys and cloud credentials. Pin versions and verify hashes rather than tracking latest.",
          "resolution": "PyPI quarantined the project and both versions were removed; they no longer resolve. 1.82.6 and 1.83.0 are unaffected and remain available.",
          "scale_note": "Package-level download counts were never published. CloudSEK put the campaign's captured dataset at roughly 434,000 files, mapping potential exposure to more than 2,500 organisations — that is the whole TeamPCP campaign, not LiteLLM alone, and the file count is not a deduplicated count of victims.",
          "window_note": "How long the packages were installable is disputed by the parties involved. LiteLLM says they were live from 10:39 UTC and quarantined after roughly 40 minutes, while PyPI's own incident report puts LiteLLM's exposure at 2 hours 32 minutes from upload to quarantine, and the GitHub issue tracking it records the upload at around 08:30 UTC. LiteLLM's own advice is the safest reading: treat any install between 10:39 and 16:00 UTC on 24 March 2026 as suspect.",
          "worse_version": "1.82.8 is the more dangerous of the two. It shipped a litellm_init.pth file, which Python executes at interpreter startup, so the stealer ran on any Python process on the machine whether or not anything imported LiteLLM. In 1.82.7 the payload sat in proxy_server.py and only ran when litellm.proxy was imported."
        }
      ],
      "pricing_model": "oss_with_managed_tier",
      "pricing_model_note": "OSS $0 free forever, self-hosted. Enterprise is talk-to-sales, sized to annual gateway request capacity, deployment architecture and support needs — 'never per token'. Volume discount tiers exist but no minimum amount is published.",
      "min_commitment": "Enterprise is annual term; no stated minimum amount.",
      "overage_note": "None stated for OSS or Enterprise. Practical exposure is self-hosted infrastructure and database cost.",
      "cache_support": "hybrid",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact-match and semantic self-hosted caches: In-Memory, Disk, Redis, Qdrant Semantic, Redis Semantic, Valkey Semantic, S3 bucket, GCS bucket. LiteLLM prices no tokens; provider cached-token rates flow through in its cost calculations. Saving is 100% the customer's; the cost you add is running Redis/Qdrant yourself.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Not stated on the pages fetched.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Free. /key/generate accepts models, user_id, team_id, metadata, aliases, budgets and limits.",
          "tier": "oss",
          "supported": true
        },
        "rate_limiting": {
          "note": "Free, pre-request. tpm_limit, rpm_limit, max_parallel_requests, /key/block and /key/unblock.",
          "tier": "oss",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated on the pages fetched.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Free. Keys specify allowed models, evaluated against the key row itself.",
          "tier": "oss",
          "supported": true
        },
        "budget_caps_per_key": {
          "note": "Free, pre-request. max_budget + budget_duration, with admin-side upperbound_key_generate_params.",
          "tier": "oss",
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "Teams carry max_budget; owner budgets apply to keys with no team.",
          "tier": "oss",
          "supported": true
        }
      },
      "cost_attribution": "The strongest of any product here: spend logged per hashed api_key, internal user, end-user/customer, team_id, tags, model_group, api_base, User-Agent (tracked as a tag) and arbitrary custom headers/metadata.",
      "cost_export": "API + database: /spend/logs, /user/info, /user/daily/activity, /gateway/daily/activity, /global/spend/report?group_by=customer|team|api_key|internal_user, x-litellm-response-cost header, and the LiteLLM_SpendLogs table you own. CSV, webhook, S3 and warehouse export not stated as first-class features.",
      "product_kind_note": "Titled \"LiteLLM AI Gateway (LLM Proxy)\" and elsewhere \"LiteLLM Proxy (LLM Gateway)\", it is a self-hosted proxy that maps many provider APIs to OpenAI-format input/output ([LiteLLM AI Gateway](https://docs.litellm.ai/docs/simple_proxy), [LiteLLM providers](https://docs.litellm.ai/docs/providers)). Not a model platform: \"input, output and exceptions are mapped to the OpenAI format for all supported models\" ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "deployment_modes": [
        "self_host",
        "hybrid_vpc",
        "air_gapped"
      ],
      "deployment_note": "Official images publish to `ghcr.io/berriai` mirrored at `docker.litellm.ai/berriai` (e.g. `docker.litellm.ai/berriai/litellm:v1.90.2`); the `litellm-helm` chart covers monolithic deployments on EKS/GKE/AKS and a componentized `litellm` chart splits `gateway` (LLM traffic, port 4000), `backend` (management API/UI, 4001) and `ui` (3000); official Terraform modules exist for AWS ECS Fargate and Google Cloud Run, and plain Kubernetes manifests are documented ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)). Licensing is a flag, not a build: `litellm_license` is \"optional, omit for open source\" ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)); Enterprise adds SSO+SCIM and support, self-served as \"an instant key, emailed to you\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "yes",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap: \"Just change the `base_url`, `api_key` and `model`… LiteLLM Proxy is OpenAI-Compatible, it works with any project that calls OpenAI\", and it is also Azure OpenAI-, Anthropic- and Vertex AI-compatible (`/chat/completion`, `/embedding` for Vertex) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)). No proprietary SDK required — documented clients are the OpenAI SDK, LangChain, LlamaIndex, Instructor and curl ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys), [LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)). Streaming works with `\"stream\": true` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); Assistants and Fine-Tuning endpoints are also passed through ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "model_count_low": null,
      "model_count_high": null,
      "model_count_source_note": "n.a.: no total is stated; coverage is expressed per provider (\"LiteLLM supports all models on VLLM\", \"We support ALL Fireworks AI models\", CometAPI \"500+ AI models\") ([LiteLLM providers](https://docs.litellm.ai/docs/providers), undated).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "n.a.: the providers index states no total, though it enumerates a long list of provider sections (OpenAI, Azure OpenAI, Azure AI, Vertex AI, Bedrock, HuggingFace, Nvidia NIM, Perplexity, RunwayML, WatsonX, xAI, Databricks, Snowflake Cortex, Together AI, vLLM, Ollama and many more) ([LiteLLM providers](https://docs.litellm.ai/docs/providers), undated).",
      "first_party_vs_partner": "All third-party or customer-run: LiteLLM ships no models and integrates provider APIs plus self-hosted servers (vLLM, Ollama, LM Studio, Llamafile, Lemonade, Nvidia Riva, custom torch-serve) ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).",
      "custom_endpoint_support": "Yes: \"Call your custom torch-serve / internal LLM APIs via LiteLLM\", with documented support for vLLM (\"all models on VLLM\"), Ollama (\"all models from Ollama\"), AWS SageMaker (\"All Sagemaker Huggingface Jumpstart Models\"), LM Studio, Llamafile, and OpenAI-compatible endpoints selected by choosing `openai` as provider with an upstream URL ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).",
      "byok_mode": "byok_only",
      "merchant_of_record": "n.a. as an explicit statement: the LiteLLM pages fetched do not state who invoices model usage; LiteLLM Enterprise sells the software licence while the deployment runs on the customer's infrastructure and keys ([LiteLLM Enterprise](https://www.litellm.ai/enterprise), [LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "key_management_note": "Keys stay in the customer's deployment: Kubernetes manifests use a Secret for keys and a ConfigMap for `config.yaml` ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)), and Enterprise adds secret-manager integrations — AWS Secrets Manager-style vaults, Google KMS, Google Secret Manager, HashiCorp Vault, CyberArk and custom secret managers — plus key rotations and read/write to the secret manager ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Access is organized as \"Organizations → Teams → Projects → Keys\"; OSS includes request/response logging, Enterprise adds per-key/per-team routing to Langfuse, Langsmith and Arize, log export to GCS/Azure Blob, and \"disable logging per team — GDPR-friendly opt-out at the team level\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Since it is self-hosted, no vendor plane sees request content by default ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).",
      "api_openai_chat_completions_note": "Yes: `/chat/completions` (examples use `curl http://0.0.0.0:4000/v1/chat/completions`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "api_anthropic_messages_note": "Yes: \"LiteLLM Proxy is Anthropic-compatible\" with `/messages` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "api_openai_responses_note": "Yes, referenced as a supported surface: Manus agents are called \"through LiteLLM's OpenAI-compatible Responses API\" ([LiteLLM providers](https://docs.litellm.ai/docs/providers)). Explicit `/responses` path: n.a.",
      "api_embeddings_note": "Yes: `/embeddings` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "api_images_note": "Yes: `/image/generations` ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); provider-level image generation and editing are documented for Black Forest Labs, Fal AI and RunwayML ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).",
      "api_audio_note": "Yes, both directions: `/audio/transcriptions` (STT) and `/audio/speech` (TTS) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); provider examples include Deepgram's `/listen` and Nvidia Riva transcription ([LiteLLM providers](https://docs.litellm.ai/docs/providers)).",
      "api_batch_note": "Yes: \"Batches API endpoints\" are exposed, plus a separate beta \"batch completions\" that fans one request out to N models via a comma-separated model string (`\"model\"=\"llama3,gpt-3.5-turbo\"`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "deployment_modes_note": "self-host via Docker, Helm, Terraform, Kubernetes, or a local binary ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy), [LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)); hybrid/VPC in the customer's own cloud (the AWS Terraform module reuses an existing VPC, subnets, database and Redis) ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)); air-gapped on Enterprise — \"self-hosted or air-gapped, and yours to run\" ([LiteLLM Enterprise](https://www.litellm.ai/enterprise)). Hosted SaaS: n.a. (not documented on the pages fetched).",
      "byok_mode_note": "byok_only: the proxy is configured with the customer's own upstream provider keys, and it can additionally \"allow users to pass in their OpenAI API key\"; callers authenticate with LiteLLM virtual keys (e.g. `sk-test-proxy-key-123`) ([LiteLLM proxy client examples](https://docs.litellm.ai/docs/proxy/user_keys)).",
      "access_meta": {
        "qualified": [
          "api_openai_responses",
          "api_audio"
        ],
        "model_count_kind": null,
        "provider_count_kind": null
      },
      "rel_timeout_config": "per_request",
      "rel_timeout_note": "`per_request` and `config_file`. Config: `litellm_settings.request_timeout: 10` (\"raise Timeout error if call takes longer than 10s\") ([LiteLLM reliability](https://docs.litellm.ai/docs/proxy/reliability)); `router_settings.timeout: 30`, per-deployment `litellm_params.timeout: 300`, and per-request `\"timeout\": 1` in the JSON body (or `extra_body={\"timeout\": 1}` from OpenAI SDKs). Streaming has its own key `stream_timeout` (first-chunk deadline). No default value is stated ([LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).",
      "rel_retry_config": "per_request",
      "rel_retry_note": "`per_request` and `config_file`. `num_retries` (e.g. `litellm_settings.num_retries: 3`), with precedence `x-litellm-num-retries` header → request body → deployment `litellm_params` → `litellm_settings`. Default count is not stated: `n.a.` Backoff: exponential for `RateLimitError`, immediate retry for generic errors; minimum delay configurable via `retry_after`. Per-error counts via `RetryPolicy` (e.g. `TimeoutErrorRetries: 2`, `RateLimitErrorRetries: 3`). Note `max_retries` is the provider SDK's internal counter and is forced to `0` on router paths to avoid `(1 + N) ** 2` upstream calls ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "`config_file` (also per-request). ORDERED: `fallbacks: [{\"gpt-3.5-turbo\": [\"gpt-4\"]}]`, and \"Fallbacks are done in-order\". Specialized variants: `default_fallbacks`, `content_policy_fallbacks`, `context_window_fallbacks`. Fallback fires only after `num_retries` is exhausted ([LiteLLM reliability](https://docs.litellm.ai/docs/proxy/reliability)). WEIGHTED failover also exists inside a model group via `enable_weighted_failover` (requires `routing_strategy=\"simple-shuffle\"`, uses `weight`/`rpm`/`tpm`, capped by `max_fallbacks`, default `5`; async entrypoints only) ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).",
      "rel_fallback_shape": "weighted",
      "rel_health_check_config": "config_file",
      "rel_health_check_note": "`config_file`, and it is cooldown-based ejection (LiteLLM does not call it circuit breaking): `allowed_fails` (default `3`), `cooldown_time` (default `5s`), `disable_cooldowns`, plus `AllowedFailsPolicy` per error type. Cooldown triggers: `429`, >50% failures in the current minute, and non-retryable `401`/`404`/`408`. Cooled deployments are removed from the pool and automatically reintroduced ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).",
      "rel_multi_region_config": "config_file",
      "rel_multi_region_note": "\"LiteLLM will handle retrying in different regions if a call fails\", with `region_name` (e.g. `\"eu\"`) auto-inferred for Vertex AI, Bedrock and WatsonX, and weighted failover documented across Azure regional copies (`eastus2`, `swedencentral`). LiteLLM does not run the regions itself — you register per-region deployments ([LiteLLM routing](https://docs.litellm.ai/docs/routing)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Deployments sharing a `model_name` form a load-balanced model group; `routing_strategy` selects the algorithm (e.g. `\"least-busy\"`, `\"simple-shuffle\"`), and weights are supported through `weight`/`rpm`/`tpm` in the weighted-failover path ([LiteLLM routing](https://docs.litellm.ai/docs/routing), [LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "Currently in transition and the repo states both: the repository description is \"The fastest, litest AI Gateway. Rust core with Python SDK\", while the language breakdown is Python 84%, TypeScript 13.2%, Rust 0.7%. Best single label from the evidence: `interpreted_proxy` with a stated Rust core ([BerriAI/litellm](https://github.com/BerriAI/litellm)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "Docker images `ghcr.io/berriai/litellm:<release-tag>` (cosign-signed; `-stable` tags after 12-hour load tests) plus componentized `litellm-gateway`/`-backend`/`-ui` images, Helm charts `litellm-helm` and `litellm` (`oci://ghcr.io/berriai/litellm-helm`), and Terraform modules for AWS ECS Fargate and GCP Cloud Run ([BerriAI/litellm](https://github.com/BerriAI/litellm), [LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).",
      "latency_claims": [
        {
          "url": "https://github.com/BerriAI/litellm",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "8",
          "metric": "p95",
          "publisher": "vendor",
          "conditions": "At 1k RPS. The README does not say whether this is added overhead or end-to-end, and gives no payload or hardware."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://github.com/BerriAI/litellm",
          "unit": "RPS",
          "scope": "capacity",
          "value": "1,000",
          "metric": "load condition",
          "publisher": "vendor",
          "conditions": "Stated only as the load the 8ms p95 figure was taken at; hardware not stated."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported via `\"stream\": true`. Documented streaming machinery/caveats: `stream_timeout` bounds only the first chunk and can be used to abort hanging providers and retry another model; SSE keepalives via `keepalive_seconds` (clamped 1–300 s; `0` at deployment level is a hard disable) or proxy-wide `sse_keepalive_ping_interval_seconds`, sending `: ping` comment frames; client-side override requires `allow_client_keepalive_override: true` (header `x-litellm-keepalive-seconds`) ([LiteLLM timeouts](https://docs.litellm.ai/docs/proxy/timeout)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Presidio integration supports per-entity `MASK` or `BLOCK`, runs in `pre_call` mode, and can skip system/tool messages; masked data is forwarded downstream with `pass_data: true` ([Guardrails quick start](https://docs.litellm.ai/docs/proxy/guardrails/quick_start), [Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))",
      "guard_injection": "sync_block",
      "guard_injection_note": "`sync_block` via third parties — e.g. Lakera configured as `guardrail: lakera, mode: pre_call`, plus PromptGuard, Gray Swan, and others; detection is delegated to the vendor's classifier ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder), [Guardrail providers](https://docs.litellm.ai/docs/guardrail_providers))",
      "guard_content": "sync_block",
      "guard_content_note": "`sync_block` — moderation guardrails (e.g. OpenAI moderation, Azure Content Safety, Bedrock Guardrails) block on policy failure such as toxic content ([Guardrail providers](https://docs.litellm.ai/docs/guardrail_providers), [Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policies are declared in YAML pipelines with `on_pass`, `on_fail`, `on_error` branches per step (`allow`, `block`, `next`, `modify_response` with a custom message), and custom guardrail classes can be registered; key-level `permissions` control which guardrails apply ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder), [Guardrails quick start](https://docs.litellm.ai/docs/proxy/guardrails/quick_start))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "`models` allowlists on keys and teams restrict callable models ([Model access](https://docs.litellm.ai/docs/proxy/model_access))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "Microsoft Presidio",
        "AWS Bedrock Guardrails",
        "Azure Content Safety",
        "OpenAI Moderation",
        "Lakera",
        "Aporia",
        "Pangea",
        "Guardrails AI",
        "Pillar Security"
      ],
      "guard_failure_mode": "configurable",
      "guard_failure_mode_note": "**Configurable, defaults to the policy branch**: technical errors (timeouts, network errors, guardrail not loaded, connection failure) are handled by `on_error`, and \"If `on_error` is omitted, the pipeline uses `on_fail` for error outcomes\". The docs' own example fails open: \"If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed)\" ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder))",
      "log_content": "configurable",
      "log_content_note": "\"Store Prompts in Spend Logs\" controls whether request messages and response content are stored; when disabled only metadata (tokens, cost, model) is kept ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`yes` — leave \"Store Prompts in Spend Logs\" off or set `general_settings.store_prompts_in_spend_logs: false`; additionally `turn_off_message_logging`, per-request header `x-litellm-enable-message-redaction`, and the `no-log` flag suppress message content on callbacks ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings), [Logging](https://docs.litellm.ai/docs/proxy/logging))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry is a first-class callback, alongside Langfuse, Arize, Langtrace, MLflow, and Langsmith for chain/agent-level traces ([Logging](https://docs.litellm.ai/docs/proxy/logging))",
      "export_destinations": [
        "OpenTelemetry",
        "Datadog",
        "Langfuse",
        "Langsmith",
        "Arize Phoenix",
        "Prometheus",
        "S3",
        "GCS",
        "Athina",
        "Braintrust",
        "Helicone",
        "Lago",
        "Logfire",
        "Sentry",
        "Slack",
        "Webhook",
        "Galileo",
        "OpenMeter",
        "DynamoDB",
        "Azure Blob Storage"
      ],
      "export_destinations_note": "Langfuse, OpenTelemetry, GCS buckets, Amazon S3, Azure Blob Storage, GCS Pub/Sub, AWS SQS, DynamoDB, Datadog, Azure Sentinel, Lunary, MLflow, Deepeval, Langsmith, Arize, Langtrace, Galileo, OpenMeter, Sentry, Athina ([Logging](https://docs.litellm.ai/docs/proxy/logging))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — no native feedback/score endpoint on the fetched pages; scoring is delegated to callback platforms ([Logging](https://docs.litellm.ai/docs/proxy/logging))",
      "eval_hooks": "partial",
      "eval_hooks_note": "Indirectly, and without LiteLLM SaaS: Deepeval, MLflow, Langsmith, Galileo, Athina and Arize callbacks receive traffic for offline/online evaluation in those tools ([Logging](https://docs.litellm.ai/docs/proxy/logging))",
      "observability_requires_saas": "No — the proxy is self-hosted, spend logs live in the customer's database, and callbacks ship to destinations the customer chooses ([Logging](https://docs.litellm.ai/docs/proxy/logging), [Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))",
      "log_retention_note": "Configurable via `maximum_spend_logs_retention_period` (e.g. `7d`, `30d`) settable from the Admin UI without a restart; if unset, logs follow config/default behaviour and no fixed default is documented ([Spend log settings](https://docs.litellm.ai/docs/proxy/ui_spend_log_settings))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "One published figure: **`8ms P95 latency` at `1k RPS`** (metric: P95; the README does not state whether this is gateway overhead or end-to-end, and gives no payload, hardware or cache conditions — conditions not stated; it links to a benchmarks page) ([BerriAI/litellm](https://github.com/BerriAI/litellm)).",
        "multi_surface": {
          "rel_retry": [
            "per_request",
            "config_file"
          ],
          "rel_timeout": [
            "per_request",
            "config_file"
          ]
        },
        "throughput_prose": "**`1k RPS`**, stated only as the load condition attached to the 8 ms P95 figure; hardware not stated ([BerriAI/litellm](https://github.com/BerriAI/litellm)).",
        "benchmark_independence": "The 8 ms / 1k RPS figure is vendor self-published ([BerriAI/litellm](https://github.com/BerriAI/litellm)). **Flag:** the widely circulated LiteLLM latency numbers (P50 38.65 s, ~40 ms overhead, 44.84 req/s) are published by a RIVAL, Maxim AI's Bifrost, not by LiteLLM ([Bifrost benchmarks](https://www.getmaxim.ai/bifrost/resources/benchmarks)) — they are not comparable to LiteLLM's own claim and should not be reported as LiteLLM data."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "choose_at_setup",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Deepest reliability surface of any self-hostable option here (retry precedence chain, per-error-type retry and failure policies, cooldowns, ordered + weighted failover, region inference), and the docs explicitly warn about retry amplification if the provider SDK's own `max_retries` is left on ([LiteLLM routing](https://docs.litellm.ai/docs/routing)). Kubernetes probes are documented at `/health/readiness` and `/health/liveliness` ([LiteLLM production deployment](https://docs.litellm.ai/docs/proxy/deploy)).",
      "guard_notes": "LiteLLM is the only product in this set that gives an explicit, documented three-way distinction between pass, policy failure, and technical error, with independent branch actions for each — the cleanest fail-open/fail-closed control available ([Policy flow builder](https://docs.litellm.ai/docs/proxy/guardrails/policy_flow_builder)).",
      "fit_agent_features_note": "LiteLLM supports OpenAI-compatible tool/function calls for MCP tools and can automatically execute returned MCP tool calls when `require_approval: \"never\"`, then feed the results back to the model. Multi-step tool-result-to-model flow is documented through `litellm.experimental_mcp_client.call_openai_tool`; sessions, threads, and A2A are not documented. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No for the documented first call: it uses `huggingface/bigcode/starcoder` and states `HUGGINGFACE_API_KEY` is \"[OPTIONAL]\" ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start)). Production secrets do include a provider key, e.g. `--from-literal=OPENAI_API_KEY=\"<provider-key>\"` ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_client_sdk_langs": [
        "Python"
      ],
      "fit_client_sdks_note": "Documented clients and integrations include the OpenAI SDK with `openai.OpenAI`, the Anthropic SDK with `anthropic.Anthropic`, the LiteLLM Python SDK, Mistral SDK, LlamaIndex, and LangChain for JavaScript and Python. The quick-start examples also show `langchain.chat_models.ChatOpenAI` and `langchain.embeddings.OpenAIEmbeddings`. ([LiteLLM proxy quick start](https://docs.litellm.ai/docs/proxy/quick_start))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "For AWS Bedrock AgentCore MCP servers, LiteLLM supports AWS SigV4 and can use AWS access keys or the boto3 credential chain, including IAM roles and environment variables. The documented configuration includes `auth_type: \"aws_sigv4\"`, `aws_region_name`, and `aws_service_name: bedrock-agentcore`. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))",
      "fit_cloudflare_workers": "upstream_model_provider",
      "fit_cloudflare_workers_note": "LiteLLM documents Cloudflare Workers AI support using the `cloudflare/` model prefix, including `cloudflare/@cf/meta/llama-2-7b-chat-int8` and `cloudflare/@hf/thebloke/codellama-7b-instruct-awq`. ([LiteLLM Cloudflare Workers AI provider](https://docs.litellm.ai/docs/providers/cloudflare_workers))",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented) on either page; no enterprise/licence requirement is stated and `litellm_license` is optional in the Terraform examples ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start), [Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_ecosystem_notes": "LiteLLM is documented as either a Proxy Server or a Python SDK; the Proxy Server can run via the LiteLLM CLI or Docker and listens at `http://0.0.0.0:4000`. The product documentation also describes self-hosted deployment on AWS, Google Cloud, Azure, and Kubernetes clusters. ([LiteLLM documentation home](https://docs.litellm.ai/))",
      "fit_eval_prereqs": "No account, credit card, cloud account, cluster, or provider key is stated; the Hugging Face key in the example is explicitly \"[OPTIONAL]\". Python 3.10+ is required for LiteLLM 1.84.0+ ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))",
      "fit_eval_shape": "cli_or_container",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `POST http://0.0.0.0:4000/chat/completions` with JSON containing `\"model\"` and `\"messages\"` ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "litellm-helm at oci://ghcr.io/berriai/litellm-helm — the GitHub page describes it as community maintained",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "LiteLLM deployment documentation lists the Helm charts `litellm-helm` at `oci://ghcr.io/berriai/litellm-helm` and `litellm` at `oci://ghcr.io/berriai/litellm/chart/litellm`. ([LiteLLM deployment documentation](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "LiteLLM Proxy provides an MCP Gateway with endpoints including `/mcp-rest/tools/list` and `/mcp-rest/tools/call`. It supports listing and calling tools, prompts, resources, Streamable HTTP, SSE, and stdio, with MCP permissions by Key, Team, Organization, and end user. ([LiteLLM MCP documentation](https://docs.litellm.ai/docs/mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_credit_card",
          "fit_gateway_platform",
          "fit_model_gate",
          "fit_vercel_ai_sdk"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented) ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start))",
      "fit_onboarding_notes": "Version trap documented: on Python 3.9 pip silently resolves to 1.83.9 without an error, so you must check `python --version`, upgrade to 3.10+ and reinstall; `uv tool install` provisions a compatible Python automatically. Config via `your_config.yaml` and `litellm --config your_config.yaml`. Production adds `LITELLM_SALT_KEY` (set once, never change), a `/health/readiness` check, and an Admin UI at `/ui`; the proxy claims \"1.5k+ requests/second during load tests\" ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start), [Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. LiteLLM documents OpenAI Python SDK drop-in usage with `api_key=\"anything\"` and the exact proxy base URL `base_url=\"http://0.0.0.0:4000\"`, using `openai.OpenAI`. ([LiteLLM documentation home](https://docs.litellm.ai/))",
      "fit_prod_prereqs": "A Kubernetes cluster (EKS, GKE, AKS or other), external PostgreSQL (required for authentication/tracking) and Redis (required for multiple instances), cloud ingress/load balancer, and secrets management; the Terraform path provisions networking, databases, Redis, object storage, secrets, load balancer and migrations (AWS: Aurora PostgreSQL, ElastiCache, S3, ALB; GCP: Cloud SQL, Memorystore, GCS, HTTPS load balancer) ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "infra_rollout",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LiteLLM documents integrations with LangChain and LlamaIndex. Examples include LangChain classes such as `ChatOpenAI` and `OpenAIEmbeddings`, and LlamaIndex classes including `AzureOpenAI`, `AzureOpenAIEmbedding`, `VectorStoreIndex`, and `SimpleDirectoryReader`. ([LiteLLM user keys and integrations](https://docs.litellm.ai/docs/proxy/user_keys))",
      "fit_quickstart_steps": 0,
      "fit_quickstart_steps_caveat": "The getting-started page is prose plus commands rather than a numbered procedure.",
      "fit_quickstart_url": "https://docs.litellm.ai/docs/proxy/quick_start",
      "fit_self_host_docs": "command_published",
      "fit_self_host_install": "`uv tool install 'litellm[proxy]'` / `pip install 'litellm[proxy]'` for local ([Quick Start - LiteLLM Proxy CLI](https://docs.litellm.ai/docs/proxy/quick_start)); production images `ghcr.io/berriai/litellm` and `docker.litellm.ai/berriai/litellm:v1.90.2`, plus `helm install litellm oci://ghcr.io/berriai/litellm-helm -f values.yaml` and `helm upgrade --install litellm oci://ghcr.io/berriai/litellm/chart/litellm --version 1.89.2 -f values.yaml`, with `kubectl create secret generic litellm-masterkey|litellm-db|litellm-env ...` ([Production Deployment - LiteLLM](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_terraform": "official_modules",
      "fit_terraform_note": "LiteLLM deployment documentation lists official Terraform modules `BerriAI/litellm/aws` and `BerriAI/litellm/google`, along with the resource-management provider `terraform-provider-litellm`. ([LiteLLM deployment documentation](https://docs.litellm.ai/docs/proxy/deploy))",
      "fit_terraform_ref": "BerriAI/litellm/aws and BerriAI/litellm/google, plus a terraform-provider-litellm",
      "fit_third_party_documented": {
        "kubernetes": "community maintainers, per the chart's own GitHub page"
      },
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "not_documented",
      "fit_vercel_ai_sdk_note": "No official Vercel AI SDK provider package is documented; instead LiteLLM documents Vercel AI Gateway as an upstream provider route (`vercel_ai_gateway/`), and Vercel documents LiteLLM as a framework integration using the Python `litellm` package. ([Vercel AI Gateway LiteLLM framework integration](https://vercel.com/docs/ai-gateway/ecosystem/framework-integrations/litellm))",
      "fit_vercel_ai_sdk_package": null
    },
    {
      "slug": "llmgateway",
      "name": "LLM Gateway",
      "company": "Polar Lights LLC",
      "tagline": "AGPL-licensed OpenAI-compatible gateway available as one self-hosted Docker image or a hosted service with credit fees.",
      "category": "open_source",
      "deployment": "both",
      "license": "AGPL-3.0",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://llmgateway.io/",
      "url_docs": "https://docs.llmgateway.io",
      "url_pricing": "https://llmgateway.io/pricing",
      "url_status": "https://status.llmgateway.io",
      "url_github": "https://github.com/theopenco/llmgateway",
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": 601,
      "model_count": 200,
      "model_count_note": "Vendor-stated 200+ models across 40+ providers on the pricing page; the open-source page cites 280+ models and 35+ providers, and new models are added within 48 hours.",
      "provider_count": 40,
      "token_markup_pct": 0,
      "credit_fee_pct": 5,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "BYOK included on the free plan at no fee; self-hosted requests carry no credit fee and no BYOK metering.",
      "seat_fee_usd_month": 0,
      "free_tier": "$0 forever: no seats or minimums, BYOK free, 3 free models limited to 20 req/min, 30-day data retention; hosted credits carry a 5% platform fee.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Full request/response payload retention",
          "amount": "$0.01 per 1M tokens"
        },
        {
          "label": "Credit purchases on the hosted service",
          "amount": "5% platform fee"
        }
      ],
      "self_host_cost_note": "The whole platform (gateway, API, dashboard, worker) ships as one Docker image, ghcr.io/theopenco/llmgateway-unified:latest, free forever under AGPLv3 with no credit fee or BYOK metering; the same codebase powers the managed cloud. Code in the repo's ee/ directory requires a separate commercial enterprise license (contact@llmgateway.io).",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": null,
      "config_as_code": false,
      "portability_note": "Activity page is view-only; no documented export.",
      "api_surfaces": [
        "OpenAI Chat Completions"
      ],
      "sdk_note": "Drop-in OpenAI SDK compatibility by pointing the base URL at https://api.llmgateway.io/v1.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No proxy-overhead benchmark published by the vendor and no independent benchmark found; routing is described as scored on live uptime, throughput, price and latency.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": "Enterprise offers region pinning and routing restricted to providers meeting SOC 2 / ISO / GDPR requirements, but no EU residency guarantee is stated.",
      "zdr": null,
      "zdr_note": "Not branded as ZDR. The gateway blocks requests that would reach a non-conforming provider before any data leaves it, and the DevPass and chat tiers are always metadata-only with no way to enable payload storage.",
      "air_gapped": null,
      "best_for": "Small teams that want an OpenRouter-style hosted gateway with a genuine zero-cost self-host escape hatch and no token markup.",
      "strengths": [
        "No token markup: you pay each provider's own rates, with a flat 5% only on hosted credit purchases",
        "Entire platform self-hosts from a single Docker image under AGPLv3, free forever",
        "Free plan has no seats or minimums and includes BYOK at no fee",
        "SOC 2 Type II certified despite being a very small operation",
        "Enterprise tier covers SSO/SAML/SCIM, guardrails, immutable SIEM-ready audit logs and provider compliance routing"
      ],
      "weaknesses": [
        "Tiny project by comparison (601 GitHub stars, latest tagged release v1.2.0 from September 2025) and operated by a single small Delaware LLC",
        "Publishes no latency or throughput benchmarks, and no independent benchmark exists",
        "Model and provider counts are inconsistent between its pricing and open-source pages",
        "Advanced governance (guardrails, audit logs, region pinning, SLA) is gated behind an unpriced enterprise license plus the ee/ code license"
      ],
      "avoid_if": "You need proven scale, published performance data, or vendor depth to support a large regulated deployment.",
      "reviews": [
        {
          "url": "https://www.youtube.com/watch?v=6mBV7R3ZDmw",
          "why": "Independent developer show demos llmgateway.io live with its founders: self-hosting, provider keys, base-URL swap, local models and Docker deployment.",
          "date": "2025-11-10",
          "type": "video",
          "title": "DevNTell - Unifying AI Model Access with LLM Gateway feat. Co-Founders Ismail & Luca",
          "publisher": "YouTube - DevNTell (Developer DAO)"
        }
      ],
      "sources": {
        "soc2": "https://llmgateway.io/",
        "company": "https://llmgateway.io/about",
        "license": "https://github.com/theopenco/llmgateway",
        "sdk_note": "https://llmgateway.io/",
        "url_docs": "https://docs.llmgateway.io",
        "free_tier": "https://llmgateway.io/pricing",
        "extra_fees": "https://llmgateway.io/pricing",
        "hq_country": "https://llmgateway.io/about",
        "url_status": "https://status.llmgateway.io",
        "zdr_status": "https://llmgateway.io/blog/soc2-type-ii",
        "cap_budgets": "https://llmgateway.io/pricing",
        "model_count": "https://llmgateway.io/pricing",
        "api_surfaces": "https://llmgateway.io/",
        "cap_failover": "https://llmgateway.io/about",
        "github_stars": "https://github.com/theopenco/llmgateway",
        "product_fate": "https://github.com/theopenco/llmgateway/releases",
        "regions_note": "https://llmgateway.io/legal/privacy",
        "byok_fee_note": "https://llmgateway.io/blog/open-source-openrouter-alternatives",
        "vendor_status": "https://llmgateway.io/about",
        "byok_supported": "https://llmgateway.io/pricing",
        "cap_guardrails": "https://llmgateway.io/pricing",
        "certifications": "https://llmgateway.io/blog/soc2-type-ii",
        "config_as_code": "https://docs.llmgateway.io/custom-providers",
        "credit_fee_pct": "https://llmgateway.io/pricing",
        "provider_count": "https://llmgateway.io/pricing",
        "retention_days": "https://docs.llmgateway.io/features/data-retention",
        "cap_rate_limits": "https://llmgateway.io/pricing",
        "logs_by_default": "https://docs.llmgateway.io/features/data-retention",
        "cap_cache_simple": "https://llmgateway.io/about",
        "model_count_note": "https://llmgateway.io/open-source",
        "token_markup_pct": "https://llmgateway.io/pricing",
        "cap_observability": "https://llmgateway.io/about",
        "cap_pii_redaction": "https://llmgateway.io/pricing",
        "eu_residency_note": "https://llmgateway.io/pricing",
        "last_release_date": "https://github.com/theopenco/llmgateway/releases/tag/v1.14.0",
        "openai_compatible": "https://llmgateway.io/",
        "subprocessors_url": "https://llmgateway.io/blog/soc2-type-ii",
        "cap_load_balancing": "https://llmgateway.io/about",
        "llmgateway_pricing": "https://llmgateway.io/pricing",
        "seat_fee_usd_month": "https://llmgateway.io/pricing",
        "guardrails_location": "https://llmgateway.io/blog/soc2-type-ii",
        "latency_source_note": "https://llmgateway.io/blog/open-source-openrouter-alternatives",
        "requires_vendor_sdk": "https://docs.llmgateway.io/",
        "self_host_cost_note": "https://llmgateway.io/open-source",
        "prompt_transits_vendor": "https://llmgateway.io/legal/privacy",
        "cap_conditional_routing": "https://llmgateway.io/pricing",
        "trains_on_customer_data": "https://llmgateway.io/legal/privacy"
      },
      "verified_at": {
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "free_tier": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "github_stars": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:44.171859+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed with weekly minor releases through August 2026. Operated by Polar Lights LLC, a Delaware company; no funding or ownership information is published.",
      "last_release_date": "2026-08-24",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "Hosted, where it sits between your application and every provider, or self-hosted with data in your own PostgreSQL. One disclosure worth reading: the hosted service may route some requests through stealth providers whose identity is not publicly disclosed.",
      "logs_by_default": "metadata_only",
      "logs_note": "Metadata-only retention is the default and gives usage analytics without storing payloads.",
      "retention_days": 30,
      "retention_note": "Thirty days for all users, custom on enterprise. Stored Responses-API responses are also held 30 days unless you set store to false.",
      "zdr_status": "depends",
      "trains_on_customer_data": "no",
      "training_note": "Customer prompts, inputs and returned responses are not used to train any models.",
      "subprocessors_url": "https://security.llmgateway.io/",
      "regions_note": "Processed and stored in the EU or the US, with per-project routing overrides to pin regulated workloads to specific regions and providers. Self-hosting supported.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II, independently examined",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPA for enterprise",
          "status": "available"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "both",
      "guardrails_note": "Prompt-injection detection, PII redaction and content moderation are enforced at the gateway before any prompt leaves your boundary — which in a self-hosted install is literally your boundary.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "hybrid",
      "pricing_model_note": "Free OSS self-host (AGPLv3) + credit top-up fee on the hosted platform (5% flat, +1.5% international card fee) + optional storage charge + Enterprise quote. No seats, no minimums, no subscription. Provider rates passed through 0% markup. BYOK 0% fee.",
      "min_commitment": "None — 'no minimums, no subscription'. Enterprise is custom with volume discounts and a 30-day trial.",
      "overage_note": "The only after-the-fact charge is storage: full data retention bills stored requests at $0.01 per 1M tokens; metadata retention is free. Free-plan retention is 30 days; Enterprise is unlimited. No request-volume overage or export fee stated.",
      "cache_support": "none",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Prompt caching is offered ('cache prompts for faster responses'); BYOK users 'still get unified analytics, caching, and failover'. Mechanism (exact-match / semantic / provider passthrough) not stated. LLM Gateway's only fee is 5% on credit purchases (0% on BYOK), so cache savings flow to the customer.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "'Real-time webhook alerts to your channels' (Discord and Slack).",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "20 req/min on 3 free models; paid models unlimited; Enterprise custom. Pre-request.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Real-time webhook alerts.",
          "tier": null,
          "supported": true
        },
        "model_allowlists": {
          "note": "Not stated. Closest are provider compliance policies ('route only to SOC 2 / ISO 27001 / GDPR providers') and region pinning.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated as per-key; the published equivalent is 'cost ceilings per project' plus 'Budgets and spend controls'.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_team": {
          "note": "'Team management' and 'Admin controls' listed without budget specifics.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "Per project (cost ceilings and per-project routing overrides) with 'Advanced analytics'. Per key/user/team/tag/customer not stated.",
      "cost_export": "No CSV/API/webhook/S3/warehouse cost export stated; only alerting webhooks and enterprise audit logs.",
      "product_kind_note": "LLM Gateway calls itself \"an open-source API gateway for Large Language Models (LLMs)\" acting as \"middleware between your applications and various LLM providers\", routing requests, managing provider keys, and tracking token usage and cost ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Its site frames it as \"One API for 40+ providers\" ([llmgateway.io](https://llmgateway.io/)).",
      "deployment_modes": [
        "saas",
        "self_host"
      ],
      "deployment_note": "Self-hosting is offered under AGPLv3 and is free: \"LLM Gateway is open source under AGPLv3, so you can self-host the gateway for free — or use the hosted platform and pay only the credit fee\" ([LLM Gateway pricing](https://llmgateway.io/pricing)); \"the self-hosted version… ensures your LLM traffic never leaves your infrastructure if desired\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Self-hosted deployments are tuned with environment variables such as `EXPLORATION_RATE`, `PREFERRED_PROVIDER_TTL`, `PREFERRED_PROVIDER_UPTIME_THRESHOLD` and `PREFERRED_PROVIDER_SCORE_MARGIN`; per-project routing customization from the dashboard is gated to the Enterprise plan ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). The dedicated self-hosting doc page failed to load (see gaps).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "not_documented",
      "api_images": "partial",
      "api_audio": "not_documented",
      "api_batch": "not_documented",
      "api_surface_note": "Drop-in base-URL swap: \"Just swap out your current LLM provider URL with the LLM Gateway API endpoint\" — \"a single drop-in endpoint\" at `https://api.llmgateway.io/v1/…`, \"maintains compatibility with the OpenAI API format\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21; [quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Streaming: \"pass `stream: true` to any request — Gateway will proxy the event stream unchanged\" ([LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Caveats: the latency routing factor applies only to streaming requests (non-streaming falls back to uptime selection), and a non-`auto` `tool_choice` may be downgraded to `auto` when no eligible mapping can honor it ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). Auth is bearer-token with project-scoped API keys ([LLM Gateway chat completions](https://docs.llmgateway.io/v1_chat_completions), 2026-06-08).",
      "model_count_low": 200,
      "model_count_high": 200,
      "model_count_source_note": "\"Route requests across 200+ models\" ([llmgateway.io](https://llmgateway.io/), undated); docs state no number ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).",
      "provider_count_low": 40,
      "provider_count_high": 40,
      "provider_count_source_note": "\"40+ providers\" ([llmgateway.io](https://llmgateway.io/), undated). Docs pages state no number, referring only to \"multiple LLM providers\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21).",
      "first_party_vs_partner": "All third-party routed; the gateway \"operates between applications and various LLM providers\" and does not state owning infrastructure ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Region pinning is supported (`provider/model:region`) ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).",
      "custom_endpoint_support": "Partially documented: \"custom providers\" exist, with keys \"scoped by their own catalogue\", and a customer-specific upstream can be routed (setting the Alibaba Model Studio workspace ID \"routes through the customer's own endpoint\"); no vLLM/Ollama/SageMaker/private-URL registration flow is documented ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).",
      "byok_mode": "both",
      "merchant_of_record": "Split by mode: with BYOK \"you pay your providers directly\" and there is \"no platform fee\"; on credits, LLM Gateway invoices, charging \"a flat 5% when you buy credits — no seats, no minimums, no subscription\" (plus a +1.5% international card fee), with volume discounts and negotiated terms on Enterprise ([LLM Gateway pricing](https://llmgateway.io/pricing); [LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).",
      "key_management_note": "Provider keys are added on a provider-keys page and can be labelled, rotated across multiple keys per provider, and are surfaced per attempt as `providerKeyId` / `providerKeyLabel` with an opaque `apiKeyHash` fingerprint; candidate keys appear as `routingMetadata.eligibleProviderKeys` (omitted in credits mode), and the dashboard badges each call \"your key\" or \"LLM Gateway key\" ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). Caller keys are project-scoped ([LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22). Storage mechanism (vault/KMS): n.a. ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). On hosted, calls appear in the dashboard with latency, cost and provider breakdown; self-hosting means \"your LLM traffic never leaves your infrastructure if desired\" ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). The vendor states it is \"SOC 2 Type II Certified\" ([llmgateway.io](https://llmgateway.io/)).",
      "api_openai_chat_completions_note": "Yes: `POST /v1/chat/completions` under `https://api.llmgateway.io/v1/…` ([LLM Gateway chat completions](https://docs.llmgateway.io/v1_chat_completions), 2026-06-08; [LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22).",
      "api_anthropic_messages_note": "Yes: \"the Anthropic Messages endpoint (`/v1/messages`)\" is documented and, per the quickstart, can reach \"any LLM model\" ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27; [LLM Gateway quickstart](https://docs.llmgateway.io/quick-start), 2026-06-22).",
      "api_openai_responses_note": "n.a.",
      "api_embeddings_note": "n.a. (not documented on the pages fetched).",
      "api_images_note": "Partially: image-generation models are supported in routing (an \"image price\" weight \"replaces the price weight for image-generation models\"), but no image endpoint path is documented ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27).",
      "api_audio_note": "n.a.",
      "api_batch_note": "n.a.",
      "deployment_modes_note": "hosted SaaS (\"Hosted Version: For immediate use without setup, visit llmgateway.io to create an account and get an API key\") and self-host (\"Deploy LLM Gateway on your own infrastructure for complete control over your data and configuration\") ([LLM Gateway overview](https://docs.llmgateway.io/overview), 2026-04-21). Hybrid/VPC, on-prem, air-gapped: n.a.",
      "byok_mode_note": "both, and mixable per request: `credentialSource` is either `byok` (\"your own provider key\" — the provider bills the customer and the attempt \"is not deducted from your credits\") or `platform` (\"an LLM Gateway credential\" — deducted from the credit balance); in hybrid mode a BYOK attempt can fail and fall back to an LLM Gateway credential within the same request ([LLM Gateway routing](https://docs.llmgateway.io/features/routing), 2026-08-27). \"Bring your own keys — free forever\" ([llmgateway.io](https://llmgateway.io/)).",
      "access_meta": {
        "qualified": [],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "vendor_total"
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` across all four fetched pages (docs home, self-host, self-host/docker, repo) ([LLM Gateway docs](https://docs.llmgateway.io/)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([LLM Gateway docs](https://docs.llmgateway.io/)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "`not_documented` as a configurable key. Behavior is asserted without configuration: the gateway \"fails over to a healthy provider when one errors\" — ordering, weighting and key name are all unstated ([LLM Gateway docs](https://docs.llmgateway.io/)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "Only the same \"fails over to a healthy provider when one errors\" statement; no health check, breaker or ejection key ([LLM Gateway docs](https://docs.llmgateway.io/)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "Self-hosting on AWS/GCP/Azure and any Kubernetes cluster is documented, but no cross-region failover ([LLM Gateway self-host](https://docs.llmgateway.io/self-host)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "`config_file` at the credential level only: \"LLM Gateway supports multiple API keys per provider for load balancing — provide comma-separated values\" via environment variables. No key name for the policy and no weights ([LLM Gateway Docker self-host](https://docs.llmgateway.io/self-host/docker)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "`interpreted_proxy` (JS runtime). Repo is TypeScript 95.7%; components are `apps/ui` (Vite + React), `apps/api` (Hono backend) and `apps/gateway` ([theopenco/llmgateway](https://github.com/theopenco/llmgateway)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "Single unified Docker container bundling UI, API, gateway, PostgreSQL and Redis; Docker Compose for per-service scaling; an \"official Helm chart\" for EKS/GKE/AKS. Image name and chart repo are not stated ([LLM Gateway self-host](https://docs.llmgateway.io/self-host), [LLM Gateway Docker self-host](https://docs.llmgateway.io/self-host/docker)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "not_documented",
      "streaming_note": "`not_documented` on the fetched pages; no streaming statement or caveat found ([LLM Gateway docs](https://docs.llmgateway.io/)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "Data handling is a retention setting (`retentionLevel: none` avoids storing sensitive content), not a request-path scanner ([Data retention](https://docs.llmgateway.io/features/data-retention), [Logging and analytics](https://deepwiki.com/theopenco/llmgateway/5.3-logging-and-analytics))",
      "guard_injection": "not_documented",
      "guard_injection_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))",
      "guard_content": "not_documented",
      "guard_content_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))",
      "guard_custom": "not_documented",
      "guard_custom_note": "Organisation Policies cover data handling and provider routing rather than content rules ([Policies](https://docs.llmgateway.io/learn/policies), [Audit logs](https://docs.llmgateway.io/features/audit-logs))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "`yes`, at provider granularity — routing can be restricted to compliant providers via organisation policy ([Audit logs](https://docs.llmgateway.io/features/audit-logs), [Policies](https://docs.llmgateway.io/learn/policies))",
      "guard_execution_location": "both",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([Overview](https://docs.llmgateway.io/overview))",
      "log_content": "metadata_only",
      "log_content_note": "\"Metadata Only\" is the default retention level, storing timestamps, model, tokens, and costs \"without full payloads\"; \"Retain All Data\" stores complete request and response payloads including messages, tool calls, and attachments. The log table has `content`, `messages`, and `reasoningContent` columns used only when payload retention is on ([Data retention](https://docs.llmgateway.io/features/data-retention), [Logging and analytics](https://deepwiki.com/theopenco/llmgateway/5.3-logging-and-analytics))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Organization Settings → Policies → Data Retention Level; set to metadata-only or `none`. DevPass and chat subscriptions are always metadata-only with no way to enable payload storage. Responses API records are the exception: kept 30 days for `previous_response_id` chaining regardless of policy, unless `store: false` is sent ([Data retention](https://docs.llmgateway.io/features/data-retention))",
      "tracing_support": "not_documented",
      "tracing_note": "`n.a.` — no OpenTelemetry or agent tracing documented ([Overview](https://docs.llmgateway.io/overview))",
      "export_destinations": [],
      "export_destinations_note": "`n.a.` — activity and logs are exposed through LLMGateway's own management API rather than shipped to external sinks ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` ([Overview](https://docs.llmgateway.io/overview))",
      "observability_requires_saas": "No — all components including logging can be self-hosted ([Self-host](https://docs.llmgateway.io/self-host), [LLMGateway repo](https://github.com/theopenco/llmgateway))",
      "log_retention_note": "The pages disagree; see Contradictions. The data-retention page says 30 days for all users with Enterprise custom ([Data retention](https://docs.llmgateway.io/features/data-retention)); the policies page says Free 30 days, Enterprise custom ([Policies](https://docs.llmgateway.io/learn/policies)); the activity API reference says Free 3 days, Pro 90 days, Enterprise up to 1 year ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity)); the repo advertises \"Extended data retention (90 days vs 3 days)\" ([LLMGateway repo](https://github.com/theopenco/llmgateway))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` on all fetched pages ([LLM Gateway docs](https://docs.llmgateway.io/)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` ([LLM Gateway docs](https://docs.llmgateway.io/)).",
        "benchmark_independence": "No figures published ([theopenco/llmgateway](https://github.com/theopenco/llmgateway))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "not_documented",
      "guard_custom_default": "not_documented",
      "rel_notes": "Architecture is documented in operational terms (stateless services, PostgreSQL as source of truth, Redis for cache and worker queue, AGPLv3 license), but the reliability control surface is essentially undocumented — a developer choosing on reliability would have to read the source ([LLM Gateway self-host](https://docs.llmgateway.io/self-host), [LLM Gateway docs](https://docs.llmgateway.io/)).",
      "guard_notes": "Audit logs are an Enterprise feature ([Audit logs](https://docs.llmgateway.io/features/audit-logs)), and log storage is metered — activity records expose `dataStorageCost`, `creditsDataStorageCost`, and `apiKeysDataStorageCost` fields, so retaining bodies has a direct line-item cost ([Activity API](https://theopenco-llmgateway-23.mintlify.app/api-reference/management/activity)).",
      "fit_agent_features_note": "LLM Gateway is described as working with AI agents and development tools and provides pre-built templates and agent configurations. Tool calling is supported through the Vercel AI SDK; sessions, threads, multi-step orchestration, and A2A are not documented. ([LLM Gateway documentation home](https://docs.llmgateway.io/))",
      "fit_byok_required": "hosted_no_selfhost_yes",
      "fit_byok_required_note": "No for the hosted platform: \"With your own provider keys (BYOK), routing through LLM Gateway is free\" and you may instead \"use the hosted platform and pay only the credit fee\" ([Pricing - LLM Gateway](https://llmgateway.io/pricing)). Yes for self-hosting: prerequisites include \"API keys for the LLM providers you want to use (OpenAI, Anthropic, etc.)\" ([Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))",
      "fit_client_sdk_langs": [
        "TypeScript"
      ],
      "fit_client_sdks_note": "Documented integrations include the OpenAI SDK or other OpenAI-compatible SDKs, the Anthropic SDK, the Vercel AI SDK provider `@llmgateway/ai-sdk-provider`, and the TanStack AI adapter `@tanstack/ai-llmgateway`. The documented OpenAI SDK package is `openai`; languages are otherwise not stated, except that the Vercel AI SDK interface is TypeScript. ([LLM Gateway developer documentation](https://docs.llmgateway.io/developers))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "No: \"Start free with no credit card.\" and \"You can start free without a credit card.\"; plan is \"Free $0 forever.\" ([Pricing - LLM Gateway](https://llmgateway.io/pricing))",
      "fit_ecosystem_notes": "LLM Gateway is available as a hosted service and can be self-hosted under AGPLv3 using Docker, Docker Compose, or Kubernetes; hosted usage is described as pay-as-you-go with a flat 5% platform fee, and BYOK is supported. It provides one API across 40+ providers and routes across 200+ models, with integrations including REST, CLI, MCP, OpenAI-compatible SDKs, the Anthropic SDK, the Vercel AI SDK provider, and the TanStack AI adapter. ([LLM Gateway AI Gateway product page](https://llmgateway.io/products/ai-gateway))",
      "fit_eval_prereqs": "Dashboard sign-in and an API key from a created Project (exported in a shell or `.env` file). No credit card, cloud account, cluster, or provider key is stated ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start))",
      "fit_eval_shape": "base_url_swap",
      "fit_eval_snippet": "partial",
      "fit_eval_snippet_note": "Partially: the endpoint shape `https://api.llmgateway.io/v1/…` authenticated with `LLM_GATEWAY_API_KEY` is documented and the page has per-language tabs, but no copy-pasteable request body rendered in the fetched content ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "OCI chart on GHCR; deploys gateway, API, UI and worker, and needs PostgreSQL plus Redis",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "LLM Gateway is self-hostable on Kubernetes, and the Kubernetes documentation describes an official Helm chart published as an OCI artifact on GitHub Container Registry. The chart deploys the gateway, API, UI, and worker and requires PostgreSQL and Redis. ([LLM Gateway Kubernetes self-hosting documentation](https://docs.llmgateway.io/self-host/kubernetes))",
      "fit_mcp": "hosted_mcp_server",
      "fit_mcp_note": "LLM Gateway provides a hosted MCP server at `https://api.llmgateway.io/mcp` using streamable HTTP and bearer authentication. It is compatible with Claude Code, Codex, Cursor, and other MCP-compatible clients, and exposes `chat`, `generate-image`, `generate-nano-banana`, `list-models`, and `list-image-models` tools. ([LLM Gateway MCP server](https://llmgateway.io/mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_cloudflare_workers",
          "fit_gateway_platform"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "No model gate documented; the licensing gate applies to enterprise features only: \"Enterprise features require a signed license in production,\" with \"An expired license has a seven-day grace period.\" ([Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker), [Self Host LLM Gateway](https://docs.llmgateway.io/self-host))",
      "fit_onboarding_notes": "Hosted path works in \"any language or framework,\" streaming via `stream: true` is proxied unchanged, every call appears in the dashboard with latency, cost and provider breakdown, and an Anthropic-compatible endpoint is available. Self-hosted setup requires creating an organization and project in the local UI before generating keys, and warns \"Do not bind-mount a host directory directly to `/var/lib/postgresql/data`\" ([Quickstart | LLM Gateway Docs](https://docs.llmgateway.io/quick-start), [Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. The existing OpenAI SDK can be kept while changing the base URL to `https://api.llmgateway.io/v1`; the documented example uses `apiKey: process.env.LLM_GATEWAY_API_KEY`. ([LLM Gateway AI Gateway product page](https://llmgateway.io/products/ai-gateway))",
      "fit_prod_prereqs": "Stateless gateway, API, UI and background worker services; PostgreSQL for users/projects/keys/usage; Redis for response caching and the worker queue; provider API keys (OpenAI, Anthropic, Google, others); a Kubernetes cluster for the Helm deployment (EKS, GKE, AKS or self-managed); managed PostgreSQL and Redis recommended; cloud service mappings (AWS EKS/RDS/ElastiCache/Secrets Manager; GCP GKE/Cloud SQL/Memorystore/Secret Manager; Azure AKS/Azure Database for PostgreSQL/Azure Cache for Redis/Key Vault) ([Self Host LLM Gateway](https://docs.llmgateway.io/self-host))",
      "fit_prod_selfhost_shape": "infra_rollout",
      "fit_prod_shape": "base_url_swap",
      "fit_python_frameworks": [],
      "fit_python_frameworks_note": "n.a. (not documented)",
      "fit_quickstart_steps": 6,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.llmgateway.io/quick-start",
      "fit_self_host_docs": "documented_no_command",
      "fit_self_host_install": "Offered via Docker (all services in one container), Docker Compose, or the official Helm chart on Kubernetes, but no literal command string is printed on the self-host pages; the Docker page's post-install procedure is 4 steps ending \"Test the gateway by making API calls to http://localhost:4001\" (web UI at http://localhost:3002) ([Self Host LLM Gateway](https://docs.llmgateway.io/self-host), [Docker | LLM Gateway Docs](https://docs.llmgateway.io/self-host/docker))",
      "fit_terraform": "official_modules",
      "fit_terraform_note": "The Enterprise plan includes Terraform modules that provision AWS EKS, RDS, ElastiCache, networking, and secrets, and deploy LLM Gateway. The Terraform registry name is not stated. ([LLM Gateway AWS self-hosting documentation](https://docs.llmgateway.io/self-host/aws))",
      "fit_terraform_ref": "Enterprise-plan modules provisioning EKS, RDS, ElastiCache, networking and secrets; no Registry name",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "LLM Gateway provides the first-party npm package `@llmgateway/ai-sdk-provider` for the Vercel AI SDK, including `generateText`, `streamText`, and `generateObject`. The documentation also states that `@ai-sdk/openai` can use the gateway with a custom base URL. ([LLM Gateway Vercel AI SDK documentation](https://docs.llmgateway.io/developers/ai-sdk))",
      "fit_vercel_ai_sdk_package": "@llmgateway/ai-sdk-provider"
    },
    {
      "slug": "amazon-bedrock",
      "name": "Amazon Bedrock",
      "company": "Amazon Web Services",
      "tagline": "AWS-managed service for calling foundation models from 19 model providers through one AWS API.",
      "category": "cloud_platform",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://aws.amazon.com/bedrock/",
      "url_docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html",
      "url_pricing": "https://aws.amazon.com/bedrock/pricing/",
      "url_status": null,
      "url_github": null,
      "url_changelog": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html",
      "url_models_api": null,
      "github_stars": null,
      "model_count": 100,
      "model_count_note": "Docs say \"100+ foundation models\"; the product page says \"hundreds of foundation models\" — no exact catalog count is published.",
      "provider_count": 19,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": null,
      "free_tier": "No Bedrock-specific free tier; new AWS customers get up to $200 in AWS credits ($100 immediately plus up to $100 earned, over 6 months).",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Intelligent Prompt Routing",
          "amount": "$1.00 per 1,000 requests"
        },
        {
          "label": "Guardrails content filters",
          "amount": "$0.15 per 1,000 text units"
        },
        {
          "label": "Guardrails sensitive information filter",
          "amount": "$0.10 per 1,000 text units"
        },
        {
          "label": "Guardrails Automated Reasoning checks",
          "amount": "$0.17 per 1,000 text units"
        },
        {
          "label": "Prompt Optimizer",
          "amount": "$0.03 per 1,000 tokens"
        },
        {
          "label": "Knowledge Bases vector index storage",
          "amount": "$5.00 per GB per month"
        },
        {
          "label": "Knowledge Bases Retrieve API",
          "amount": "$1.00 per 1,000 calls (Agentic Retrieval $4.00 per 1,000)"
        },
        {
          "label": "Human evaluation of model output",
          "amount": "$0.21 per task"
        }
      ],
      "self_host_cost_note": "Managed only, no self-host option. Bedrock sets its own per-token prices rather than marking up a third party: Standard, Flex and Batch (both 50% below Standard) and Priority (75% above Standard) tiers, plus Provisioned Throughput billed per model unit-hour with no-commitment, 1-month or 6-month terms.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": "boto3 remains the primary SDK, but the newer endpoint accepts the OpenAI client.",
      "api_surfaces": [
        "Amazon Bedrock InvokeModel",
        "Converse API",
        "OpenAI Chat Completions (/openai/v1)",
        "OpenAI Responses API",
        "Anthropic Messages"
      ],
      "sdk_note": "The OpenAI SDK works by setting OPENAI_BASE_URL to https://bedrock-runtime.{region}.amazonaws.com/openai/v1; the older bedrock-mantle /v1 route remains supported. Anthropic's client can target https://bedrock-runtime.{region}.amazonaws.com/anthropic.",
      "cap_failover": null,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": null,
      "cap_prompt_management": null,
      "cap_evals": true,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": true,
      "cap_audio": null,
      "cap_video": true,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": null,
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": true,
      "eu_residency": null,
      "eu_residency_note": "Bedrock supports Regional inference plus Global and Geo cross-Region inference profiles; the specific EU Region list for Bedrock was not confirmed from vendor docs in this research.",
      "zdr": true,
      "zdr_note": "Enforceable rather than promised: with retention mode set to none, no request or response is written to durable storage or shared with the provider, and Bedrock blocks requests to models that require retention. Access is evaluated per account and per model with the provider.",
      "air_gapped": null,
      "best_for": "Teams already standardized on AWS that need many model vendors behind one IAM-governed, compliance-attested API.",
      "strengths": [
        "19 model providers plus custom model import under one AWS API and one bill",
        "Deep compliance posture: ISO, SOC, CSA STAR Level 2, GDPR, FedRAMP High, HIPAA-eligible with an AWS BAA",
        "Built-in Guardrails, evaluations, Knowledge Bases and AgentCore instead of bolt-on tooling",
        "Cost levers: Batch and Flex at 50% off Standard, prompt caching, Intelligent Prompt Routing, Provisioned Throughput commitments",
        "OpenAI-compatible Responses and Chat Completions endpoints ease migration"
      ],
      "weaknesses": [
        "Not a cross-cloud gateway: it only routes to models AWS hosts, so no failover to OpenAI, Azure or Google endpoints",
        "Deep AWS lock-in — IAM, regions and fine-tuned artifacts do not port to Vertex AI or Foundry",
        "Pricing is fragmented across tiers, Provisioned Throughput units and per-feature surcharges, making forecasts hard",
        "Model availability varies by Region, so a model you tested may be unavailable where your data must live"
      ],
      "avoid_if": "You want vendor-neutral routing across multiple clouds or a single flat per-token price list.",
      "reviews": [
        {
          "url": "https://www.doit.com/blog/the-engineering-guide-to-amazon-bedrock-cost-optimization",
          "why": "Third-party engineering guide covering Bedrock on-demand vs provisioned throughput, batch inference, intelligent prompt routing and prompt caching from customer engagements.",
          "date": "2026-04-16",
          "type": "written",
          "title": "The Engineering Guide to Amazon Bedrock Cost Optimization",
          "publisher": "DoiT"
        },
        {
          "url": "https://www.constellationr.com/insights/news/amazon-bedrock-vs-diy-approaches-benchmarked",
          "why": "Analyst writeup of Holger Mueller's benchmark comparing Bedrock's managed model access against do-it-yourself model hosting on cost and time-to-deploy.",
          "date": "2024-12-04",
          "type": "written",
          "title": "Amazon Bedrock vs. DIY approaches benchmarked",
          "publisher": "Constellation Research"
        },
        {
          "url": "https://www.youtube.com/watch?v=y1slHK9R_Vg",
          "why": "Independent AWS-focused channel deploys an agent to Bedrock AgentCore Runtime end to end and tests Gateway, Identity, Memory and cross-session behaviour.",
          "date": "2026-03-11",
          "type": "video",
          "title": "AWS Bedrock AgentCore - The Complete Beginners Guide",
          "publisher": "YouTube - Johnny Chivers"
        },
        {
          "url": "https://news.ycombinator.com/item?id=48473166",
          "why": "Practitioners describe concrete blast radius of the data-sharing change: zero-data-retention contract clauses, regulated-industry escalations, and cost comparisons with buying Anthropic directly.",
          "date": "2026-06-10",
          "type": "community",
          "title": "AWS Bedrock to require sharing data with Anthropic",
          "publisher": "Hacker News"
        },
        {
          "url": "https://www.reddit.com/r/aws/comments/1qt3gne/aws_bedrock_in_production_anyone_else_finding_it/",
          "why": "28-comment thread from teams running Bedrock ~6 months, focused on throttling, capacity limits and prototypes breaking at scale including during the us-east-1 outage.",
          "date": "2026-02-01",
          "type": "community",
          "title": "AWS Bedrock in production: anyone else finding it a mixed bag?",
          "publisher": "Reddit - r/aws"
        }
      ],
      "sources": {
        "zdr": "https://aws.amazon.com/bedrock/",
        "gdpr": "https://aws.amazon.com/bedrock/",
        "name": "https://aws.amazon.com/bedrock/",
        "soc2": "https://aws.amazon.com/bedrock/",
        "company": "https://aws.amazon.com/bedrock/",
        "tagline": "https://aws.amazon.com/bedrock/pricing/",
        "sdk_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html",
        "url_docs": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html",
        "url_home": "https://aws.amazon.com/bedrock/",
        "zdr_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/data-protection.html",
        "cap_batch": "https://aws.amazon.com/bedrock/pricing/",
        "cap_evals": "https://aws.amazon.com/bedrock/pricing/",
        "cap_video": "https://aws.amazon.com/bedrock/pricing/",
        "free_tier": "https://aws.amazon.com/free/",
        "hipaa_baa": "https://aws.amazon.com/compliance/hipaa-eligible-services-reference/",
        "extra_fees": "https://aws.amazon.com/bedrock/pricing/",
        "hq_country": "https://en.wikipedia.org/wiki/Amazon_(company)",
        "zdr_status": "https://docs.aws.amazon.com/bedrock/latest/userguide/data-retention.html",
        "model_count": "https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html",
        "url_pricing": "https://aws.amazon.com/bedrock/pricing/",
        "api_surfaces": "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-openai-chat-completions.html",
        "product_fate": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-ug-doc-history.html",
        "regions_note": "https://aws.amazon.com/bedrock/faqs/",
        "cap_image_gen": "https://aws.amazon.com/bedrock/pricing/",
        "url_changelog": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html",
        "vendor_status": "https://aws.amazon.com/bedrock/sla/",
        "cap_guardrails": "https://aws.amazon.com/bedrock/pricing/",
        "certifications": "https://aws.amazon.com/bedrock/faqs/",
        "config_as_code": "https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-resource-bedrock-intelligentpromptrouter.html",
        "provider_count": "https://aws.amazon.com/bedrock/pricing/",
        "retention_days": "https://docs.aws.amazon.com/bedrock/latest/userguide/abuse-detection.html",
        "sla_uptime_pct": "https://aws.amazon.com/bedrock/sla/",
        "bedrock_pricing": "https://aws.amazon.com/bedrock/pricing/",
        "logs_by_default": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html",
        "cap_cache_simple": "https://aws.amazon.com/bedrock/pricing/",
        "model_count_note": "https://aws.amazon.com/bedrock/",
        "cap_pii_redaction": "https://aws.amazon.com/bedrock/pricing/",
        "eu_residency_note": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-regions.html",
        "last_release_date": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-ug-doc-history.html",
        "openai_compatible": "https://docs.aws.amazon.com/bedrock/latest/userguide/doc-history.html",
        "subprocessors_url": "https://aws.amazon.com/compliance/sub-processors/",
        "cap_load_balancing": "https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-regions.html",
        "guardrails_location": "https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html",
        "requires_vendor_sdk": "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html",
        "self_host_cost_note": "https://aws.amazon.com/bedrock/pricing/",
        "data_export_available": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html",
        "bedrock_prompt_caching": "https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html",
        "prompt_transits_vendor": "https://aws.amazon.com/bedrock/faqs/",
        "cap_conditional_routing": "https://aws.amazon.com/bedrock/pricing/",
        "trains_on_customer_data": "https://aws.amazon.com/bedrock/faqs/"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "gdpr": "2026-08-29",
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_batch": "2026-08-29",
        "cap_evals": "2026-08-29",
        "cap_video": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "extra_fees": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "provider_count": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:32.97827+00:00",
      "vendor_status": "hyperscaler",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "A first-party AWS service under continuous development, bought through an existing AWS contract rather than as a new vendor relationship.",
      "last_release_date": "2026-08-24",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://aws.amazon.com/bedrock/sla/",
      "claim_basis": {},
      "last_release_note": "AWS publishes no versioned Bedrock changelog, so this is the most recent entry in the user guide's document history — a documentation update, not a product release.",
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Serverless inside AWS, and you can keep it off the public internet with PrivateLink. Model providers are walled off: they have no access to Bedrock logs or to your prompts and completions.",
      "logs_by_default": "none",
      "logs_note": "Model invocation logging is disabled by default, and when you enable it the logs go to your own S3 or CloudWatch in the same account and region. Bedrock's stated baseline is a zero-data-retention model.",
      "retention_days": 0,
      "retention_note": "Nothing stored by default. Named exceptions run to 30 days for certain third-party models, including classifier-flagged traffic on some OpenAI models and inputs and outputs on some Anthropic models.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "Your content is not used to improve base models and is not shared with model providers. Neither AWS nor the third-party providers use Bedrock inputs or outputs to train any model.",
      "subprocessors_url": "https://aws.amazon.com/compliance/sub-processors/",
      "regions_note": "Content is encrypted and stored at rest in the region where you use Bedrock. Cross-region inference moves retained data to the processing region. PrivateLink reaches into your VPC, but the service itself is not customer-hosted.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "in scope for SOC 1, 2 and 3",
          "status": "certified"
        },
        "fedramp": {
          "detail": "Moderate/Class C in commercial regions; High/Class D in GovCloud",
          "status": "certified"
        },
        "gdpr_dpa": {
          "detail": "AWS DPA and subprocessor notice",
          "status": "certified"
        },
        "iso27001": {
          "detail": "in scope, alongside 27017, 27018, 27701 and others",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "HIPAA-eligible; standard BAA available",
          "status": "certified"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "Sensitive-information filters block or mask PII inside Bedrock, and can be called through the ApplyGuardrail API without invoking a model — but they run on AWS's side.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference, bundled routing: gateway/API layer carries no separately stated fee. Per-token (on-demand), per batch token, or per provisioned model unit hour. Service tiers: Priority +75% to Standard; Flex −50%; Batch −50%.",
      "min_commitment": "None for on-demand. Provisioned Throughput uses 1-month and 6-month commitments (e.g. Cohere Command $39.60/hr/model unit for 1 mo vs $23.77 for 6 mo; Llama 2 70B $21.18 vs $13.08); pricing requires contacting your account team.",
      "overage_note": "No log/trace retention overage, request-volume overage or data-export charge. Adjacent charges are usage-metered: model customization, storage, retrieval, guardrails, evaluations, Advanced Prompt Optimizer (charged for on-demand Standard-tier tokens used in optimization).",
      "cache_support": "hybrid",
      "cache_read_discount_pct": 90,
      "cache_write_premium_pct": 25,
      "cache_pricing_note": "Implicit and explicit prompt caching (prefix-based, not semantic). Explicit uses cache checkpoints (up to 4 per request); TTLs commonly 5 min (Anthropic 1h option, GPT-5.6 default 30m). Implicit caching is best-effort — identical prompts don't guarantee a hit. Cache reads billed at 90% discount for GPT-5.6 (Claude 3.5 Sonnet v2 cache read $0.60 vs $6.00 input = 90% off on the pricing page). Cache writes billed at 1.25× uncached input rate (+25%). Cache saves money conditionally: writes cost 25% more up front and hits are never guaranteed, so a low-hit-rate workload can pay more than it saves. AWS captures nothing extra beyond the write premium.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Not stated as a cost control on the pricing page.",
          "tier": null,
          "supported": null
        },
        "spend_webhooks": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated on the pricing page. Cost shaping levers Bedrock does publish: service tier (Flex/Priority), batch, Provisioned Throughput commitments.",
      "cost_export": "Not stated on the pricing page.",
      "product_kind_note": "AWS calls Bedrock \"a fully managed service that provides secure, enterprise-grade access to high-performing foundation models from leading AI companies, enabling you to build and scale generative AI applications\" ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html)); the FAQ adds that it is \"serverless,\" so customers \"don't have to manage any infrastructure\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). It is a cloud model platform, not a third-party routing proxy.",
      "deployment_modes": [
        "saas",
        "hybrid_vpc"
      ],
      "deployment_note": "There is no customer-installed gateway: applications call regional `bedrock-runtime` (or `bedrock-mantle`) endpoints ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html); [Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). Customer-owned weights arrive through Custom Model Import, which \"only supports Llama 2/3, Mistral, and Flan architectures\", and models fine-tuned in SageMaker can be brought back only if their architecture is import-supported ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)). Feature gating is by region/eligibility rather than tier — e.g. Guardrails Standard tier \"requires opting into cross-region inference,\" and some models are limited-availability ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/); [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "yes",
      "api_embeddings": "not_documented",
      "api_images": "not_documented",
      "api_audio": "not_documented",
      "api_batch": "yes",
      "api_surface_note": "Bedrock's primary surface is proprietary (`InvokeModel`, `InvokeModelWithResponseStream`, `StartAsyncInvoke`, `InvokeModelWithBidirectionalStream`, `Converse`), reached through AWS SDKs — \"iOS and Android SDKs, as well as Java, JS, Python, CLI, .Net, Ruby, PHP, Go, and C++\", with \"streaming supported on all the SDKs\" ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html); [Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). An OpenAI-compatible base-URL swap is possible for the chat-completions path using a Bedrock API key ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). The `Converse` API is \"a unified API that abstracts FM differences and enables model switching with a single parameter change\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)).",
      "model_count_low": 100,
      "model_count_high": 100,
      "model_count_source_note": "\"100+ foundation models\" ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html), undated); separately, Bedrock Marketplace offers \"over 100 popular, emerging, or specialized models\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). The former per-model catalog page has been consolidated into a \"models at a glance\" page ([Supported foundation models](https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html), undated).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party platform with partner catalog, so no routed-provider count: AWS states \"100+ foundation models from industry-leading providers\" and names Amazon, Anthropic, DeepSeek, Moonshot AI, MiniMax, OpenAI, xAI ([What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html), undated). The FAQ lists 13 providers — AI21 Labs, Amazon, Anthropic, Cohere, DeepSeek, Luma AI, Meta, Mistral AI, OpenAI, poolside \"(coming soon)\", Stability AI, TwelveLabs, Writer — without a total ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20).",
      "first_party_vs_partner": "Documented mix. Amazon Titan models are \"created by AWS,\" \"exclusive to Amazon Bedrock,\" and served through \"a fully managed API\"; third-party models are supplied by partner providers and some Bedrock models are sold by third-party providers as \"Third-Party Content\"; Bedrock Marketplace models are \"offered by various providers\" and deployed to fully managed SageMaker endpoints ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). Inference is supported \"in all Regions and with all models supported by Amazon Bedrock\" ([Supported Regions and models for inference](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-supported.html)).",
      "custom_endpoint_support": "Partly: customers can bring their own weights via Custom Model Import (Llama 2/3, Mistral, Flan architectures) and deploy Marketplace models to their own managed SageMaker endpoints; Guardrails can also be applied to \"self-hosted models outside Amazon Bedrock\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Custom model import overview](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html)). Registering an arbitrary external private URL as a Bedrock model: n.a.",
      "byok_mode": "not_applicable",
      "merchant_of_record": "AWS: usage is billed on the AWS account (\"You are charged…\") per the Bedrock pricing page ([Amazon Bedrock pricing](https://aws.amazon.com/bedrock/pricing/)), including per-use charges for Guardrails, while access through SageMaker Unified Studio \"comes at no extra cost,\" with users paying for \"the usage of the underlying resources\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20). An explicit statement naming the invoicing entity for third-party models: n.a. (the FAQ only labels some models \"Third-Party Content\").",
      "key_management_note": "Authentication is AWS-native: Amazon Bedrock API keys authenticate with a bearer token \"instead of AWS credentials\", where short-term keys inherit permissions from the IAM principal that generated them and long-term keys create an IAM user with attached policies; SigV4 AWS credentials also work ([Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html); [Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). There are no upstream provider secrets to store. Requests are processed inside the customer's AWS account boundary and can be kept off the public internet with PrivateLink ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20); batch inputs/outputs live in the customer's own S3 buckets ([Batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)).",
      "api_openai_chat_completions_note": "Yes: `https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions` and `https://bedrock-mantle.{region}.api.aws/v1/chat/completions` ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html)). Native alternatives are `InvokeModel` and the unified `Converse` API ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20).",
      "api_anthropic_messages_note": "n.a. as a native Anthropic Messages endpoint; Anthropic models are reached through Bedrock's own `InvokeModel`/`Converse` APIs ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).",
      "api_openai_responses_note": "Yes: OpenAI GPT-5.6 Sol, Terra and Luna are available \"through the Responses API on Amazon Bedrock,\" and server-side Web Search is invoked through \"a single tool-use parameter in the Responses API\" ([Custom model import overview / Bedrock what's-new content](https://docs.aws.amazon.com/bedrock/latest/userguide/custom-model-import.html); [What is Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html)). Exact path: n.a.",
      "api_embeddings_note": "n.a. as a dedicated endpoint path; embeddings run as model invocations on the `bedrock-runtime` endpoint (`InvokeModel`) ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).",
      "api_images_note": "n.a. as a dedicated endpoint path; image/video generation providers (Stability AI, Luma AI, TwelveLabs) are invoked through the same runtime APIs, including `StartAsyncInvoke` for asynchronous generations ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20; [Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).",
      "api_audio_note": "n.a. on the pages fetched (no STT/TTS endpoint documented in Bedrock's inference API pages), though `InvokeModelWithBidirectionalStream` exists for streaming multimodal interaction ([Inference using Invoke API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-api.html)).",
      "api_batch_note": "Yes: \"With batch inference, you can submit multiple prompts and generate responses asynchronously,\" formatted with either `InvokeModel` or `Converse` request format, with input/output JSONL files in S3; it does not support tool calling or structured output (`response_format`), each record is processed independently without multi-turn, and \"batch inference isn't supported for provisioned models\" ([Batch inference](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)).",
      "deployment_modes_note": "hosted cloud service (fully managed, serverless) ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/), 2026-08-20); customer-VPC private connectivity via AWS PrivateLink — \"establish private connectivity between your FMs and your Amazon Virtual Private Cloud (Amazon VPC) without exposing your traffic to the Internet\" ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)); dedicated managed endpoints for Marketplace models, deployed \"onto fully managed endpoints,\" including \"a fully managed SageMaker endpoint\" with a chosen instance count/type ([Bedrock FAQs](https://aws.amazon.com/bedrock/faqs/)). Self-host, on-prem, air-gapped: n.a.",
      "byok_mode_note": "not_applicable: Bedrock is a first-party inference platform; there are no upstream provider keys to bring, and access is authenticated with AWS credentials (SigV4) or Amazon Bedrock API keys ([Chat Completions API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions-mantle.html); [Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html)).",
      "access_meta": {
        "qualified": [
          "api_anthropic_messages",
          "api_embeddings",
          "api_images",
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "No timeout header, URI parameter or body field. Timeout surfaces only as an error: `ModelTimeoutException`, HTTP `408`, \"Processing time exceeded the model timeout length\" ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented` as a service-side setting. The API reference states that for `ModelNotReadyException` (HTTP `429`) \"The AWS SDK will automatically retry the operation up to 5 times\"; no retry key or backoff strategy is published, so default backoff is `n.a.` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "`not_documented` for model/provider fallback. The only documented fallback-like behavior is latency-mode degradation: when the latency-optimization quota is reached \"we will attempt to serve the request with Standard latency\", and Llama 3.1 405B requests above ~11K total tokens \"fall back to the standard mode\" — controlled by `performanceConfig.latency: standard | optimized`, not by a fallback list ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "No health check, circuit breaker or ejection control; only error types (`throttlingException`, `serviceUnavailableException`, `modelStreamErrorException`) ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "rel_multi_region_config": "not_configurable",
      "rel_multi_region_note": "`not_configurable` in the sense asked: AWS runs the regions and picks for you. Cross-Region inference uses inference profiles that \"define a foundation model (FM) and the AWS Regions to which requests can be routed\"; for geographic or global profiles \"Amazon Bedrock automatically selects a commercial AWS Region to process the request\", and manual Region enablement is not required. You choose a profile, not a failover policy ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html), [Bedrock inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html)).",
      "rel_load_balancing_config": "not_configurable",
      "rel_load_balancing_note": "An inference profile \"distributes model invocation requests across these Regions\", with no weights or policy keys exposed ([Bedrock inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html)).",
      "architecture_class": "hyperscaler_api",
      "architecture_note": "A managed AWS regional service invoked through AWS SDKs; no runtime disclosed ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "`no` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported via `InvokeModelWithResponseStream` and `ConverseStream`. Caveats: check `responseStreamingSupported` from `GetFoundationModel` before assuming a model streams; \"The AWS CLI doesn't support streaming operations in Amazon Bedrock, including `ConverseStream`\"; mid-stream failures arrive as `modelStreamErrorException` (HTTP `424`, \"A streaming error occurred. Retry your request.\"); requires `bedrock:InvokeModelWithResponseStream` ([Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Sensitive information filters detect PII entity types and custom regex entities, with per-entity behaviour of `BLOCK` or `MASK` on both input and output ([Sensitive information filters](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html), [Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))",
      "guard_injection": "sync_block",
      "guard_injection_note": "\"Prompt attack\" is one of the content-filter categories with configurable strength thresholds; the detection mechanism is an AWS-managed classifier and is not described in detail ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "guard_content": "sync_block",
      "guard_content_note": "Content filters cover hate, insults, sexual, violence, misconduct, and prompt attacks, with configurable thresholds applied to prompts and responses ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Customer policy forms are denied topics (natural-language topic definitions), word filters including custom word/phrase lists (profanity, competitor names), custom regex entities, and Automated Reasoning checks for policy validation ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "IAM identity-based policies gate `bedrock:InvokeModel` on specific model resources, and IAM condition keys can additionally require that a guardrail be attached to the call ([IAM identity-based policy examples](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "No documented timeout/fail-open behaviour for the guardrail evaluation itself ([Guardrail components](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html))",
      "log_content": "nothing",
      "log_content_note": "Model invocation logging is disabled until the customer enables it; once enabled it captures full input and output JSON (payloads above 100 KB are stored in S3) to CloudWatch Logs and/or S3, and can include guardrail trace detail ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html), [IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Leaving model invocation logging off keeps CloudWatch operational metrics without any prompt/completion storage; the feature is opt-in per account/region ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))",
      "tracing_support": "otel",
      "tracing_note": "`n.a.` from the fetched pages — guardrail traces are logged as part of invocation logs, but OpenTelemetry support is not documented on the pages fetched ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))",
      "export_destinations": [
        "CloudWatch Logs",
        "S3",
        "CloudTrail"
      ],
      "export_destinations_note": "Amazon CloudWatch Logs and Amazon S3 for invocation logs ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html)); AWS CloudTrail records guardrail configuration and enforcement actions ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` on the fetched pages; the closest documented mechanism is `ApplyGuardrail`, which evaluates text without invoking a model and can therefore be run over stored traffic ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/))",
      "observability_requires_saas": null,
      "log_retention_note": "Determined by the destination the customer chooses (CloudWatch Logs group retention or S3 lifecycle policy); no Bedrock-side default window is stated ([Model invocation logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` for numeric figures. Latency-optimized inference is described only qualitatively (\"faster response times\", \"significantly reduced latency without compromising accuracy\") and per-request latency is reported back as `metadata.metrics.latencyMs` ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html), [Bedrock ConverseStream API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` Throughput is a purchasing construct: \"To increase throughput for a model, you can purchase Provisioned Throughput\", and \"Inference profiles currently don't support Provisioned Throughput\" ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html)).",
        "benchmark_independence": "No figures published, vendor or otherwise ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html))."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "sync_block",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Cross-Region inference adds no routing cost, is billed at the source Region's price, stays on the AWS network, is encrypted in transit, and is auditable via CloudTrail `additionalEventData.inferenceRegion` ([Bedrock cross-Region inference](https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference.html)). Latency-optimized inference was in preview on the fetched page and is limited to specific models/regions ([Bedrock latency-optimized inference](https://docs.aws.amazon.com/bedrock/latest/userguide/latency-optimized-inference.html)).",
      "guard_notes": "Two features stand out for auditors: `ApplyGuardrail` lets the same policy be enforced for models running outside Bedrock, and IAM condition-based enforcement can make an un-guardrailed `InvokeModel` call impossible rather than merely discouraged ([IAM policy-based enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/)).",
      "fit_agent_features_note": "Amazon Bedrock Mantle supports stateful conversation management, `previous_response_id`, streaming, multi-turn interactions, client-side tool use, background processing, and asynchronous inference on `bedrock-mantle`; server-side tools are unavailable on `bedrock-runtime`. AgentCore additionally documents stateful MCP sessions, elicitation, sampling, progress notifications, managed memory and session state, and multi-step agent workflows. ([Amazon Bedrock Mantle agent capabilities](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. Authentication uses a Bedrock short-term API key generated in the console ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))",
      "fit_client_sdk_langs": [
        "C++",
        "Go",
        "Java",
        "JavaScript",
        "Kotlin",
        ".NET",
        "PHP",
        "PowerShell",
        "Python",
        "Ruby",
        "Rust",
        "SAP ABAP",
        "Swift"
      ],
      "fit_client_sdks_note": "The OpenAI Python SDK is documented with `from openai import OpenAI`; the examples require Python 3.7 or later and use `pip install openai boto3`. AWS SDKs are available for C++, Go, Java, JavaScript, Kotlin, .NET, PHP, PowerShell, Python/Boto3, Ruby, Rust, SAP ABAP, and Swift; the JavaScript v3 Bedrock Runtime package is `@aws-sdk/client-bedrock-runtime`, with classes including `BedrockRuntimeClient`, `ConverseCommand`, `ConverseStreamCommand`, and `InvokeModelCommand`. ([Amazon Bedrock SDK general information](https://docs.aws.amazon.com/bedrock/latest/userguide/sdk-general-information-section.html))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Amazon Bedrock supports AWS credentials for HTTP requests, while OpenAI SDK usage requires an Amazon Bedrock API key. For AgentCore Gateway, IAM SigV4 is the default inbound authorization mechanism and IAM-based authentication is supported. ([Amazon Bedrock authentication and endpoints](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))",
      "fit_cloudflare_workers": "third_party_documented",
      "fit_cloudflare_workers_note": "Documented by Cloudflare: Bedrock can be called from a Cloudflare Worker through Cloudflare AI Gateway using the `aws4fetch` library for client-side AWS SigV4 signing. ([Cloudflare AI Gateway - Amazon Bedrock provider](https://developers.cloudflare.com/ai-gateway/usage/providers/bedrock/))",
      "fit_credit_card": "required_for_model_access",
      "fit_credit_card_note": "Not stated in the quickstart; the pricing page says \"Get started for free.\" However model access requires \"Valid AWS Marketplace payment method.\" ([Amazon Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))",
      "fit_ecosystem_notes": "Amazon Bedrock provides managed, serverless foundation models through unified `bedrock-runtime` and `bedrock-mantle` endpoints; `bedrock-runtime` is recommended for new applications, while Projects are created only on `bedrock-mantle`. The OpenAI-compatible endpoints are powered by Mantle, have endpoint-specific quotas, and are available in the documented AWS Regions; Amazon Bedrock Knowledge Bases can use Amazon S3 and Amazon OpenSearch Serverless. ([Amazon Bedrock Mantle overview](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))",
      "fit_eval_prereqs": "An AWS account (step 1) and Python already installed; an SDK for the chosen API; environment variables for the API key. Marketplace-related IAM permissions matter for model access ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "partial",
      "fit_eval_snippet_note": "Partially: the page instructs you to save Python code as `bedrock-first-request.py` and run `python3 bedrock-first-request.py`, and recommends the `bedrock-runtime` endpoint, but no request snippet was rendered in the fetched content ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))",
      "fit_gateway_platform": "platform_gateway",
      "fit_gateway_platform_note": "Amazon Bedrock AgentCore Gateway is the documented gateway platform. The evidence does not document Kong, APISIX, Envoy, Istio, or nginx gateway plugins for Amazon Bedrock. ([AgentCore Gateway MCP server](https://docs.aws.amazon.com/prescriptive-guidance/latest/semantic-layer-agentic-ai-ontology-reasoning-virtual-knowledge-graph/mcp-server.html))",
      "fit_helm_chart": "EKS Helm documentation only; no Bedrock chart or operator",
      "fit_kubernetes": "generic_platform_only",
      "fit_kubernetes_note": "Amazon EKS documentation describes Helm for installing and managing charts, including charts from the `eks-charts` GitHub repository or ArtifactHub. No official Amazon Bedrock gateway Helm chart or operator is documented. ([Helm on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/helm.html))",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Amazon Bedrock AgentCore supports MCP Streamable HTTP transport, normally with `stateless_http=True`; the `/mcp` endpoint supports MCP RPC messages, tool invocation and management, capability discovery, resource access, and multi-step workflows. Stateful sessions use the `Mcp-Session-Id` header, and the required container port is `8000`. ([AgentCore Runtime MCP protocol contract](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-mcp-protocol-contract.html))",
      "fit_meta": {
        "not_documented_fields": []
      },
      "fit_model_gate": "approval_required",
      "fit_model_gate_note": "Yes, partly automated: \"Access to all Amazon Bedrock foundation models is enabled by default with the correct AWS Marketplace permissions\" and \"When you invoke a third-party model for the first time in your account, Amazon Bedrock automatically initiates the subscription process in the background\"; Anthropic models require a First Time Use (FTU) form (companyName, companyWebsite, intendedUsers, industryOption, useCases) after which \"Access to the model is granted immediately after use case details are successfully submitted\"; subscription setup \"may take up to 15 minutes.\" GovCloud console access is a 10-step procedure ending \"the Access status changes to Access granted or Available to request,\" and a 5-step programmatic procedure exists ([Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html)). Non-GA models are gated: \"Access to Claude Mythos 5 and Claude Mythos Preview is gated and requires approval. Contact your Anthropic account team to request access on Bedrock.\" ([Amazon Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/))",
      "fit_onboarding_notes": "Multiple API surfaces are available (Anthropic-native Messages, OpenAI-compatible Responses, Chat Completions, Converse, Invoke), so SDK choice depends on the API. After permissions are granted, completion \"may take up to 2 minutes.\" Availability responses expose `agreementAvailability: AVAILABLE`, `authorizationStatus: AUTHORIZED`, `entitlementAvailability: AVAILABLE`, `regionAvailability: AVAILABLE` ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Existing OpenAI SDKs can be used by changing the base URL and API key. Documented base URLs are `https://bedrock-runtime.<your-region>.amazonaws.com/openai/v1` (recommended for new applications) and `https://bedrock-mantle.<your-region>.api.aws/v1`; the OpenAI SDK requires an Amazon Bedrock API key. ([Amazon Bedrock OpenAI-compatible APIs](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html))",
      "fit_prod_prereqs": "IAM roles / temporary credentials instead of the short-term API key; for managing model access via SDK/CLI: attach `AmazonBedrockFullAccess`, configure the AWS SDK, AWS CLI version 2.27.42 or later, and know the model ID; Marketplace permissions `aws-marketplace:Subscribe`, `Unsubscribe`, `ViewSubscriptions`; a valid AWS Marketplace payment method; GovCloud third-party models require access enabled in both the linked commercial account and the GovCloud account ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html), [Request access to models - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "cloud_console",
      "fit_python_frameworks": [
        "LlamaIndex",
        "Strands"
      ],
      "fit_python_frameworks_note": "LlamaIndex integrations are documented for Amazon Bedrock, including `Bedrock`, `BedrockEmbedding`, `AmazonKnowledgeBasesRetriever`, `RouterQueryEngine`, `SubQuestionQueryEngine`, and `ReActAgent`. AWS Strands is also explicitly documented as a supported agent framework for AgentCore Runtime. ([LlamaIndex and Amazon Bedrock RAG](https://aws.amazon.com/blogs/machine-learning/build-powerful-rag-pipelines-with-llamaindex-and-amazon-bedrock/))",
      "fit_quickstart_steps": 5,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "platform_provider_resources",
      "fit_terraform_note": "The HashiCorp AWS provider documents the Terraform resource `aws_bedrockagentcore_registry`. The resource is deprecated; the evidence states that functionality will move from the `bedrock-agentcore` namespace to `agent-registry` on August 6, 2026, while the resource continues to work until September 17, 2026. ([Terraform AWS AgentCore Registry resource](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/bedrockagentcore_registry))",
      "fit_terraform_ref": "hashicorp/aws resource aws_bedrockagentcore_registry — deprecated, moving to agent-registry on 2026-08-06 and working until 2026-09-17",
      "fit_third_party_documented": {
        "cloudflare_workers": "Cloudflare"
      },
      "fit_vendor_time_claim": "Vendor claim (page description), verbatim: \"Get started with Amazon Bedrock in minutes.\" ([Quickstart - Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Per an AWS Vercel case study, Vercel AI SDK integrates with Amazon Bedrock through the dedicated provider package `@ai-sdk/amazon-bedrock`. ([AWS Vercel case study](https://aws.amazon.com/solutions/case-studies/vercel-case-study/))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/amazon-bedrock"
    },
    {
      "slug": "azure-ai-foundry",
      "name": "Azure AI Foundry",
      "company": "Microsoft",
      "tagline": "Microsoft's Azure platform for deploying models from its own and partner catalogs, now branded Microsoft Foundry.",
      "category": "cloud_platform",
      "deployment": "both",
      "license": "Proprietary",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://azure.microsoft.com/en-us/products/ai-foundry",
      "url_docs": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
      "url_pricing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
      "url_status": null,
      "url_github": null,
      "url_changelog": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry",
      "url_models_api": null,
      "github_stars": null,
      "model_count": 11000,
      "model_count_note": "Vendor states \"11K+ Foundry Models\" / \"over 11,000+ models\", spanning Azure OpenAI, Microsoft, and partner catalogs.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": 0,
      "free_tier": "Azure free account gives $200 credit valid 30 days; the Foundry platform itself is free to use and you pay only for the models and compute you deploy.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Region-pinned Managed Compute GPU deployments",
          "amount": "Billed per GPU-hour (A100, H100, H200, MI300 SKUs)"
        },
        {
          "label": "Provisioned throughput (PTU)",
          "amount": "Reserved capacity billed per PTU rather than per token"
        }
      ],
      "self_host_cost_note": "Mostly managed, with Foundry Local for on-device execution. Microsoft sets its own per-token prices rather than marking up a third party; Global Batch and Data Zone Batch deployments are 50% below the standard rate with a 24-hour completion target, and DeveloperTier deployments are cheap but capped at 24 hours with no SLA and no data-residency guarantee.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "Azure OpenAI (AzureOpenAI client, /openai/deployments/{deployment})",
        "Azure AI Model Inference",
        "Foundry Agent Service"
      ],
      "sdk_note": "Azure clients are built on the official OpenAI packages via the AzureOpenAI class; endpoints take the form https://RESOURCE-NAME.openai.azure.com with a deployment name in the path rather than a model name.",
      "cap_failover": null,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": null,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": true,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": null,
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": null,
      "eu_residency": true,
      "eu_residency_note": "DataZoneStandard, DataZoneProvisionedManaged and DataZoneBatch deployments keep processing inside the EU data zone, which follows the Azure EU Data Boundary and may include EFTA countries (Norway, Switzerland); single-region Standard deployments process in the customer-specified geography, while Global deployments may process anywhere. Abuse-monitoring data for EEA deployments is reviewed only by Microsoft employees located in the EEA.",
      "zdr": true,
      "zdr_note": "Managed customers may apply for modified abuse monitoring; once approved, the storage and human-review step is not performed. It is an application, not a switch.",
      "air_gapped": null,
      "best_for": "Microsoft-centric enterprises that want first-party OpenAI models plus a very large partner catalog under Azure governance.",
      "strengths": [
        "Largest published catalog of the three hyperscalers at 11,000+ models, with a built-in Model Router",
        "Fine-grained residency control through Global, Data Zone, single-region and Azure Government deployment types",
        "Modified abuse monitoring gives approved customers an effective zero-retention path",
        "Azure OpenAI clients are thin wrappers on the official OpenAI SDKs, so migration is mostly a base-URL change",
        "Platform surface is free; you pay only for deployed models and compute"
      ],
      "weaknesses": [
        "No cross-cloud routing: you can only reach models Microsoft hosts, so multi-cloud failover needs a separate gateway",
        "Heavy Azure coupling — resources, quotas, deployments and PTU commitments are all Azure-specific",
        "Constant renaming (Azure AI Studio to Azure AI Foundry to Microsoft Foundry) makes docs and portal naming inconsistent",
        "Compliance attestations are documented at the Azure service level, not specifically under the Foundry brand, so scope needs verification per service"
      ],
      "avoid_if": "You need vendor-neutral routing across clouds, or you want a single flat token price without capacity planning around PTUs and deployment types.",
      "reviews": [
        {
          "url": "https://blog.chiffers.com/new-vs-old-azure-ai-foundry-how-it-affects-your-code/",
          "why": "Practitioner maps the breaking changes between old and new Foundry projects at the code level (azure-ai-projects 2.x, AIProjectClient, Responses vs Assistants API).",
          "date": "2026-06-09",
          "type": "written",
          "title": "New vs Old Azure AI Foundry: How it affects your code",
          "publisher": "blog.chiffers.com"
        },
        {
          "url": "https://www.infoq.com/news/2025/05/azure-ai-foundry-agents-ga/",
          "why": "InfoQ's Steef-Jan Wiggers details what GA changed in Foundry's agent service - connected agents, SDK surface, A2A support - rather than restating Microsoft's messaging.",
          "date": "2025-05-26",
          "type": "written",
          "title": "Azure AI Foundry Agent Service GA Introduces Multi-Agent Workflows",
          "publisher": "InfoQ"
        },
        {
          "url": "https://www.youtube.com/watch?v=Sq8Cq7RZM2o",
          "why": "Independent Azure trainer walks the Foundry portal, model catalog, playground, evaluations and agents, flagging where the abstraction leaks.",
          "date": "2025-03-05",
          "type": "video",
          "title": "Azure AI Foundry Overview",
          "publisher": "YouTube - John Savill's Technical Training"
        },
        {
          "url": "https://www.reddit.com/r/AZURE/comments/1vea8u8/azure_ai_foundry_gpt4o_to_gpt51_migration_changed/",
          "why": "13-comment migration thread with concrete constraints: datazone-standard deployments lacking GPT-5.1, and quota/capacity negotiations through account managers.",
          "date": "2026-08-03",
          "type": "community",
          "title": "Azure AI Foundry: GPT-4o to GPT-5.1 migration changed our RAG agent's response style",
          "publisher": "Reddit - r/AZURE"
        },
        {
          "url": "https://www.reddit.com/r/ArtificialInteligence/comments/1nhic70/azure_ai_foundry_is_awful/",
          "why": "12-comment complaint thread with specifics: an agent silently losing its AI Search tool connection and 800k-token context configuration problems.",
          "date": "2025-09-15",
          "type": "community",
          "title": "Azure AI Foundry is awful",
          "publisher": "Reddit - r/ArtificialInteligence"
        }
      ],
      "sources": {
        "zdr": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy",
        "name": "https://azure.microsoft.com/en-us/products/ai-foundry",
        "soc2": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-soc-2",
        "company": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
        "tagline": "https://azure.microsoft.com/en-us/products/ai-foundry",
        "sdk_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages",
        "url_docs": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
        "url_home": "https://azure.microsoft.com/en-us/products/ai-foundry",
        "zdr_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy",
        "cap_batch": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types",
        "cap_evals": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
        "free_tier": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "hipaa_baa": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-hipaa-us",
        "deployment": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
        "extra_fees": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "hq_country": "https://en.wikipedia.org/wiki/Microsoft",
        "zdr_status": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy",
        "model_count": "https://azure.microsoft.com/en-us/products/ai-foundry",
        "url_pricing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "api_surfaces": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages",
        "eu_residency": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types",
        "product_fate": "https://learn.microsoft.com/en-us/azure/foundry/how-to/high-availability-resiliency",
        "regions_note": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy",
        "cap_image_gen": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "url_changelog": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry",
        "vendor_status": "https://learn.microsoft.com/en-us/azure/foundry/whats-new-foundry",
        "cap_embeddings": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/azure-government",
        "cap_guardrails": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy",
        "certifications": "https://learn.microsoft.com/en-us/azure/compliance/offerings/offering-fedramp",
        "config_as_code": "https://learn.microsoft.com/en-us/azure/foundry/how-to/create-resource-terraform",
        "retention_days": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy",
        "sla_uptime_pct": "https://azure.microsoft.com/en-us/products/ai-foundry/models/openai",
        "logs_by_default": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/abuse-monitoring",
        "model_count_note": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "cap_observability": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry",
        "cap_pii_redaction": "https://learn.microsoft.com/en-us/azure/ai-foundry/whats-new-azure-ai-foundry",
        "eu_residency_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy",
        "last_release_date": "https://learn.microsoft.com/en-us/azure/foundry/whats-new-foundry",
        "openai_compatible": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/supported-languages",
        "subprocessors_url": "https://www.microsoft.com/en-us/trust-center/privacy/data-access",
        "cap_load_balancing": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types",
        "seat_fee_usd_month": "https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-azure-ai-foundry",
        "guardrails_location": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/content-safety/data-privacy",
        "requires_vendor_sdk": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints",
        "self_host_cost_note": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/deployment-types",
        "azure_prompt_caching": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/prompt-caching",
        "azure_foundry_pricing": "https://azure.microsoft.com/en-us/pricing/details/microsoft-foundry/",
        "data_export_available": "https://learn.microsoft.com/en-us/azure/foundry/how-to/diagnostic-logging",
        "prompt_transits_vendor": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy",
        "cap_conditional_routing": "https://azure.microsoft.com/en-us/pricing/details/phi-3/",
        "trains_on_customer_data": "https://learn.microsoft.com/en-us/azure/foundry/responsible-ai/openai/data-privacy",
        "azure_foundry_models_pricing": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_batch": "2026-08-29",
        "cap_evals": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "eu_residency": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:34.758692+00:00",
      "vendor_status": "hyperscaler",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "A first-party Microsoft Azure product line with monthly feature roll-ups. Worth noting that the SLA does not cover everything: Microsoft states the Agent Service carries no availability or state-durability SLA.",
      "last_release_date": "2026-08-13",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://azure.microsoft.com/en-us/products/ai-foundry/models/openai",
      "claim_basis": {},
      "last_release_note": "Microsoft publishes no per-release dates, so this is when the “What's new in Microsoft Foundry” page was last updated.",
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Microsoft hosts the models it sells inside its own Azure environment, and those models do not interact with services operated by the model's original provider.",
      "logs_by_default": "none",
      "logs_note": "Models are stateless and prompts are not stored by the system for automated review. Abuse monitoring is the exception: flagged content goes to a store logically separated per customer resource, reachable only by authorised Microsoft staff through secure workstations with just-in-time approval.",
      "retention_days": null,
      "retention_note": "No day count is published on the current page. It states where flagged content is stored and who may see it, but not for how long — do not assume the 30-day figure that appeared in older versions.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "Prompts, completions, embeddings and training data are not used to train generative foundation models, or to improve Microsoft or third-party products, without your explicit permission or instruction.",
      "subprocessors_url": "https://www.microsoft.com/en-us/trust-center/privacy/data-access",
      "regions_note": "Processing stays in your specified geography — with a significant exception. Any deployment type labelled Global may process prompts and responses in any geography, and DataZone widens the boundary too.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "the Azure SOC offering page did not load during research",
          "status": "not_published"
        },
        "fedramp": {
          "detail": "High P-ATO issued by the JAB for Azure and Azure Government",
          "status": "certified"
        },
        "gdpr_dpa": {
          "detail": "Microsoft Data Protection Addendum",
          "status": "certified"
        },
        "iso27001": {
          "detail": "audited certification",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "the HIPAA page carried no BAA statement",
          "status": "not_published"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "Content Safety models run inside Microsoft's Azure infrastructure. Data stays in Azure and in the region you chose, and is not stored or human-reviewed — but the filtering happens on Microsoft's side.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference with an explicitly free platform layer. 'The Foundry platform is free to use and explore'; individual features billed at their normal rates. The gateway/routing layer is $0; models are billed separately.",
      "min_commitment": "None required. Microsoft Agent pre-purchase plan is a 1-year metered plan buying Agent Commit Units at tiered discounts — 20,000 ACUs = 5%, 100,000 = 10%, 500,000 = 15% (dollar prices shown as '$-').",
      "overage_note": "None stated on either pricing page.",
      "cache_support": "prefix",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": 0,
      "cache_pricing_note": "Exact-prefix prompt caching, enabled by default, no opt-out. The first 1,024 tokens must be identical; a single character difference in the first 1,024 tokens = miss. Hits extend every 128 identical tokens. Caches clear within 5–10 minutes of inactivity and always within an hour, and are not shared across subscriptions. Discount: Standard cached-token discount is not published as a number; Provisioned deployments get up to 100% discount on input tokens. No cache-write charge documented. On Provisioned capacity, cached input can be effectively free.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Not stated as a cost control.",
          "tier": null,
          "supported": null
        },
        "spend_webhooks": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated on the pricing pages fetched.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated on the pricing pages.",
      "cost_export": "Not stated on the pricing pages.",
      "product_kind_note": "Microsoft calls it \"Microsoft Foundry Models\", \"your one-stop destination for discovering, evaluating, and deploying powerful AI models\" and \"the hub for discovering and using a wide range of models to build generative AI applications\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). It is a cloud model platform with a catalog split between models Azure sells and hosts and partner/community models; a `model-router` model exists inside it but the product is not a third-party routing proxy ([Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26).",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "Serverless deployment types are Global/Data Zone/Regional Standard, Provisioned and Batch variants plus Developer ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Gating is by subscription, quota and region rather than SKU tier: pay-per-token requires the billing account's country to be one where the provider makes the offer available and a project resource in the model's region; some models need quota requests (Tier 5/6 have quota by default) or registration (`computer-use-preview`, `grok-code-fast-1`, `grok-4`), and partner/community models \"require Azure Marketplace\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview); [Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)). Managed compute deploys to \"Azure Machine Learning managed compute for real-time inference\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Two coexisting surfaces: a single Azure-schema models inference endpoint (\"usually with the form `https://<resource-name>.services.ai.azure.com/models`\") that \"allows customers to use a single endpoint with the same authentication and schema\" across deployed models, and the Azure OpenAI API which \"exposes the full capabilities of OpenAI models\" while \"non-OpenAI models can also be used for compatible functionalities\" ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26). It is not a plain OpenAI base-URL swap: routing is by deployment name (case-insensitive) matched to the request's `model`/`name` field, each Azure OpenAI deployment has its own URL (`/deployments/<deployment-name>`), and requests carry an `api-version` query parameter ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference)). SDKs: `azure-ai-inference` (Python/JS/C#/Java) for the models endpoint and `openai` / `Azure.AI.OpenAI` for the Azure OpenAI surface ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference)).",
      "model_count_low": 10000,
      "model_count_high": 10000,
      "model_count_source_note": "\"over 10,000 models\" in the catalog, with roughly \"50 new models\" published per month ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28); the partner page describes itself as listing only \"a selection of Foundry Models from partners and community\" ([Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party platform plus partner marketplace, so no routed-provider total is published; named providers include Azure OpenAI, Microsoft, Anthropic, Mistral, Meta, Cohere, NVIDIA, DeepSeek and Hugging Face ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).",
      "first_party_vs_partner": "Explicit two-way split. \"Foundry Models sold by Azure\" are hosted and sold by Microsoft under Microsoft Product Terms and billed as \"First Party Consumption Services\" via Azure meters — \"Models sold by Azure are also hosted by Azure and operated by Azure as part of the Foundry Models service\", covering all Azure OpenAI models plus selected models from top providers. \"Foundry Models from partners and community\" come from third-party organizations, partners, research labs and community contributors (including Anthropic's Claude family and Hugging Face open models), \"require Azure Marketplace\", and their providers \"define the license terms and set the price for use of their models using Azure Marketplace\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).",
      "custom_endpoint_support": "Partly: customers can deploy model weights themselves onto managed compute VMs inside their Foundry resource, which then exposes a REST inference API ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Registering an external customer-hosted endpoint (vLLM, Ollama, private URL) with the Foundry inference endpoint: n.a. on the pages fetched.",
      "byok_mode": "not_applicable",
      "merchant_of_record": "Microsoft/Azure for models sold by Azure, billed as \"First Party Consumption Services\" via Azure meters; for partner and community models the provider \"set[s] the price for use of their models using Azure Marketplace\", so charges flow through Azure Marketplace (and Student, Visual Studio Enterprise and Free-credit subscriptions cannot purchase Marketplace SaaS offers) ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)). Serverless usage is billed on inputs/outputs (typically tokens) and managed compute on VM core hours ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)).",
      "key_management_note": "Two authentication methods: API keys sent in the `api-key` HTTP header (\"all API requests must include the API Key in the `api-key` HTTP header\") and Microsoft Entra ID bearer tokens in the `Authorization` header ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24); the models inference SDKs use `AzureKeyCredential` / `AZURE_INFERENCE_CREDENTIAL` ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26). Keys are scoped to the Foundry/Azure OpenAI resource, and there are no upstream third-party secrets to vault since Microsoft hosts the models ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Request traffic can be confined to a private endpoint with public network access disabled on the hub (propagation up to five minutes), with the caveat that Azure OpenAI On Your Data is unavailable for serverless deployments in private hubs ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)). Customer-managed KMS specifics: n.a.",
      "api_openai_chat_completions_note": "Yes, two paths: the unified models inference endpoint `POST https://<resource>.services.ai.azure.com/models/chat/completions?api-version=...` and the Azure OpenAI form `POST https://<resource>.services.ai.azure.com/openai/deployments/<deployment>/chat/completions?api-version=2024-10-21` ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).",
      "api_anthropic_messages_note": "n.a. Claude models are offered as partner models but no Anthropic Messages endpoint is documented on the pages fetched ([Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners); [How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).",
      "api_openai_responses_note": "n.a.: the `/v1/responses` path is not mentioned on the pages fetched, though the Azure OpenAI v1 APIs are referenced generally ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), last updated 2026-06-24; [Foundry Models from partners and community](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-from-partners)).",
      "api_embeddings_note": "Yes: the models inference endpoint \"supports the following modalities: Text embeddings… Image embeddings… Chat completions\" ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26); embeddings are also used for RAG over custom data ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).",
      "api_images_note": "Yes: `.../openai/deployments/{deployment-id}/images/generations` and `.../images/edits` (documented for FLUX models), plus DALL·E deployments ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24).",
      "api_audio_note": "Yes, STT documented: audio transcription (\"Transcribes audio into the input language\") and translation (\"Transcribes and translates input audio into English text\") as `multipart/form-data` requests with `file`, `prompt`, `response_format`, `temperature`, `language`, returning `json`, `text`, `srt`, `verbose_json` or `vtt` ([Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24). TTS specifically: n.a. on the pages fetched (audio generation appears only as a catalog filter) ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28).",
      "api_batch_note": "Yes at deployment level: Global Batch and Data Zone Batch deployment types exist and the Azure OpenAI API \"supports additional features like assistants, threads, files, and batch inference\"; no separate batch endpoint path is documented ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28; [How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26).",
      "deployment_modes_note": "hosted cloud service in three shapes: serverless deployments where \"Microsoft hosts the models in a managed Azure infrastructure\"; managed compute, where weights are deployed to \"dedicated virtual machines with managed compute\" billed by \"virtual machine core hours\"; and \"instant access (preview)\" with \"no deployment required\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview), 2026-07-28). Private networking is supported (disable public network access on the hub and use \"a private endpoint for the hub\"), but the page \"does not describe this as hybrid, VPC, on-premises, or air-gapped deployment\" ([Foundry Models overview](https://learn.microsoft.com/en-us/azure/foundry/concepts/foundry-models-overview)). Local execution exists for at least one model via Foundry Local (`gpt-oss-20b` \"available via managed compute and Foundry Local\") ([Foundry Models sold by Azure](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure), 2026-08-26).",
      "byok_mode_note": "not_applicable: Foundry is the model host, authenticated with Azure resource credentials rather than upstream provider keys ([How to use the Foundry Models inference endpoints](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/inference), 2025-06-26; [Azure OpenAI reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/reference), 2026-06-24).",
      "access_meta": {
        "qualified": [
          "api_openai_chat_completions",
          "api_anthropic_messages",
          "api_audio",
          "api_batch"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` on the fetched Foundry pages ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented` as a platform setting; default count and backoff: `n.a.` Microsoft's own gateway guidance tells you to build it yourself and to honor the `Retry-After` header from Azure OpenAI responses ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "`per_request` and `dashboard_only`, as a SINGLE-ALTERNATIVE spillover: deployment property `spilloverDeploymentName`, or per-request header `x-ms-spillover-deployment`, targeting one corresponding standard deployment in the same Azure OpenAI resource. If both are set, the deployment property wins. Triggers: `429` (PTUs exhausted), `400` (long-context request), `500`, `503`. If the spillover request also fails, the original PTU response is returned and `x-ms-spillover-error` carries the spillover status code. Not available for the Responses API ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management), [Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)).",
      "rel_fallback_shape": "single",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "`not_configurable` / not provided by the platform. Microsoft states plainly that Azure OpenAI \"doesn't provide any sort of health check endpoint to preemptively know whether it's available to handle requests\", and recommends that your gateway circuit-break on `429`/`500`/`503`, honor `Retry-After`, and remove faulted backends from the pool — with Azure API Management offering \"built-in circuit breaker functionality\" ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).",
      "rel_multi_region_config": "not_configurable",
      "rel_multi_region_note": "`not_configurable` at the deployment level: the vendor runs the regions. Global Provisioned traffic is \"Routed across Azure regions globally\", Data Zone Provisioned \"Stays within a geographic zone (US or EU)\", Regional Provisioned \"Stays in the deployment's specific Azure region\" ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)). Spillover itself cannot cross resources — both deployments must be in the same Azure OpenAI resource with matching data-processing level ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)). Cross-region active-passive failover is described only as a gateway pattern you implement ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).",
      "rel_load_balancing_config": "not_configurable",
      "rel_load_balancing_note": "`not_configurable` natively; no weights. Global Provisioned routes across regions automatically with no exposed policy ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)); spillover only prioritizes the provisioned deployment before overflowing to standard ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)). Multi-backend round-robin is a gateway pattern you build, and Microsoft warns endpoints in a pool must serve \"the same model at the same version\" ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).",
      "architecture_class": "hyperscaler_api",
      "architecture_note": "Managed Azure service; deployment types are selected via `sku-name` (`GlobalProvisionedManaged`, `ProvisionedManaged`), and no runtime is disclosed ([Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "`no` for the Foundry inference plane. Managed compute and serverless API endpoints are documented as Azure-hosted deployment options ([Azure AI Foundry deployments overview](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/deployments-overview)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported via `stream: true`; tokens return \"as soon as they're available\", which \"doesn't change the time to get all the tokens\" but \"reduces the time for first response\". Documented caveat: \"Many clients and intermediary layers have timeouts on individual calls\" and \"Long generation calls might be canceled due to client-side time outs\" — streaming is recommended partly to keep those connections alive ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)). Model router forwards a standard chat-completions response but drops several parameters for o-series reasoning models (`Temperature`, `Top_P`, `stop`, `presence_penalty`, `frequency_penalty`, `logit_bias`, `logprobs`) and does not support `reasoning_effort` ([Azure model router](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/model-router)).",
      "guard_pii": "sync_observe",
      "guard_pii_note": "The content-filtering system includes a PII filter applied to model output; the docs describe detection and annotation and do not state that Foundry redacts the text ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Prompt Shields \"detects and blocks\" user prompt attacks and indirect (cross-domain) prompt injections; optional binary classifier models flag jailbreak risk, and indirect-attack detection requires document delimiters in the prompt ([Prompt Shields / jailbreak detection](https://learn.microsoft.com/en-us/azure/ai-services/content-safety/concepts/jailbreak-detection), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))",
      "guard_content": "sync_block",
      "guard_content_note": "Azure AI Content Safety runs an ensemble of classification models over prompt and completion \"synchronously as the service processes prompts to generate content\", filtering hate, sexual, violence, and self-harm at configured severity thresholds; audio models such as Whisper are excluded ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Customer-defined policy takes the form of configurable severity thresholds per category and custom blocklists; separate protected-material and code classifiers can be enabled ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering), [Foundry Models content filter](https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/concepts/content-filter))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "`yes`, structurally — models are consumed as named deployments on a Foundry resource, and access is governed by Azure RBAC on those resources with diagnostic/policy control via Azure Policy; an explicit \"model allowlist\" UI is not documented on the fetched pages ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` — no fail-open/fail-closed statement for the filtering service ([Content filtering](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/content-filtering))",
      "log_content": "metadata_only",
      "log_content_note": "Platform metrics and activity logs are collected automatically, but resource logs \"aren't collected and stored until you create a diagnostic setting\"; prompts and completions are not stored by the content classifiers (\"No prompts or generated content are stored in the content classifier models\"). Abuse monitoring may retain a sample of prompts and completions for human review in a per-resource, per-geography store unless modified abuse monitoring is approved ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Do not create (or scope down) diagnostic settings and platform metrics still flow; abuse-monitoring storage and human review can be removed via approved modified abuse monitoring ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai), [Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry natively, using the OTel GenAI semantic conventions, with multi-agent spans surfaced in Azure Monitor Application Insights ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))",
      "export_destinations": [
        "Azure Monitor",
        "Log Analytics",
        "Event Hubs",
        "Azure Storage"
      ],
      "export_destinations_note": "Via diagnostic settings: Azure Monitor Logs / Log Analytics, Azure Storage, Azure Event Hubs (the route to non-Azure partner systems), and certain Microsoft monitoring partners; all metrics are exportable with diagnostic settings, plus Log Analytics workspace data export ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai)); Application Insights for traces ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — no feedback/score endpoint documented on the fetched pages ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` from the fetched pages ([Agent tracing concepts](https://learn.microsoft.com/en-us/azure/foundry/observability/concepts/trace-agent-concept))",
      "observability_requires_saas": null,
      "log_retention_note": "Determined by the destination (Log Analytics workspace, Storage, Event Hubs) rather than by Foundry; no service-side default stated ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` for numeric figures. Azure publishes latency definitions and SLA posture instead: **End-to-end Request Time** (non-streaming, measured by the API gateway), **Time to Response** (first-token, PTU/PTU-managed), **Average Token Generation Rate** (first-to-last token divided by generated tokens); Standard and Batch have \"None\" for a latency SLA while Provisioned and Priority processing have a \"Defined latency target per model\" ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency), [Azure provisioned throughput](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/provisioned-throughput)). Spillover \"might incur additional latency\" ([Azure spillover traffic management](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/spillover-traffic-management)).",
        "multi_surface": {
          "rel_fallback": [
            "per_request",
            "dashboard_only"
          ]
        },
        "throughput_prose": "Capacity-planning tables rather than benchmarks. GPT-4o mini worked examples: 800/150 tokens at 30 RPM = 28,500 total TPM = 15 PTUs; 5,000/50 at 1,000 RPM = 5,050,000 TPM = 140 PTUs; 1,000/300 at 500 RPM = 650,000 TPM = 30 PTUs. \"The number of PTUs scales roughly linearly with call rate when the workload distribution remains constant\", and estimates exclude prompt-caching benefits ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency)).",
        "benchmark_independence": "All figures are vendor (Microsoft) capacity-planning examples, not measured benchmarks and not third-party ([Azure OpenAI performance and latency](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/latency))."
      },
      "guard_pii_default": "sync_observe",
      "guard_injection_default": "sync_block",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Foundry's built-in reliability is deployment-type selection plus single-target spillover; anything resembling a gateway (retry policy, circuit breaking, multi-backend balancing, active-passive regional failover) is documented as something you build in front of it, typically with API Management ([Azure gateway for multiple Azure OpenAI deployments](https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/azure-openai-gateway-multi-backend)).",
      "guard_notes": "Anything typed into the Completions or Chat playground \"generates metrics and log data for your Azure OpenAI resource\", which surprises teams who assume the playground is ephemeral ([Monitor Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/monitor-openai)). Human reviewers only reach flagged data, via point-wise request-ID queries from Secure Access Workstations under JIT approval ([Data privacy](https://learn.microsoft.com/en-us/azure/ai-foundry/responsible-ai/openai/data-privacy)).",
      "fit_agent_features_note": "The documented agent features include stateful multi-turn sessions with `previous_response_id`, manual chaining, parallel multi-agent orchestration, function/tool calling, remote MCP tools, Code Interpreter, image generation, background tasks, streaming, response compaction, hosted agents with session-level state, and Foundry Agent Service threads, messages, and runs. A2A is not documented in the supplied evidence. ([Microsoft Foundry Responses API](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. Authentication uses `DefaultAzureCredential` ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))",
      "fit_client_sdk_langs": [
        "Python",
        "JavaScript",
        ".NET",
        "Java"
      ],
      "fit_client_sdks_note": "Official client libraries include Python `openai`, `azure-ai-projects`, and `azure-identity`; JavaScript `openai`, `@azure/ai-projects`, and `@azure/identity`; .NET `OpenAI`, `Azure.AI.Projects`, and `Azure.Identity`; and Java OpenAI and Azure Identity packages including `com.openai:openai-java` and `com.azure:azure-identity`. The Foundry SDK documents classes such as `AIProjectClient`, `ProjectsClient`, `ProjectsClientBuilder`, and `ChatCompletionsClient`; Foundry Local additionally documents C#, JavaScript, Python, and Rust SDKs. ([Microsoft Foundry SDK overview](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/sdk-overview))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Microsoft Entra ID authentication is supported through `DefaultAzureCredential`, bearer tokens, managed identities, and Azure RBAC/resource role assignments; the documented token scope is `https://ai.azure.com/.default`. Foundry Local also documents Entra authentication with an app registration using `entraAuth.tenantId` and `entraAuth.clientId`. ([Microsoft Foundry model endpoints](https://learn.microsoft.com/en-us/azure/foundry-models/concepts/endpoints))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated; only \"Create one for free.\" / \"create a free account if needed\" ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota), [Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))",
      "fit_ecosystem_notes": "Microsoft Foundry generally requires an Azure subscription, a Foundry resource or project, and deployed models or other Azure resources; deployments are Azure resources subject to Azure policies. Foundry projects expose `https://<resource-name>.services.ai.azure.com/api/projects/<project-name>` and an OpenAI-compatible `/openai/v1` endpoint, while hosted agents run on Foundry-managed isolated Micro VMs with automatic scaling, a dedicated Microsoft Entra agent identity, session-level state, and observability. Foundry Local is a separate self-contained SDK deployment for Windows, macOS, and Linux that does not require the Foundry Local CLI on end-user machines. ([Microsoft Foundry SDK overview](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/sdk-overview))",
      "fit_eval_prereqs": "An Azure subscription (\"create a free account if needed\"); a hub-based project or a Foundry project; for a Foundry project, subscription Owner access. Credit card, billing enablement, cluster and provider key are not mentioned ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `response = chat.complete(model=\"gpt-4o-mini\", messages=[...])` (Python; C#/JavaScript variants shown) ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code))",
      "fit_gateway_platform": "third_party_documented",
      "fit_gateway_platform_note": "Yes (documented by Kong): the Kong `ai-proxy-advanced` plugin has an Azure AI Foundry / Azure OpenAI chat-route example with global, service, route, consumer and consumer-group scoped configuration. ([Kong ai-proxy-advanced Azure chat route example](https://developer.konghq.com/plugins/ai-proxy-advanced/examples/azure-chat-route/))",
      "fit_helm_chart": "Foundry Local ships as an Azure Arc Kubernetes extension (Microsoft.Foundry / inference-operator) on Arc clusters 1.29+; no chart name or repo is stated",
      "fit_kubernetes": "helm_unnamed",
      "fit_kubernetes_note": "Foundry Local can be deployed as an Azure Arc Kubernetes extension with extension type `Microsoft.Foundry`, extension name `inference-operator`, and release namespace `foundry-local-operator`; Helm is also supported, but no chart name or repository is stated. The deployment requires an Azure Arc-enabled Kubernetes cluster running version 1.29 or later. ([Deploy Foundry Local with the Azure Arc extension](https://learn.microsoft.com/en-us/azure/azure-sovereign-clouds/private/foundry-local/deploy-foundry-local-arc-extension))",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Microsoft Foundry supports remote MCP servers through the Responses API using an `mcp` tool with fields including `server_label`, `server_url`, optional `require_approval`, and custom `headers`; approval uses `mcp_approval_request` and `mcp_approval_response`. Foundry Agent Service also supports MCP tool invocation and approval workflows, while Foundry Toolbox can expose managed tools and skills through a single MCP endpoint. ([Use the Responses API with Microsoft Foundry](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/responses))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers"
        ]
      },
      "fit_model_gate": "quota_gated",
      "fit_model_gate_note": "Yes, in the form of deployment plus quota: you must deploy `gpt-4o-mini` before calling it, and rate limits/quota apply (\"Rate limit is exceeded… you want to get more quota\") ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code)). Quota mechanics: \"If instant model requests are throttled, you can request a global quota increase or create a deployment with reserved capacity\"; \"Use the shared quota only for creating temporary test endpoints, not production endpoints\"; \"Some model and region combinations might not support quota increases\"; requesting an increase is a 7-step procedure ending \"Select the Request quota button in the upper-right corner\" ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))",
      "fit_onboarding_notes": "Installs differ by language: `pip install azure-ai-projects azure-ai-inference azure-identity`, `pip install openai azure-ai-projects azure-identity`, `dotnet add package Azure.Identity|Azure.AI.Projects|Azure.AI.Agents.Persistent|Azure.AI.Inference`, or `npm install`. REST access tokens expire in 60–90 minutes. Quota changes \"allow up to 15 minutes for changes to propagate,\" and \"If the issue persists after 24 hours, contact Azure support.\" ([Quickstart: Get started with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code), [Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Microsoft Foundry documents OpenAI SDK compatibility using `base_url`/`baseURL`, with the exact base URL `https://<resource>.openai.azure.com/openai/v1/`; the deployment name is passed in the `model` field and authentication can use `api_key`. Foundry project endpoints also expose an OpenAI-compatible `/openai/v1` endpoint. ([Microsoft Foundry model endpoints](https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/endpoints))",
      "fit_prod_prereqs": "RBAC and quota administration: Cognitive Services Usages Reader at subscription level to view quota; Owner or Contributor on the subscription to request increases; Cognitive Services Contributor + Usages Reader to edit quota allocations; an Azure subscription and a Foundry project ([Manage and increase quotas for resources](https://learn.microsoft.com/en-us/azure/foundry/how-to/quota))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "cloud_console",
      "fit_python_frameworks": [
        "LangChain",
        "LangGraph",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Microsoft Foundry documents LangChain and LangGraph through `langchain-azure-ai`, including `AzureAIOpenAIApiChatModel` and `AzureAIDocumentIntelligenceTool`; vector-store packages include `langchain-azure-cosmosdb`. Foundry classic documents LlamaIndex through `llama-index-llms-azure-inference` and `llama-index-embeddings-azure-inference`, with `AzureAICompletionsModel` and `AzureAIEmbeddingsModel`. ([Microsoft Foundry LangChain integration](https://learn.microsoft.com/en-us/azure/foundry/how-to/develop/langchain))",
      "fit_quickstart_steps": 54,
      "fit_quickstart_steps_caveat": "54 numbered items interleave several language tracks and portal tracks rather than forming one linear procedure, and the final item concerns managing files on a vector store rather than making a first call. Treat this as \"not one procedure\", not as 54 steps of work.",
      "fit_quickstart_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "platform_provider_resources",
      "fit_terraform_note": "Terraform support is documented through the Terraform AzAPI Provider and AzureRM Provider for managing Microsoft Foundry resources, deployments, projects, connections, and capability hosts. The Terraform Registry documents provider `hashicorp/azurerm` and resource `azurerm_ai_foundry`; the newer Foundry guidance recommends provisioning the new resource with `azurerm_cognitive_account` rather than legacy hub-based projects. ([Create a Microsoft Foundry resource with Terraform](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-resource-terraform))",
      "fit_terraform_ref": "AzAPI and AzureRM; azurerm_ai_foundry exists, though newer guidance points to azurerm_cognitive_account",
      "fit_third_party_documented": {
        "vercel_ai_sdk": "a community publisher (@quail-ai)",
        "gateway_platform": "Kong"
      },
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "community_provider",
      "fit_vercel_ai_sdk_note": "A community Vercel AI SDK provider is documented as `@quail-ai/azure-ai-provider`, created with `createAzure`. This is an unofficial/community integration, not Microsoft vendor documentation. ([Community Azure AI provider for Vercel AI SDK](https://ai-sdk.dev/providers/community-providers/azure-ai))",
      "fit_vercel_ai_sdk_package": "@quail-ai/azure-ai-provider"
    },
    {
      "slug": "cloudflare-ai-gateway",
      "name": "Cloudflare AI Gateway",
      "company": "Cloudflare",
      "tagline": "Edge proxy in front of a curated set of AI providers, with caching, rate limiting, DLP and analytics.",
      "category": "cloud_platform",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2009,
      "hq_country": "United States",
      "url_home": "https://developers.cloudflare.com/ai-gateway/",
      "url_docs": "https://developers.cloudflare.com/ai-gateway/features/",
      "url_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
      "url_status": "https://www.cloudflarestatus.com",
      "url_github": null,
      "url_changelog": "https://developers.cloudflare.com/ai-gateway/changelog/",
      "url_models_api": null,
      "github_stars": null,
      "model_count": null,
      "model_count_note": "Cloudflare publishes no gateway model count. Its upstream list is narrower than the marketplaces: docs say all features work across 20+ supported AI providers, and the provider-native page enumerates 23 (Workers AI, Bedrock, Anthropic, Azure OpenAI, Baseten, Cartesia, Cerebras, Cohere, Deepgram, DeepSeek, ElevenLabs, Fal AI, Google AI Studio, Vertex AI, Groq, HuggingFace, Ideogram, Mistral, OpenAI, OpenRouter, Parallel, Perplexity, Replicate, xAI).",
      "provider_count": 23,
      "token_markup_pct": 0,
      "credit_fee_pct": 5,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "BYOK is a built-in feature at no stated extra cost: provider keys are stored encrypted in Cloudflare and rotated without code changes across 20+ providers.",
      "seat_fee_usd_month": 0,
      "free_tier": "AI Gateway core features (dashboard analytics, caching, rate limiting) are free on all plans; DLP scanning is free on all plans; Workers Free stores 100,000 logs total across gateways, Workers Paid 10,000,000 logs per gateway.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Unified Billing credit purchases",
          "amount": "5% fee on all credits purchased ($100 of credit costs $105)"
        },
        {
          "label": "Logpush",
          "amount": "Workers Paid only: 10 million requests/month included, then $0.05 per million"
        },
        {
          "label": "Guardrails inference",
          "amount": "Billed as Workers AI token usage for @cf/meta/llama-guard-3-8b evaluations"
        }
      ],
      "self_host_cost_note": null,
      "openai_compatible": null,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "Provider-native APIs (OpenAI, Anthropic, Google, Workers AI and others)"
      ],
      "sdk_note": "One line of code: change the provider base URL to the gateway endpoint; provider-native request formats are preserved.",
      "cap_failover": true,
      "cap_load_balancing": null,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": true,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": null,
      "latency_source_note": "No vendor figure. A third-party hands-on (Kalvium Labs, 2026-04-24) reports 20-60 ms added per request measured across several production systems on US and European cloud regions, i.e. under 5% of a typical 0.8-4 s LLM call; Cloudflare states caching can cut latency up to 90% on cache hits.",
      "throughput_rps": null,
      "soc2": null,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": null,
      "zdr": null,
      "zdr_note": "No ZDR commitment is published. The equivalent is disabling payload collection per request.",
      "air_gapped": false,
      "best_for": "Teams already on Cloudflare Workers who want free caching, analytics, spend limits, DLP and guardrails at the edge.",
      "strengths": [
        "Core gateway features are free on every Cloudflare plan, with no markup on provider tokens",
        "Free DLP scanning for PII, financial and healthcare data, plus llama-guard-based guardrails",
        "Dynamic Routing gives visual, code-free routing by user segment, geography, content or A/B split",
        "Spend limits scoped by model, provider or custom metadata, with automatic blocking",
        "Edge caching claimed to cut latency up to 90% and analytics via GraphQL API"
      ],
      "weaknesses": [
        "Upstream provider list (~23) is far narrower than OpenRouter or Requesty, and no model count or public catalog endpoint is published",
        "Unified Billing adds a 5% fee on credit purchases",
        "No published SOC 2 / HIPAA / ZDR posture specific to AI Gateway",
        "Log storage caps (100k on Workers Free) and Logpush restricted to Workers Paid"
      ],
      "avoid_if": "You need a broad model marketplace, a public pricing catalog endpoint, or documented gateway-level compliance commitments.",
      "reviews": [
        {
          "url": "https://dzone.com/articles/conversational-ai-cloudflare-workers-ai-gateway",
          "why": "Mayur Vekariya's build article puts AI Gateway in front of Workers AI with KV, Durable Objects and Vectorize, showing configuration and caching behaviour in a real app.",
          "date": "2025-10-07",
          "type": "written",
          "title": "Building a Production-Ready Conversational AI Agent With Cloudflare Workers and AI Gateway",
          "publisher": "DZone"
        },
        {
          "url": "https://www.infoq.com/news/2024/06/cloudflare-ai-gateway/",
          "why": "Renato Losio's independent GA writeup explains what the gateway proxies (Workers AI, OpenAI, Bedrock, Vertex, Azure OpenAI, HuggingFace) and the caching/rate-limiting story.",
          "date": "2024-06-02",
          "type": "written",
          "title": "Cloudflare AI Gateway Now Generally Available",
          "publisher": "InfoQ"
        },
        {
          "url": "https://www.youtube.com/watch?v=_CXZ1uo8ZQA",
          "why": "Mehul Mohan dissects AI-gateway architecture (normalization, fallback, analytics) and where Cloudflare's offering sits against OpenRouter and Vercel.",
          "date": "2025-11-21",
          "type": "video",
          "title": "How to build your own AI gateway (and do a generational ragebait on twitter)",
          "publisher": "YouTube - Mehul Mohan"
        },
        {
          "url": "https://www.reddit.com/r/CloudFlare/comments/1uwe2kn/ai_gateway_is_not_production_ready/",
          "why": "21-comment thread with hard failure reports: random 500s from the gateway, an essential model down for over a week, and users planning a move to Vercel AI Gateway.",
          "date": "2026-07-14",
          "type": "community",
          "title": "AI Gateway is not production ready",
          "publisher": "Reddit - r/CloudFlare"
        },
        {
          "url": "https://www.reddit.com/r/LLMDevs/comments/1lycx6i/either_i_dont_get_cloudflares_ai_gateway_or_it/",
          "why": "6-comment thread on the gateway's actual security model, including per-IP rate limits and the risk of credit drain when clients call it directly.",
          "date": "2025-07-12",
          "type": "community",
          "title": "Either I don't get Cloudflare's AI gateway, or it does not do what I expected it to",
          "publisher": "Reddit - r/LLMDevs"
        }
      ],
      "sources": {
        "license": "https://developers.cloudflare.com/ai-gateway/",
        "sdk_note": "https://developers.cloudflare.com/ai-gateway/",
        "url_docs": "https://developers.cloudflare.com/ai-gateway/features/",
        "cap_audio": "https://developers.cloudflare.com/ai-gateway/usage/providers/",
        "free_tier": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "air_gapped": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "deployment": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "extra_fees": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "hq_country": "https://en.wikipedia.org/wiki/Cloudflare",
        "url_status": "https://www.cloudflarestatus.com",
        "zdr_status": "https://developers.cloudflare.com/ai-gateway/observability/logging/",
        "cap_budgets": "https://developers.cloudflare.com/ai-gateway/features/",
        "url_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "api_surfaces": "https://developers.cloudflare.com/ai-gateway/usage/providers/",
        "cap_failover": "https://developers.cloudflare.com/ai-gateway/",
        "founded_year": "https://en.wikipedia.org/wiki/Cloudflare",
        "product_fate": "https://developers.cloudflare.com/ai-gateway/changelog/",
        "regions_note": "https://www.cloudflare.com/trust-hub/responsible-ai/",
        "byok_fee_note": "https://developers.cloudflare.com/ai-gateway/features/",
        "cap_image_gen": "https://developers.cloudflare.com/ai-gateway/usage/providers/",
        "url_changelog": "https://developers.cloudflare.com/ai-gateway/changelog/",
        "vendor_status": "https://developers.cloudflare.com/ai-gateway/changelog/",
        "byok_supported": "https://developers.cloudflare.com/ai-gateway/features/",
        "cap_guardrails": "https://developers.cloudflare.com/ai-gateway/features/",
        "certifications": "https://www.cloudflare.com/trust-hub/compliance-resources/soc-2/",
        "config_as_code": "https://developers.cloudflare.com/api/terraform/resources/ai_gateway/",
        "credit_fee_pct": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "provider_count": "https://developers.cloudflare.com/ai-gateway/usage/providers/",
        "retention_days": "https://developers.cloudflare.com/ai-gateway/reference/limits/",
        "cap_rate_limits": "https://developers.cloudflare.com/ai-gateway/features/",
        "logs_by_default": "https://developers.cloudflare.com/ai-gateway/observability/logging/",
        "cap_cache_simple": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "model_count_note": "https://developers.cloudflare.com/ai-gateway/usage/providers/",
        "token_markup_pct": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "cap_observability": "https://developers.cloudflare.com/ai-gateway/features/",
        "cap_pii_redaction": "https://developers.cloudflare.com/ai-gateway/features/",
        "last_release_date": "https://developers.cloudflare.com/ai-gateway/changelog/",
        "subprocessors_url": "https://www.cloudflare.com/gdpr/subprocessors/",
        "seat_fee_usd_month": "https://developers.cloudflare.com/ai-gateway/reference/pricing/",
        "guardrails_location": "https://developers.cloudflare.com/ai-gateway/features/guardrails/",
        "latency_source_note": "https://www.kalviumlabs.ai/blog/cloudflare-ai-gateway-vs-direct-openai-api/",
        "requires_vendor_sdk": "https://developers.cloudflare.com/ai-gateway/usage/chat-completion/",
        "data_export_available": "https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/",
        "prompt_transits_vendor": "https://developers.cloudflare.com/ai-gateway/features/guardrails/",
        "cap_conditional_routing": "https://developers.cloudflare.com/ai-gateway/features/",
        "trains_on_customer_data": "https://www.cloudflare.com/trust-hub/responsible-ai/",
        "cloudflare_ai_gateway_caching": "https://developers.cloudflare.com/ai-gateway/features/caching/",
        "cloudflare_ai_gateway_pricing": "https://developers.cloudflare.com/ai-gateway/reference/pricing/"
      },
      "verified_at": {
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "cap_audio": "2026-08-29",
        "free_tier": "2026-08-29",
        "air_gapped": "2026-08-29",
        "deployment": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "byok_fee_note": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "url_changelog": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "credit_fee_pct": "2026-08-29",
        "provider_count": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "token_markup_pct": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "seat_fee_usd_month": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:35.832825+00:00",
      "vendor_status": "hyperscaler",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "A first-party Cloudflare product shipping steadily, including unified Workers AI billing and anomalous-spend insights in August 2026.",
      "last_release_date": "2026-08-19",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "A proxy that sits between your application and the model, so every request passes through Cloudflare.",
      "logs_by_default": "full_content",
      "logs_note": "The most permissive default in this catalogue. Logs are enabled by default for each gateway and include the user prompt and the model response alongside metadata. Opting out is per request, via the cf-aig-collect-log-payload header.",
      "retention_days": null,
      "retention_note": "Retention is capped by count, not time: 100,000 logs per account on the free plan and 10 million per gateway on paid. Logs persist until the cap is reached, at which point new logs stop being saved.",
      "zdr_status": "not_published",
      "trains_on_customer_data": "no",
      "training_note": "Cloudflare does not use customer content to train any large language models, and does not train its own machine-learning models on customer content without consent.",
      "subprocessors_url": "https://www.cloudflare.com/gdpr/subprocessors/",
      "regions_note": "A global edge with platform-level data-residency support, but no AI-Gateway-specific region pinning and no customer-VPC deployment.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II covering security, confidentiality and availability",
          "status": "certified"
        },
        "fedramp": {
          "detail": "High authorisation for Cloudflare for Government",
          "status": "certified"
        },
        "gdpr_dpa": {
          "detail": "subprocessor list and GDPR programme published",
          "status": "certified"
        },
        "iso27001": {
          "detail": "also 27018 and 27701",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "services described as aligning to HIPAA needs; no BAA offer found",
          "status": "alignment_only"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "Guardrails inspect interactions in real time using a Llama Guard model on Workers AI — Cloudflare's infrastructure, not yours.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "markup_only",
      "pricing_model_note": "Free gateway core + credit top-up fee (Unified Billing, 5% on credit purchases) + plan-tied log/Logpush limits. Inference passed through at cost — 0% markup.",
      "min_commitment": "None stated. Enterprise customers reach out to their account team.",
      "overage_note": "The bite is on logs, not requests. Persistent log storage capped at 100,000 logs total across all gateways on Workers Free and 10,000,000 logs per gateway on Workers Paid, with automatic log deletion. Logpush is Workers Paid only, includes 10M requests/month then charges +$0.05/million. Guardrails usage is billed as Workers AI token-based inference.",
      "cache_support": "exact",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact-match caching: based on exact match of the entire request. Disabled by default. Serves cached provider responses from Cloudflare's cache. No differential cached-token pricing published. A cache hit skips the provider call entirely — customer captures the full saving and Cloudflare charges nothing for the cache itself. The only related cost is log storage of the cached requests.",
      "cost_controls": {
        "enforcement": "pre_request",
        "spend_alerts": {
          "note": "Not stated; only stated monitoring is 'You can monitor your usage in the AI Gateway dashboard' (after-the-fact).",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Supported, free on all plans, enforced pre-request.",
          "tier": "free",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated.",
      "cost_export": "Logpush on Workers Paid (10M requests/month included, then $0.05/million). CSV, cost API, webhook and warehouse export not stated.",
      "product_kind_note": "Cloudflare positions AI Gateway as the way to \"gain visibility and control over your AI apps\", gathering \"insights on how people are using your application with analytics and logging\" and controlling scale with \"caching, rate limiting, as well as request retries, model fallback, and more\" ([AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20). Functionally it is an observability plane that also proxies, exposing \"an OpenAI-compatible `/chat/completions` endpoint\" for \"integration with multiple AI providers using a single URL\" ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07).",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "No installation: \"The `default` gateway is created automatically on your first request — no setup needed\", and a specific gateway ID can replace `default`; the endpoint embeds the Cloudflare account ID (\"Replace `{account_id}` with your Cloudflare account ID\") ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07). Tier gating is on ancillary features, not deployment: core features (dashboard analytics, caching, rate limiting) and DLP scanning are free on all plans; persistent-log storage is capped at 100,000 logs total on Workers Free vs 10,000,000 logs per gateway on Workers Paid; Logpush is \"only available on the Workers Paid plan\" (10 million requests/month included, then $0.05/million) ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "not_documented",
      "api_images": "not_documented",
      "api_audio": "not_documented",
      "api_batch": "not_documented",
      "api_surface_note": "Two modes: a unified OpenAI-compatible endpoint where you switch providers by changing the `model` and `apiKey` parameters (examples use the OpenAI JS SDK), and provider-native pass-through where you keep the provider's own request format and paths ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07; [Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20). Gateway-level auth adds a `cf-aig-authorization` header, and when using stored keys you \"remove provider authorization headers from your requests\" while still passing `cf-aig-authorization`; a non-default stored key is selected with `cf-aig-byok-alias` ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31). Streaming caveats: n.a.",
      "model_count_low": null,
      "model_count_high": null,
      "model_count_source_note": "n.a.: no model total is published; the overview only invites users to \"explore all AI models available through AI Gateway\" ([AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20).",
      "provider_count_low": 24,
      "provider_count_high": 24,
      "provider_count_source_note": "24 providers are listed by name — Workers AI, Amazon Bedrock, Anthropic, Azure OpenAI, Baseten, Cartesia, Cerebras, Cohere, Deepgram, DeepSeek, ElevenLabs, Fal AI, Google AI Studio, Google Vertex AI, Groq, HuggingFace, Ideogram, Mistral AI, OpenAI, OpenRouter, Parallel, Perplexity, Replicate, xAI — with no stated total ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).",
      "first_party_vs_partner": "Documented mix: mostly third-party routed providers, plus Cloudflare's own Workers AI as a supported provider, and Guardrails inference runs on Workers AI using `@cf/meta/llama-guard-3-8b` ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20; [AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19).",
      "custom_endpoint_support": "Yes: register a Custom Provider via the API or \"Compute & AI > AI Gateway > Custom Providers\" with a unique `slug` and an HTTPS `base_url`, and self-hosted models are explicitly supported; management calls need a Cloudflare API token with `AI Gateway - Edit` ([Custom providers](https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/)). vLLM/Ollama/SageMaker by name: n.a.",
      "byok_mode": "both",
      "merchant_of_record": "Split by mode: with BYOK the upstream provider bills the customer's own account; with Unified Billing Cloudflare invoices, applying \"a 5% fee… to all credits purchased through Unified Billing\" (a $100 credit purchase results in a $105 charge) while \"inference pricing from providers is passed through with no markup — you pay the same per-token rates as you would directly with the provider\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/)). Guardrails usage \"is billed as Workers AI token-based inference\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/)). An explicit statement of the invoicing entity for provider tokens: n.a.",
      "key_management_note": "Stored provider keys live in Cloudflare Secrets Store, which gives \"secure storage and limit exposure\", \"easier key rotation\" and pairing with \"rate limit, budget limit and other restrictions with Dynamic Routes\"; dashboard-created secrets are named automatically, API-created secrets must follow the required naming convention because \"the `secret_id` returned by Secrets Store is not used by AI Gateway for runtime lookup\" ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31). Multiple keys per provider are supported, each with an alias (`default` when unspecified), selectable per request via `cf-aig-byok-alias`; the dashboard shows keys by provider with last-used time and `active`/`expired`/`invalid` status, rotation takes effect immediately \"without any code changes or downtime\", and keys can be deleted ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/)). Because it is hosted-only, requests transit Cloudflare and can be persisted as logs (with plan-based storage caps and free DLP scanning); the overview notes logging gives \"insight on requests and errors\" but does not state whether Cloudflare stores request content by default ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20).",
      "api_openai_chat_completions_note": "Yes: an OpenAI-compatible unified endpoint at `/compat/chat/completions` (provider-native routes also accept paths such as `/v1/chat/completions`), with models addressed in `{provider}/{model}` form, e.g. `openai/gpt-5-mini`, `google-ai-studio/gemini-2.5-flash`, `anthropic/claude-sonnet-4-5` ([Unified API (OpenAI compat)](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), 2026-08-07; [Custom providers](https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/)).",
      "api_anthropic_messages_note": "Yes via provider-native passthrough: Anthropic is a supported provider whose native endpoint is proxied ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20). The specific `/v1/messages` path is not shown on the pages fetched: n.a.",
      "api_openai_responses_note": "n.a. (not documented on the pages fetched).",
      "api_embeddings_note": "n.a. as a documented gateway path; embedding-capable providers (Cohere, Workers AI, Google) are proxied provider-natively ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).",
      "api_images_note": "n.a. as a documented gateway path, though image providers (Fal AI, Ideogram, Replicate) are supported provider-natively ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).",
      "api_audio_note": "n.a. as a documented gateway path; audio providers are supported provider-natively — Deepgram and ElevenLabs (STT/TTS) and Cartesia ([Provider Native](https://developers.cloudflare.com/ai-gateway/usage/providers/), 2026-04-20).",
      "api_batch_note": "n.a.",
      "deployment_modes_note": "hosted SaaS on Cloudflare's network only, \"available to use on all plans\" ([AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19; [AI Gateway overview](https://developers.cloudflare.com/ai-gateway/), 2026-04-20). Self-host, hybrid/VPC, on-prem, air-gapped: n.a.",
      "byok_mode_note": "both: BYOK lets you \"securely store your AI provider API keys directly in the Cloudflare dashboard\" ([BYOK (Store Keys)](https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/), 2026-07-31), while Unified Billing runs usage against Cloudflare account credits ([Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/); [AI Gateway pricing](https://developers.cloudflare.com/ai-gateway/reference/pricing/), 2026-05-19). Provider credentials take precedence over Unified Billing credits when both exist ([Unified Billing](https://developers.cloudflare.com/ai-gateway/features/unified-billing/)).",
      "access_meta": {
        "qualified": [
          "api_anthropic_messages",
          "api_embeddings",
          "api_images",
          "api_audio"
        ],
        "model_count_kind": null,
        "provider_count_kind": "enumerated"
      },
      "rel_timeout_config": "per_request",
      "rel_timeout_note": "Header `cf-aig-request-timeout`, set on a provider-specific endpoint, in milliseconds. Behavior is first-response based: as long as the first part of the response returns in time, \"such as when streaming a response\", the gateway waits. Default value not stated ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).",
      "rel_retry_config": "per_request",
      "rel_retry_note": "`per_request` plus `dashboard_only` defaults. Headers `cf-aig-max-attempts`, `cf-aig-retry-delay` (ms, max 5 seconds) and `cf-aig-backoff` (`constant` | `linear` | `exponential`), with \"a maximum of five retry attempts\". \"On the final retry attempt, your gateway will wait until the request completes, regardless of how long it takes.\" Gateway-level defaults (\"up to 5 attempts\", constant/linear/exponential, 100ms–5s delay) apply to all requests and can be overridden per request. The default attempt count itself is not published: `n.a.` ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/), [Cloudflare auto-retry changelog](https://developers.cloudflare.com/changelog/post/2026-04-02-auto-retry-upstream-failures/)).",
      "rel_retry_default_note": "`per_request` plus `dashboard_only` defaults. \"On the final retry attempt, your gateway will wait until the request completes, regardless of how long it takes.\" Gateway-level defaults (\"up to 5 attempts\", constant/linear/exponential, 100ms–5s delay) apply to all requests and can be overridden per request.",
      "rel_fallback_config": "per_request",
      "rel_fallback_note": "ORDERED, expressed as an array of request objects to the Universal Endpoint: \"You can add as many fallbacks as you need, just by adding another object in the array\", and the response header `cf-aig-step` reports which step served (`0` primary, `1` second model, `2` third, incrementing per fallback). No weighted split at this layer, and the fallback key itself is not named ([Cloudflare AI Gateway fallbacks](https://developers.cloudflare.com/ai-gateway/configuration/fallbacks/)).",
      "rel_fallback_shape": "weighted",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "There is no upstream health check, circuit breaker or provider-ejection control; the documented mechanism is retry-then-fallback on error ([Cloudflare auto-retry changelog](https://developers.cloudflare.com/changelog/post/2026-04-02-auto-retry-upstream-failures/)).",
      "rel_multi_region_config": "not_configurable",
      "rel_multi_region_note": "Cloudflare runs the network: the Gateway Worker \"operates near the user's location\" on \"Cloudflare's vast network of data centers\", but no user-configurable cross-region failover exists ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).",
      "rel_load_balancing_config": "dashboard_only",
      "rel_load_balancing_note": "`dashboard_only` (plus JSON config), via Dynamic Routing: the **Percentage** node \"Routes requests probabilistically across multiple outputs\", i.e. weighted-by-percentage splitting; Rate Limit and Budget Limit nodes \"switch to fallback when exceeded\". Routes are built in a visual interface or JSON-based configuration ([Cloudflare Dynamic Routing](https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/)). The product page adds \"Automatically route requests based on latency, cost, or availability\" ([Cloudflare AI Gateway product page](https://www.cloudflare.com/products/ai-gateway/)).",
      "architecture_class": "edge_worker",
      "architecture_note": "Explicitly built on Cloudflare Workers (\"a serverless platform that runs on the Cloudflare network\"), with a Gateway Worker and Durable Objects (each Account Manager with its own SQLite database) for logging ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "No self-host artifact; usage is via the gateway URL or Workers bindings such as `env.AI.gateway(\"my-gateway\").getUrl(\"openai\")` ([Cloudflare AI Gateway product page](https://www.cloudflare.com/products/ai-gateway/)).",
      "latency_claims": [],
      "throughput_claims": [
        {
          "url": "https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/",
          "unit": "requests/year",
          "scope": "marketing",
          "value": "over 2B",
          "metric": "fleet total",
          "publisher": "vendor",
          "conditions": "Aggregate proxied volume in one year, not a capacity benchmark."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported. Caveat tied to the timeout semantics: `cf-aig-request-timeout` is evaluated on the first part of the response, so once streaming starts the gateway will wait for the rest ([Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "AI Gateway DLP scans prompts and responses in real time using Cloudflare's DLP detection engines and account-level profiles. On **Block**, \"the provider response is discarded and replaced with a DLP error response (status 400)\"; on **Flag**, findings are attached to the `cf-aig-dlp` response header and logged while the original response is still returned. The page does not state that DLP redacts ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Prompt-injection detection uses the Workers AI model `@cf/meta/prompt-guard-2-86m`, exposed as category P1 with per-category Flag / Ignore / Block actions ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/), [Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/))",
      "guard_content": "sync_block",
      "guard_content_note": "Llama Guard 3 8B on Workers AI evaluates hazard categories S1–S13 inline, again with Flag / Ignore / Block per category ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Custom policy takes the form of Cloudflare One DLP detection profiles (predefined or custom) selected at the gateway level; \"There is no per-request header to select specific DLP profiles or to bypass DLP scanning\", and different policies require separate gateways ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages — gateway-level configuration governs guardrails and DLP, but an administrative model/provider allow-deny list is not documented in what was fetched ([Guardrails](https://developers.cloudflare.com/ai-gateway/features/guardrails/), [DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "configurable",
      "guard_failure_mode_note": "Documented per action rather than per outage: with a category set to Block, \"the request will be blocked\" (fail closed for that category), while Flag returns the response and records the finding (fail open) ([Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/), [DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/)). Behaviour when the guardrail model itself is unavailable is `not_documented`",
      "log_content": "full_request_and_response",
      "log_content_note": "Logging is enabled by default and includes prompt and response payloads, along with DLP FLAG/BLOCK fields ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`cf-aig-collect-log-payload: false` keeps the log entry as metadata only, and `cf-aig-collect-log: false` disables the log entry for that request ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))",
      "tracing_support": "not_documented",
      "tracing_note": "`n.a.` — no OpenTelemetry or agent-trace representation documented for AI Gateway ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))",
      "export_destinations": [
        "Logpush",
        "R2",
        "Webhook"
      ],
      "export_destinations_note": "Workers Logpush to R2 and other object-storage/HTTP destinations, limited to 10 million logs per gateway across up to 4 Logpush jobs ([Logpush](https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/))",
      "feedback_capture": "yes",
      "feedback_capture_note": "Yes — the response header `cf-aig-log-id` identifies the log entry, and a `PATCH` to the AI Gateway logs API records feedback (for example `-1` for negative), requiring an API token with `AI Gateway - Edit`/Write ([Add human feedback using API](https://developers.cloudflare.com/ai-gateway/evaluations/add-human-feedback-api/))",
      "eval_hooks": "yes",
      "eval_hooks_note": "Yes, on Cloudflare's platform — Evaluations run against datasets built from filtered logs, reporting performance, speed, and cost; human feedback is the first evaluator and is in open beta ([Evaluations](https://developers.cloudflare.com/ai-gateway/evaluations/))",
      "observability_requires_saas": null,
      "log_retention_note": "Storage-limit driven and configurable: when the gateway's log limit is reached, the oldest logs are automatically deleted ([Logging](https://developers.cloudflare.com/ai-gateway/observability/logging/))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` for gateway overhead. The only timing published is operational: \"New requests show up within 15 seconds\" for log visibility; positioning the Durable Object near the Gateway Worker \"significantly enhances the speed of log insertion and query operations\" with no number ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).",
        "multi_surface": {
          "rel_retry": [
            "per_request",
            "dashboard_only"
          ]
        },
        "throughput_prose": "No RPS or tokens/sec benchmark. Scale figures only: AI Gateway proxied \"over 2 billion requests in just one year\"; storage limits of 10 million logs per Durable Object / per gateway and 100 million per account at the default 10 gateways ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/)).",
        "benchmark_independence": "Vendor-published operational statistics only; no benchmark and no claims about competitors ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/))."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "choose_at_setup",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Two different retry/fallback surfaces coexist: header-driven retries with `cf-aig-*` on provider endpoints, and node-based Dynamic Routing where a fallback is a graph edge. The scaling blog also shows an older-style `\"retry\": 2` / `\"delay\": 200` / `\"onComplete\"` JSON shape, which does not match the header names in the current docs ([Cloudflare AI Gateway scaling blog](https://blog.cloudflare.com/billions-and-billions-of-logs-scaling-ai-gateway-with-the-cloudflare/), [Cloudflare AI Gateway request handling](https://developers.cloudflare.com/ai-gateway/configuration/request-handling/)).",
      "guard_notes": "Two enforcement caveats matter. Guardrails do not enforce on streaming responses — streaming traffic is evaluated/logged through the REST path but not blocked ([Guardrails usage considerations](https://developers.cloudflare.com/ai-gateway/features/guardrails/usage-considerations/)). DLP on streaming SSE responses \"buffers the full streamed response before scanning\", which removes the latency benefit of streaming, while request-only scanning avoids that cost; base64 images, external URLs, and binary multipart data are not inspected ([DLP](https://developers.cloudflare.com/ai-gateway/features/dlp/)).",
      "fit_agent_features_note": "The provided Cloudflare documentation describes function calling and tool use through supported models and provider tools, Agents SDK sessions, durable state, WebSockets, scheduling, recoverable execution, and multi-step agent harnesses controlling planning, tool use, tool results, streaming, and continuation. A2A is not documented. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No for the documented Workers AI call; provider authentication options are \"Unified Billing, BYOK, or request headers\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))",
      "fit_client_sdk_langs": [
        "JavaScript",
        "Python"
      ],
      "fit_client_sdks_note": "Documented clients include the OpenAI SDK for JavaScript and Python; Anthropic SDK for JavaScript and Python; `@google/genai`; `@mistralai/mistralai`; the Cohere Python SDK; Groq SDK for JavaScript; HuggingFace `@huggingface/inference`; Fal AI `@fal-ai/client`; `ai-gateway-provider` for the Vercel AI SDK; and Cloudflare Workers `env.AI` bindings. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_cloud_iam": "upstream_credentials_only",
      "fit_cloud_iam_note": "Amazon Bedrock support is documented with AWS SigV4 credentials, including `accessKeyId`, `secretAccessKey`, `region`, and an optional AWS STS `sessionToken`. Google Vertex AI supports service-account JSON credentials or direct GCP access tokens, including GCP service-account authentication; Azure OpenAI documentation specifies an Azure OpenAI API key rather than Entra or managed identity. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_cloudflare_workers": "user_facing",
      "fit_cloudflare_workers_note": "Cloudflare Workers support is documented through the `env.AI` binding, including `env.AI.run()` and `env.AI.gateway()`, third-party models, Workers AI models, logging, feedback, metadata, and gateway configuration. Workers AI can also integrate with AI Gateway by adding a gateway object to a Workers AI request; the gateway must be in the same account as the Worker. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated as required: \"AI Gateway is available to use on all plans.\" and \"AI Gateway's core features available today are offered for free.\" ([Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))",
      "fit_ecosystem_notes": "AI Gateway is operated through a Cloudflare account and dashboard/control plane, with Cloudflare API tokens, gateway configuration, billing, logging, analytics, and provider-key management. It supports Workers AI, Anthropic, Google Gemini, OpenAI, Replicate, custom providers with HTTPS endpoints and custom domains, caching, rate limiting, retries, DLP, OpenTelemetry, and Stripe-related configuration; self-hosting and on-premises deployment are not documented. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_eval_prereqs": "A Cloudflare account, your Account ID, and a Cloudflare API token with \"AI Gateway - Read\", \"AI Gateway - Edit\" and \"Workers AI - Read\" permissions. No credit card, cloud account, cluster or provider key stated; pricing page says \"all it takes is a Cloudflare account and one line of code to get started.\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "no",
      "fit_eval_snippet_note": "No copy-pasteable completion request was rendered on the fetched pages; the documented request shape is a Workers AI model with the `@cf/` prefix plus header `cf-aig-gateway-id: default`, and the unified API exposes an OpenAI-compatible `/chat/completions` endpoint where you \"replace `{account_id}` with your Cloudflare account ID\" ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Unified API (OpenAI compat) · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": null,
      "fit_kubernetes": "not_documented",
      "fit_kubernetes_note": "n.a. (not documented)",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Cloudflare Agents explicitly support MCP tools, but an AI Gateway MCP server, MCP gateway or registry, and dedicated MCP tool proxy are not documented. Separately, Cloudflare Zero Trust documents MCP server portals that proxy upstream MCP tools and prompts behind one HTTP endpoint at `/mcp`. ([Cloudflare MCP server portals](https://developers.cloudflare.com/cloudflare-one/access-controls/ai-controls/mcp-portals/))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_gateway_platform",
          "fit_kubernetes"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or waitlist gate is stated ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/), [Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/))",
      "fit_onboarding_notes": "Gateway creation can be skipped: the `cf-aig-gateway-id: default` header auto-creates the default gateway on the first authenticated request. Workers AI requests use Standard billing by default and can be switched to Unified billing ([Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "OpenAI-compatible SDK usage is documented. The exact documented base URL is `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/openai`; the OpenAI JavaScript and Python SDKs are shown, and Grok documentation also says to replace the xAI endpoint with `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/grok`. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_prod_prereqs": "n.a. (not documented): \"Production infrastructure requirements: None stated.\" Prepaid AI Gateway credits are mentioned for unified billing ([Pricing · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/reference/pricing/), [Getting started · Cloudflare AI Gateway docs](https://developers.cloudflare.com/ai-gateway/get-started/))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "cloud_console",
      "fit_python_frameworks": [
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LlamaIndex integration is documented through the `llama-index-llms-cloudflare-ai-gateway` package and the `CloudflareAIGateway` class (`Bases: LLM`). LangChain support for Cloudflare AI Gateway is not documented in the provided evidence. ([LlamaIndex Cloudflare AI Gateway integration](https://developers.llamaindex.ai/python/framework-api-reference/llms/cloudflare_ai_gateway/))",
      "fit_quickstart_steps": 13,
      "fit_quickstart_steps_caveat": "13 entries spread across three separate numbered lists (2 + 7 + 4), so no single steps-to-first-call figure exists.",
      "fit_quickstart_url": "https://developers.cloudflare.com/ai-gateway/get-started/",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "official_provider",
      "fit_terraform_note": "Official Terraform support is documented through the `cloudflare/cloudflare` provider. Documented resources include `cloudflare_ai_gateway` and `cloudflare_ai_gateway_dynamic_routing`, with the latter documented at `registry.terraform.io/providers/cloudflare/cloudflare/latest/docs/resources/ai_gateway_dynamic_routing`. ([Terraform Cloudflare AI Gateway resource](https://registry.terraform.io/providers/cloudflare/cloudflare/latest/docs/resources/ai_gateway))",
      "fit_terraform_ref": "cloudflare/cloudflare with cloudflare_ai_gateway and cloudflare_ai_gateway_dynamic_routing",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Vercel AI SDK integration is documented through the `ai-gateway-provider` npm package. The package provides `createAiGateway`, `createUnified`, `createOpenAI`, `createAnthropic`, `createGoogle`, and `createXai`; Cloudflare Workers usage can also use `workers-ai-provider`. ([Cloudflare AI Gateway full documentation](https://developers.cloudflare.com/ai-gateway/llms-full.txt))",
      "fit_vercel_ai_sdk_package": "ai-gateway-provider"
    },
    {
      "slug": "google-vertex-ai",
      "name": "Google Vertex AI",
      "company": "Google Cloud",
      "tagline": "Google Cloud's model platform for Gemini plus 200+ Model Garden models, now branded Gemini Enterprise Agent Platform.",
      "category": "cloud_platform",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": null,
      "hq_country": "United States",
      "url_home": "https://cloud.google.com/vertex-ai",
      "url_docs": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models",
      "url_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
      "url_status": null,
      "url_github": null,
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": null,
      "model_count": 200,
      "model_count_note": "Vendor states Model Garden offers \"200+ Google and third-party AI models and tools\".",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": null,
      "free_tier": "$300 in free credits for new Google Cloud customers; Grounding with Google Search includes 5,000 free queries per month.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Provisioned Throughput (GSU)",
          "amount": "$7.14 per GSU 1-week, $3.6986 1-month, $3.2877 3-month, $2.7397 1-year (global endpoint; non-global higher)"
        },
        {
          "label": "Grounding with Google Search",
          "amount": "5,000 free queries/month, then $14 per 1,000 (Gemini 3) or $35 per 1,000 (Gemini 2.5)"
        },
        {
          "label": "Web Grounding for Enterprise",
          "amount": "$45 per 1,000 requests"
        },
        {
          "label": "Grounding with your own data",
          "amount": "$2.50 per 1,000 requests"
        },
        {
          "label": "Tuned model endpoints",
          "amount": "1.5x base token price for Gemini 3 and newer"
        }
      ],
      "self_host_cost_note": "Managed only. Google sets its own per-token and per-modality prices rather than marking up a third party; Batch mode is 50% below interactive pricing, and Provisioned Throughput is sold in Generative AI Scale Units (GSUs) on 1-week to 1-year terms. Gemini 3.7/3.6 Flash carries introductory pricing of $0.75/1M input and $3.75/1M output through Dec 31 2026, rising to $1.50/$7.50 afterwards.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": null,
      "api_surfaces": [
        "Vertex AI Gemini API",
        "OpenAI Chat Completions (OpenAI-compatible endpoint)",
        "Interactions API",
        "Live API",
        "Online and batch prediction"
      ],
      "sdk_note": "Gemini models can be called with the OpenAI Python/TypeScript libraries against Vertex AI's OpenAI-compatible Chat Completions endpoint.",
      "cap_failover": null,
      "cap_load_balancing": null,
      "cap_conditional_routing": null,
      "cap_cache_simple": true,
      "cap_cache_semantic": null,
      "cap_observability": null,
      "cap_prompt_management": true,
      "cap_evals": true,
      "cap_guardrails": null,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": true,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": null,
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": null,
      "eu_residency": true,
      "eu_residency_note": "The EU multi-region endpoint https://aiplatform.eu.rep.googleapis.com keeps ML processing inside the EU jurisdictional boundary, and ten European regions are available (europe-west1/2/3/4/6/8/9, europe-north1, europe-central2, europe-southwest1). The global endpoint does not satisfy ML-processing residency requirements, and global endpoints for partner and open models do not support data residency.",
      "zdr": true,
      "zdr_note": "Documented as a configuration discipline rather than a switch: you must act in several areas, request an exception if you are in scope for abuse-monitoring prompt logging, and explicitly set store to false. Google warns it may not be achievable with some advanced AI features.",
      "air_gapped": null,
      "best_for": "Google Cloud customers who want Gemini alongside third-party models with strong, explicitly documented EU residency and ZDR controls.",
      "strengths": [
        "Most explicit data-governance documentation of the three: per-feature ZDR conditions and a dedicated EU multi-region endpoint",
        "Generative AI on Vertex AI / Gemini Enterprise Agent Platform is named in scope for SOC 2 Type II, audited quarterly",
        "Broad modality coverage in one platform: Gemini text, Imagen images, Veo video, Chirp audio, Lyria music, embeddings",
        "Grounding options span Google Search, Maps and your own data, with a ZDR-friendly enterprise variant",
        "Batch at 50% off plus Provisioned Throughput GSU terms from one week to one year"
      ],
      "weaknesses": [
        "No cross-cloud routing — only models Google hosts are reachable, so failover to Azure or AWS needs an external gateway",
        "Residency and ZDR guarantees do not extend to global endpoints for partner and open-weight models",
        "Pricing spans tokens, modality units, GSUs, grounding calls and cache storage, so effective cost is hard to model",
        "Value drops sharply if your data is not already in Google Cloud, given egress and integration cost"
      ],
      "avoid_if": "You are not on Google Cloud, or you need one gateway that can fail over to models hosted by other vendors.",
      "reviews": [
        {
          "url": "https://www.brahimbouine.com/blog/gcp-vertex-ai-model-serving-benchmarks/",
          "why": "Named engineer publishes his own Vertex AI endpoint benchmarks across T4/L4/A100/TPU and CPU machine types under production-shaped traffic.",
          "date": "2026-01-12",
          "type": "written",
          "title": "GCP Vertex AI Model Serving Benchmarks: Endpoint Performance Under Production Traffic",
          "publisher": "brahimbouine.com"
        },
        {
          "url": "https://www.youtube.com/watch?v=gR4mXir0I-g",
          "why": "Tech Field Day session where independent delegates question a Vertex AI PM while multimodal and long-video demos are run live.",
          "date": "2024-06-17",
          "type": "video",
          "title": "Google Cloud Vertex AI Platform",
          "publisher": "YouTube - Tech Field Day"
        },
        {
          "url": "https://www.reddit.com/r/googlecloud/comments/1r30jrr/why_vertex_is_so_slower_then_ai_studio/",
          "why": "Practitioners compare Vertex latency and 'overloaded' errors against AI Studio and the direct OpenAI API when moving to production.",
          "date": "2026-02-12",
          "type": "community",
          "title": "Why Vertex is so slower then AI Studio?",
          "publisher": "Reddit - r/googlecloud"
        },
        {
          "url": "https://www.reddit.com/r/googlecloud/comments/16oha9y/do_people_actually_use_vertex_ai/",
          "why": "26-comment thread with concrete outcomes, including a team moving ML workloads to AWS after a year and another exporting models to edge to avoid $1k/month endpoint costs.",
          "date": "2023-09-21",
          "type": "community",
          "title": "Do people actually use Vertex AI?",
          "publisher": "Reddit - r/googlecloud"
        }
      ],
      "sources": {
        "zdr": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "name": "https://cloud.google.com/vertex-ai",
        "soc2": "https://cloud.google.com/security/compliance/soc-2",
        "company": "https://cloud.google.com/vertex-ai",
        "tagline": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models",
        "sdk_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library",
        "url_docs": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models",
        "url_home": "https://cloud.google.com/vertex-ai",
        "zdr_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "cap_audio": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "cap_batch": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "cap_evals": "https://cloud.google.com/vertex-ai",
        "cap_video": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "free_tier": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "hipaa_baa": "https://cloud.google.com/security/compliance/hipaa-compliance",
        "extra_fees": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "hq_country": "https://en.wikipedia.org/wiki/Google",
        "zdr_status": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/vertex-ai-zero-data-retention",
        "model_count": "https://cloud.google.com/vertex-ai",
        "url_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "api_surfaces": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "eu_residency": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations",
        "product_fate": "https://docs.cloud.google.com/vertex-ai/docs/release-notes",
        "regions_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations",
        "cap_image_gen": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "vendor_status": "https://cloud.google.com/vertex-ai/sla",
        "cap_embeddings": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "certifications": "https://cloud.google.com/security/compliance/soc-2",
        "config_as_code": "https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/vertex_ai_endpoint",
        "retention_days": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "sla_uptime_pct": "https://cloud.google.com/vertex-ai/sla",
        "logs_by_default": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "cap_cache_simple": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "model_count_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models",
        "eu_residency_note": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations",
        "last_release_date": "https://docs.cloud.google.com/vertex-ai/docs/release-notes",
        "openai_compatible": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library",
        "subprocessors_url": "https://cloud.google.com/terms/subprocessors",
        "vertex_ai_pricing": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "guardrails_location": "https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance",
        "requires_vendor_sdk": "https://cloud.google.com/vertex-ai/generative-ai/docs/migrate/openai/overview",
        "self_host_cost_note": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
        "cap_prompt_management": "https://cloud.google.com/vertex-ai",
        "data_export_available": "https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging",
        "prompt_transits_vendor": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations",
        "trains_on_customer_data": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/vertex-ai-zero-data-retention"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_batch": "2026-08-29",
        "cap_evals": "2026-08-29",
        "cap_video": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "extra_fees": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "eu_residency": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "config_as_code": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:39.701913+00:00",
      "vendor_status": "hyperscaler",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "A first-party Google Cloud service with roughly weekly release notes. The SLA is tiered rather than single-figure: 99.9% for training, deployment and batch prediction, but 99.5% for custom-model online prediction and pipelines.",
      "last_release_date": "2026-08-23",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://cloud.google.com/vertex-ai/sla",
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Models and generative features are exposed as regional endpoints plus a global endpoint, all operated by Google.",
      "logs_by_default": "none",
      "logs_note": "Request-response logging is disabled by default and can be enabled per model and project, writing to a BigQuery table you designate. Two exceptions: abuse-monitoring logging, and Gemini's in-memory caching of customer data with a 24-hour time to live.",
      "retention_days": 0,
      "retention_note": "Zero by default, with named exceptions: 30 days for Grounding with Google Search or Maps, three days for certain grounding-derived queries, and 24 hours for in-memory cache and session resumption.",
      "zdr_status": "yes",
      "trains_on_customer_data": "no",
      "training_note": "Under the training restriction in the service terms, Google will not use your data to train or fine-tune any AI or ML model without your prior permission, across all managed models on Vertex AI.",
      "subprocessors_url": "https://cloud.google.com/terms/subprocessors",
      "regions_note": "Around 29 regional endpoints across the Americas, Europe, Asia-Pacific and the Middle East, plus jurisdictional multi-region endpoints that keep processing inside a boundary. Google explicitly warns against the global endpoint if you have processing requirements, because you cannot know which region handles the request.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II, issued quarterly",
          "status": "certified"
        },
        "fedramp": {
          "detail": "High P-ATO",
          "status": "certified"
        },
        "gdpr_dpa": {
          "detail": "Cloud Terms DPA and subprocessor list",
          "status": "certified"
        },
        "iso27001": {
          "detail": "accredited certification for the Cloud ISMS",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "BAAs entered as necessary, covering the whole infrastructure",
          "status": "available"
        }
      },
      "guardrails_location": "vendor_saas",
      "guardrails_note": "Google may log prompts to detect abuse and policy violations as part of delivering the service, which happens on Google's side.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference, bundled routing; per-token model pricing plus separately metered grounding, caching storage and provisioned throughput. No platform fee, subscription or per-seat charge — routing layer effectively $0.",
      "min_commitment": "None for pay-as-you-go. Provisioned Throughput is sold in GSUs with 1-week, 1-month, 3-month and 1-year commitments. Flexible Savings Plans are spend-based committed use discounts.",
      "overage_note": "No log/trace retention or request-volume overage. Feature meters: grounding with Google Search $35 per 1,000 requests above 1,500/day (Gemini 2.0 Flash), Web Grounding for enterprise $45 per 1,000 requests, grounding with your data $2.50 per 1,000 prompts. Explicit context cache storage billed per token-hour ($0.0000045/token-hour for Gemini 3.1 Pro / 3 Pro / 2.5 Pro; $0.000001 for Flash/Flash-Lite).",
      "cache_support": "hybrid",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Implicit cached input plus explicit Context Cache (prefix-based, not semantic). Prices cached input tokens, cache hits, explicit context-cache storage, and Anthropic-style cache writes. Examples: Gemini 3.1 Pro cached input $0.20/1M vs full input; Gemini 2.5 Pro $0.125 (≤200K) / $0.25 (>200K); Claude Opus 5 cache hit $0.50/1M vs 5-min cache write $6.25/1M; Claude Sonnet 5 cache hit $0.20 promotional / $0.30 from Sep 1 2026; DeepSeek-V3.1 cache hit $0.06; GLM-5 cache hit $0.10. Gemini implicit caching has no cache-write charge beyond storage; Anthropic models on Vertex charge explicit 5-min and 1-hour cache write rates above the input rate. A single universal cached-token discount or premium percentage is not stated. Explicit caching adds a storage meter (per token-hour) and Anthropic-style writes cost more than plain input — short-lived or low-reuse caches can be a net loss.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Not stated as a cost control on the pricing page.",
          "tier": null,
          "supported": null
        },
        "spend_webhooks": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated on the pricing page.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Billing tags are the only stated mechanism — 'to split out Computer Use costs, apply billing tags'. Per key/user/team/customer not stated. Published cost levers: Flexible Savings Plans, Provisioned Throughput GSUs, Model Optimizer configured for cost/quality/balance.",
      "cost_export": "Not stated on the pricing page.",
      "product_kind_note": "Google describes Model Garden as \"a single place to discover, customize, and deploy a wide variety of models from Google and Google partners\" on the Gemini Enterprise Agent Platform ([Model Garden](https://cloud.google.com/model-garden)), with inference served by the Gemini API in Vertex AI (\"Use `generateContent` or `streamGenerateContent` to generate content with Gemini\") ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). It is a cloud model platform, first-party plus partner catalog.",
      "deployment_modes": [
        "saas",
        "hybrid_vpc"
      ],
      "deployment_note": "Self-deployable partner models must first be purchased through Google Cloud Marketplace (\"you must first purchase them through Google Cloud Marketplace\"), via a \"Contact sales\" form and a sales representative, then deployed with \"one-click deployment\" to a Vertex AI Endpoint or with the Vertex AI SDK for Python (`model_garden.OpenModel(...).deploy(machine_type=..., accelerator_type=..., serving_container_image_uri=..., use_dedicated_endpoint=True, accept_eula=True)`); \"self-deployed models aren't serverless\" and \"only the shared public endpoint type is supported for these deployments\" ([Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09; [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11). Customers can apply existing Compute Engine reservations and committed-use discounts, but \"you cannot export the weights of self-deployed partner models\" ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "not_documented",
      "api_surface_note": "Native surface is proprietary: `generateContent` / `streamGenerateContent` over REST or the Google Gen AI SDK (Python `client.models.generate_content(...)`, Go `client.Models.GenerateContent(...)`), with streaming via `generate_content_stream` / `GenerateContentStream` ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). An OpenAI-library base-URL swap is documented against the `.../endpoints/openapi` path, but with gaps: unsupported parameters are silently dropped (\"If you pass any unsupported parameter, it is ignored\"), `candidateCount` (multiple candidates) is preview and does not work with `streamGenerateContent`, and for self-deployed Model Garden containers \"not every model deployed to these containers supports the Chat Completions API\" ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).",
      "model_count_low": 200,
      "model_count_high": 200,
      "model_count_source_note": "\"200+ available models\" ([Model Garden](https://cloud.google.com/model-garden), undated).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party platform plus partner catalog; no provider total is published. Sources are \"models from Google and Google partners\", with named third parties including Anthropic, Meta (Llama), Mistral AI, AI21 and TII ([Model Garden](https://cloud.google.com/model-garden)).",
      "first_party_vs_partner": "Documented mix: Google first-party foundation models (Gemini, Imagen, Veo, Chirp), open models (Gemma, CodeGemma, PaliGemma, Llama, Mistral, Falcon, BERT, T5-FLAN, ViT, EfficientNet), and third-party partner models such as \"Anthropic's Claude Model Family\"; partner models can be managed (MaaS) or self-deployed after a Marketplace license purchase ([Model Garden](https://cloud.google.com/model-garden); [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09).",
      "custom_endpoint_support": "Yes within Google Cloud: Model Garden lets customers \"self-deploy and serve open, partner, and custom models\", including deploying \"models with custom weights\" onto Vertex AI Endpoints using prebuilt vLLM or HF TGI containers; registering an arbitrary external private URL as a model is not documented ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).",
      "byok_mode": "not_applicable",
      "merchant_of_record": "Google Cloud: usage is billed to the Google Cloud account, charged \"only for requests that return a 200 response code\" and metered in tokens, cached tokens, images, video, audio, characters, requests, embeddings, node-hours and similar units; for self-deployed partner models \"you are billed for both the model usage and the underlying Gemini Enterprise Agent Platform infrastructure consumed\", with partner licenses obtained through Cloud Marketplace ([Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing); [Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11). New customers get \"up to $300 in free credits\" ([Model Garden](https://cloud.google.com/model-garden)).",
      "key_management_note": "Authentication is Google Cloud-native (project/location initialization, e.g. `vertexai.init(project=..., location=...)`, and the OpenAI-compat page defers to \"authentication and credentialing with the OpenAI-compatible syntax\"), so no third-party provider secrets are stored by the platform ([Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09; [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29). Self-deployed models run inside the customer's own project and VPC network, keeping serving traffic within customer-controlled infrastructure ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), 2025-12-09). CMEK/KMS specifics and vendor visibility of request content: n.a. on the pages fetched.",
      "api_openai_chat_completions_note": "Yes: an OpenAI-compatible surface at base URL `https://{location}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi`, called with `client.chat.completions.create(...)` and models such as `google/gemini-2.0-flash-001` ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), last updated 2026-06-29).",
      "api_anthropic_messages_note": "n.a. on the pages fetched (Claude is offered as a partner model, but no Anthropic Messages endpoint is documented here) ([Model Garden](https://cloud.google.com/model-garden)).",
      "api_openai_responses_note": "n.a.",
      "api_embeddings_note": "Yes as a billed capability/unit (\"embedding\" units are listed among the charged units) ([Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing)); a specific embeddings endpoint path is not on the pages fetched: n.a.",
      "api_images_note": "Yes: first-party text-to-image with \"Imagen and Gemini 3 Pro Image for text-to-image\" and \"Veo for text-to-video and image-to-video\", with image-generation and video-generation billing units ([Model Garden](https://cloud.google.com/model-garden); [Vertex AI generative AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing)). Endpoint paths: n.a.",
      "api_audio_note": "Yes, both directions as products: \"Chirp for speech-to-text\" and \"Text-to-speech\" are listed pre-trained APIs ([Model Garden](https://cloud.google.com/model-garden)); Gemini also accepts audio input MIME types (`audio/mpeg`, `audio/mp3`, `audio/wav`) with preview `audioTimestamp` support ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)). Endpoint paths: n.a.",
      "api_batch_note": "n.a. on the pages fetched.",
      "deployment_modes_note": "hosted cloud service (managed/MaaS models that \"don't require manual deployment\") and customer-project self-deployment, where self-deployed models \"run securely within your Google Cloud project and VPC network\" on chosen on-demand hardware ([Overview of self-deployed models](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-garden/self-deployed-models), last updated 2026-08-11; [Deploy partner models from Model Garden](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/deploy-partner-models), last updated 2025-12-09). Self-host outside Google Cloud, on-prem, air-gapped: n.a.",
      "byok_mode_note": "not_applicable: Google hosts the models and access is authenticated with Google Cloud project credentials; there are no upstream provider keys to supply ([Generate content with the Gemini API in Vertex AI](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference); [Call Vertex AI models using the OpenAI library](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), 2026-06-29).",
      "access_meta": {
        "qualified": [
          "api_anthropic_messages",
          "api_embeddings",
          "api_audio",
          "api_batch"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` on the fetched Vertex pages ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented` as a service setting. Google's guidance is client-side: \"Implement a retry strategy by using truncated exponential backoff.\" Default count: `n.a.` ([Vertex AI error code 429](https://cloud.google.com/vertex-ai/generative-ai/docs/error-code-429)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "No model/provider fallback list ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_configurable",
      "rel_health_check_note": "`not_configurable`; the closest documented behavior is automatic capacity-aware rerouting on the global endpoint: \"it dynamically routes your request to a region with available capacity\" and if a cached region is at capacity \"the system will automatically try the next available region to serve the request\" ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).",
      "rel_multi_region_config": "not_configurable",
      "rel_multi_region_note": "`not_configurable` beyond endpoint choice, and Google runs the routing. Setting `location` to `GLOBAL` selects a global entry point that \"does not tie your request to a single region\" and is recommended when \"architecting for maximum resilience against regional disruptions\", with the caveat that it \"does not guarantee that requests will be processed in any specific location\". Multi-region endpoints keep processing inside a jurisdiction: `us` → `https://aiplatform.us.rep.googleapis.com`, `eu` → `https://aiplatform.eu.rep.googleapis.com` ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai), [Vertex AI deployments and endpoints](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)).",
      "rel_load_balancing_config": "not_configurable",
      "rel_load_balancing_note": "Region selection by available capacity is automatic, with \"a new, separate global quota\" for the global endpoint; no weights or policy keys ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).",
      "architecture_class": "hyperscaler_api",
      "architecture_note": "Managed Google Cloud service consumed via `google-genai` / `vertexai` SDKs (Python, Go) or REST; no runtime disclosed ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "`no` ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).",
      "latency_claims": [],
      "throughput_claims": [],
      "streaming_support": "yes",
      "streaming_note": "Supported: `streamGenerateContent`, `client.models.generate_content_stream` (Python), `client.Models.GenerateContentStream` (Go), and `stream=True` through the OpenAI-compatible surface. Caveat: multiple candidates (`candidateCount`) is a Preview feature that works with `generateContent` but \"streamGenerateContent is not supported\" ([Vertex AI Gemini inference reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Two layers. Non-configurable Vertex safety filters block CSAM and personally identifiable information, surfacing an SPII finish reason (`sync_block`) ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)). Model Armor uses Sensitive Data Protection templates: with `INSPECT_ONLY` it flags (`sync_observe`), and with `INSPECT_AND_BLOCK` it \"issues a block verdict\" (`sync_block`). Notably, \"Model Armor doesn't pass the de-identified data—such as masked, redacted, or hashed content—back\" to the platform, so de-identification is not a pass-through redaction ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Vertex's JAILBREAK classifier is available but **off by default** ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)); Model Armor's `pi_and_jailbreak_filter_settings` with a confidence level \"blocks the prompt or response\" when enforcement is `INSPECT_AND_BLOCK`, and the blocked response carries `blockReason: MODEL_ARMOR` ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))",
      "guard_content": "sync_block",
      "guard_content_note": "Configurable harm categories block at chosen thresholds ([Configure safety filters](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters)); Model Armor Responsible AI filters cover `HATE_SPEECH`, `HARASSMENT`, `DANGEROUS`, `SEXUALLY_EXPLICIT` with confidence levels ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policy takes the form of Model Armor templates (filter settings and confidence levels) plus project- or org-level **floor settings** that impose a minimum; document and image screening and multi-language detection are template options ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration), [Model Armor overview](https://cloud.google.com/security-command-center/docs/model-armor-overview))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages — the Model Garden availability page states nothing about access restriction ([Available models](https://cloud.google.com/vertex-ai/generative-ai/docs/model-garden/available-models))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [
        "Google Model Armor",
        "Sensitive Data Protection"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` — the docs specify enforcement modes (`INSPECT_ONLY` vs `INSPECT_AND_BLOCK`, floor settings defaulting to `INSPECT_ONLY` on the gcloud path) but not what happens if Model Armor is unreachable ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))",
      "log_content": "nothing",
      "log_content_note": "Request-response logging must be enabled explicitly; when on, `full_request` and `full_response` (i.e. prompt and completion text) are written to BigQuery, with an `enableOtelLogging` option ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`set_request_response_logging_config(enabled=False)` disables it; Cloud Monitoring metrics are unaffected ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "tracing_support": "otel",
      "tracing_note": "Partially — `enableOtelLogging` is documented as an option on request-response logging; a full OTel agent-trace model is not described on the fetched pages ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "export_destinations": [
        "Cloud Logging",
        "Cloud Monitoring",
        "BigQuery"
      ],
      "export_destinations_note": "BigQuery tables for request-response logs ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging)); Cloud Logging for Model Armor verdicts via `enableCloudLogging` ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` on the fetched pages; logged traffic lands in BigQuery, where the customer can run their own scoring ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "observability_requires_saas": null,
      "log_retention_note": "Governed by the destination BigQuery dataset's own table/partition expiration; no Vertex-side default stated ([Request-response logging](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/request-response-logging))",
      "production_meta": {
        "sources": {},
        "qualified": [
          "guard_injection"
        ],
        "latency_prose": "`n.a.` Only qualitative statements (\"fast responses worldwide\"; prompt caching gives \"lower latency and lower costs\"; cached requests routed \"for the lowest latency\") ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai)).",
        "multi_surface": {},
        "throughput_prose": "`n.a.` Throughput is a reservation product: Provisioned Throughput reserves capacity for specific models ([Vertex AI error code 429](https://cloud.google.com/vertex-ai/generative-ai/docs/error-code-429)).",
        "benchmark_independence": "No figures published ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai))."
      },
      "guard_pii_default": "sync_block",
      "guard_injection_default": "off_by_default",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Resilience and data residency pull in opposite directions here: the global endpoint is the resilience play, regional/multi-region endpoints are the residency play, and Google's own recommendation is global for primary traffic with regional as a secondary path. Constraints to plan around: the Claude global endpoint supports pay-as-you-go only, Provisioned Throughput is regional-endpoint only, you should not send the same request to both endpoints simultaneously, and Private Google Access is not supported for multi-region endpoints (use Private Service Connect) ([Vertex AI global endpoint for Claude](https://cloud.google.com/blog/products/ai-machine-learning/global-endpoint-for-claude-models-generally-available-on-vertex-ai), [Vertex AI deployments and endpoints](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)).",
      "guard_notes": "Floor settings are the strongest governance primitive here: an organisation can force a minimum Model Armor enforcement for every project so a team cannot dial its own template down ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration)). Sanitising prompts or responses containing documents is not supported in the Vertex integration ([Model Armor Vertex integration](https://docs.cloud.google.com/model-armor/model-armor-vertex-integration)).",
      "fit_agent_features_note": "Function and tool calling are documented for Vertex integrations, including OpenAI `tools` with `tool_choice=\"auto\"`, Vercel AI SDK tools such as `googleVertex.tools.codeExecution`, and Agent Platform classes including `FunctionDeclaration`, `Tool`, and `AutomaticFunctionCallingResponder`. Google documentation also mentions ADK agents, multi-agent systems, A2A communication, and Agent Engine deployment. ([Agent Platform Python SDK reference](https://docs.cloud.google.com/python/docs/reference/agentplatform/latest))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. Authentication is a Google express-mode API key or ADC ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))",
      "fit_client_sdk_langs": [
        "Python",
        "Go",
        "Java",
        "Node.js"
      ],
      "fit_client_sdks_note": "Documented client libraries include the OpenAI library for Python and TypeScript/JavaScript, REST API, Vercel’s `@ai-sdk/google-vertex`, LangChain’s `langchain-google-vertexai`, Google’s `google-cloud-aiplatform` and `google-genai` packages, and LlamaIndex’s Vertex integration. The Vertex AI SDK is documented for Python, Go, Java, and Node.js. ([Vertex AI SDK language support](https://docs.cloud.google.com/gemini-enterprise-agent-platform/machine-learning/python-sdk/use-vertex-ai-sdk))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Google Cloud authentication is documented through Application Default Credentials and service accounts. The OpenAI integration uses `google.auth.default(scopes=[\"https://www.googleapis.com/auth/cloud-platform\"])`, refreshes credentials with `google.auth.transport.requests.Request()`, and passes `credentials.token` as `api_key`; the Vercel provider also documents `google-auth-library`, `GOOGLE_APPLICATION_CREDENTIALS`, and Edge credentials such as `GOOGLE_CLIENT_EMAIL` and `GOOGLE_PRIVATE_KEY`. ([Vertex AI OpenAI compatibility guide](https://cloud.google.com/vertex-ai/generative-ai/docs/start/openai?hl=es))",
      "fit_cloudflare_workers": "user_facing",
      "fit_cloudflare_workers_note": "Cloudflare Workers are explicitly cited as a supported Edge runtime for the Vercel AI SDK Google Vertex provider through `@ai-sdk/google-vertex/edge`. ([Vercel AI SDK Google Vertex provider](https://ai-sdk.dev/providers/ai-sdk-providers/google-vertex))",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented) — no credit-card, free-tier or trial-credit wording; billing enablement is required for existing (non-express) users ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start), [Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))",
      "fit_ecosystem_notes": "Vertex AI documentation covers Gemini models, Anthropic Claude partner models, xAI Grok partner models, Vertex MaaS open models, Model Garden, Vector Search, Agent Engine, and Google Cloud MCP services. The OpenAI-library integration is in preview and uses one-hour access tokens, while newer documentation directs users toward the Google Gen AI SDK; several older Vertex AI SDK modules are deprecated as of June 24, 2025 and scheduled for removal on June 24, 2026. ([Agent Platform Python SDK reference](https://docs.cloud.google.com/python/docs/reference/agentplatform/latest))",
      "fit_eval_prereqs": "A valid `@gmail.com` Google Account; new users sign up for express mode, obtain an express-mode API key and enable the Vertex AI API; existing users need a Google Cloud project, billing enabled, the Vertex AI API enabled, and ADC or an API key bound to a service account, plus the installed gcloud CLI; roles: `roles/aiplatform.user`, and possibly `roles/resourcemanager.projectCreator` and `roles/serviceusage.serviceUsageAdmin` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `client = genai.Client(http_options=HttpOptions(api_version=\"v1\"))` then `client.models.generate_content(model=\"gemini-2.5-flash\", contents=\"How does AI work?\")`; a REST `curl -X POST ...:generateContent` form is also shown ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start)). The Studio quickstart shows no request snippet ([Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))",
      "fit_gateway_platform": "platform_gateway",
      "fit_gateway_platform_note": "Google Cloud documents the GKE Inference Gateway API for ingress traffic, fine-grained routing, advanced load balancing, and centralized policy attachment. ([GKE inference workload deployment](https://docs.cloud.google.com/application-design-center/docs/ai-pretrained-inference-gke-cluster-workload))",
      "fit_helm_chart": "GKE documentation only; no Vertex AI chart",
      "fit_kubernetes": "generic_platform_only",
      "fit_kubernetes_note": "No Vertex-AI-specific Helm chart or operator is documented; Google documents generic Helm delivery to GKE (Artifact Registry Helm repositories, sample `hello-chart`, cluster `chart-cluster`) and a GKE inference example serving `google/gemma-7b-it` with the `vllm/vllm-openai` image. ([Google Artifact Registry Helm charts docs](https://docs.cloud.google.com/artifact-registry/docs/helm/store-helm-charts))",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Google Cloud documents remote MCP servers for using Google products and services in AI agents and applications. Google’s MCP Toolbox for Databases is documented as an open-source MCP server, including deployment alongside a Vertex AI Agent Engine workflow; Vertex-specific MCP gateway, registry, or tool-proxy functionality is not stated. ([Google Cloud MCP documentation](https://docs.cloud.google.com/mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_credit_card"
        ]
      },
      "fit_model_gate": "enablement_step",
      "fit_model_gate_note": "Yes, an API enablement step: \"Enable the Vertex AI API in the console.\" ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))",
      "fit_onboarding_notes": "Environment setup requires `GOOGLE_CLOUD_PROJECT`, `GOOGLE_CLOUD_LOCATION=global`, `GOOGLE_GENAI_USE_VERTEXAI=True`, plus `gcloud init` and `gcloud auth application-default login`. Express mode is the lower-effort branch for new users. The console-only path (Prompt Gallery → Submit) produces output without any code, and generated Python can be opened in a Colab Enterprise notebook; the default model there is `gemini-3.1-pro-preview` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start), [Quickstart: Send text prompts to Gemini using Vertex AI Studio](https://cloud.google.com/vertex-ai/generative-ai/docs/start/quickstarts/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Vertex AI documents using the OpenAI Python client with `api_key=credentials.token` and `base_url`. The documented base URLs are `https://{location}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi` and, in the Chinese page, `https://aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/endpoints/openapi`. ([Vertex AI OpenAI compatibility guide](https://cloud.google.com/vertex-ai/generative-ai/docs/start/openai?hl=es))",
      "fit_prod_prereqs": "A Google Cloud project with billing enabled, the Vertex AI API enabled, ADC or a service-account-bound API key, and the Vertex AI User role `roles/aiplatform.user` ([Quickstart: Generate text using the Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "cloud_console",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "LangChain documents Vertex AI integrations through `langchain-google-vertexai`, including `VertexAI`, `ChatVertexAI`, `VertexAIModelGarden`, `GemmaVertexAIModelGarden`, and `GemmaChatVertexAIModelGarden`; the integration page is marked deprecated and directs users to `GoogleGenerativeAI`. LlamaIndex also documents a Vertex AI integration and says Google GenAI, installed with `google-genai`, provides the same Vertex functionality. ([LangChain Google Vertex AI integration](https://docs.langchain.com/oss/python/integrations/llms/google_vertex_ai))",
      "fit_quickstart_steps": 15,
      "fit_quickstart_steps_caveat": "15 on the API quickstart. A separate console-only Vertex AI Studio quickstart has 12 steps and needs no code.",
      "fit_quickstart_url": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "platform_provider_resources",
      "fit_terraform_note": "The HashiCorp Google provider documents Vertex AI Terraform resources including `google_vertex_ai_index_endpoint`, `google_vertex_ai_index`, and `google_vertex_ai_dataset`; examples also use the `google-beta` provider. ([Terraform Google Vertex AI index endpoint](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/vertex_ai_index_endpoint))",
      "fit_terraform_ref": "hashicorp/google: google_vertex_ai_index_endpoint, google_vertex_ai_index, google_vertex_ai_dataset",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "The official Vercel AI SDK provider package is `@ai-sdk/google-vertex`. Documented submodules include `@ai-sdk/google-vertex/edge`, `/anthropic`, `/xai`, and `/maas`. ([Vercel AI SDK Google Vertex provider](https://ai-sdk.dev/providers/ai-sdk-providers/google-vertex))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/google-vertex"
    },
    {
      "slug": "truefoundry-ai-gateway",
      "name": "TrueFoundry AI Gateway",
      "company": "TrueFoundry",
      "tagline": "Closed-source enterprise AI gateway sold on request tiers, deployable as SaaS or inside the customer's own cloud.",
      "category": "cloud_platform",
      "deployment": "both",
      "license": "Proprietary",
      "founded_year": 2021,
      "hq_country": "India",
      "url_home": "https://www.truefoundry.com/ai-gateway",
      "url_docs": "https://docs.truefoundry.com/",
      "url_pricing": "https://www.truefoundry.com/pricing",
      "url_status": "https://status.truefoundry.com",
      "url_github": null,
      "url_changelog": null,
      "url_models_api": null,
      "github_stars": null,
      "model_count": 1600,
      "model_count_note": "Vendor-stated 1,600+ models (chat, completion, embedding, reranking) on the AI Gateway page; other vendor pages cite 250+ LLMs.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": true,
      "byok_fee_note": "You connect your own provider accounts and keys; TrueFoundry bills by request tier, not tokens.",
      "seat_fee_usd_month": null,
      "free_tier": "Developer plan $0/month: 50,000 requests/month, 3 users, 50,000 MCP tool calls, up to 10 saved prompts.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Pro plan",
          "amount": "$499/month (1M requests, 10 users)"
        },
        {
          "label": "Pro Plus plan",
          "amount": "$2,999/month (1M requests, 25 users)"
        },
        {
          "label": "Request/key overage",
          "amount": "$499/month per additional 2M requests and 5 API keys"
        }
      ],
      "self_host_cost_note": "No open-source edition; the data plane is proprietary. Vendor estimates roughly $600-$1,000/month of your own hosting cost to run the gateway (or control plus gateway) plane in your cloud, on top of the plan fee; fully managed SaaS carries no hosting cost. VPC, on-prem, air-gapped and multi-cloud installs are enterprise deployments, and a third-party review notes Helm-based ops effort is required.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": true,
      "portability_note": "Export is requested by email and returned as JSON, not self-serve.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "OpenAI Embeddings",
        "Rerank",
        "MCP"
      ],
      "sdk_note": "Unified OpenAI-compatible endpoint plus a playground and virtual models; MCP servers are proxied through the same gateway.",
      "cap_failover": true,
      "cap_load_balancing": true,
      "cap_conditional_routing": true,
      "cap_cache_simple": true,
      "cap_cache_semantic": true,
      "cap_observability": true,
      "cap_prompt_management": true,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": true,
      "cap_budgets": true,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": true,
      "cap_embeddings": true,
      "cap_image_gen": null,
      "cap_audio": null,
      "cap_video": null,
      "cap_batch": null,
      "latency_overhead_ms": 3,
      "latency_source_note": "Vendor claim (TrueFoundry): \"sub-3 ms internal latency\" with 99.99% uptime and 10B+ requests/month. An independent review (llmtools.cc) records the same figure as vendor-claimed ~3-4 ms with 350+ RPS on a single vCPU and states it is not independently verified; no third-party benchmark was found.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": null,
      "gdpr": true,
      "eu_residency": null,
      "eu_residency_note": "Deployment inside the customer's own cloud region is offered (\"no data leaves your domain\"), but no named EU residency commitment is published.",
      "zdr": null,
      "zdr_note": "No ZDR commitment. The equivalent is disabling request logging, or keeping the gateway in your own infrastructure.",
      "air_gapped": true,
      "best_for": "Enterprises that want a fully managed or in-VPC AI gateway with guardrails, MCP governance and SSO/RBAC, and are comfortable with closed source.",
      "strengths": [
        "Broad routing feature set: weight, latency and priority routing, fallback chains, geo-aware routing and virtual models",
        "Both simple and semantic caching plus budget and rate-limit controls per team, model and key",
        "Deployable in the customer's VPC, on-prem, air-gapped or multi-cloud with SOC 2 and HIPAA/GDPR-ready configurations",
        "Detailed built-in observability (P50/P90/P99, TTFT, traces, alerts) and MCP gateway with RBAC"
      ],
      "weaknesses": [
        "Not open source: the data plane is proprietary, so there is no free self-host path or community audit",
        "Steep jump from the $0 Developer tier to $499/month Pro, with overage billed in $499 blocks",
        "Performance figures are vendor claims only; a third-party review explicitly flags them as unverified",
        "Self-hosting adds an estimated $600-$1,000/month of infra plus Helm/Kubernetes operational burden"
      ],
      "avoid_if": "You require an open-source, auditable gateway or want to avoid per-request pricing tiers as volume grows.",
      "reviews": [
        {
          "url": "https://www.reddit.com/r/LLMDevs/comments/1uz24rc/one_ai_gateway_for_a_handful_of_internal_teams/",
          "why": "15-comment selection thread weighing TrueFoundry against LiteLLM, Portkey, Kong and Databricks' gateway for multi-team internal use, including budgeting surprises.",
          "date": "2026-07-17",
          "type": "community",
          "title": "One AI gateway for a handful of internal teams, where did you land?",
          "publisher": "Reddit - r/LLMDevs"
        }
      ],
      "sources": {
        "gdpr": "https://www.truefoundry.com/ai-gateway",
        "soc2": "https://www.truefoundry.com/ai-gateway",
        "cap_mcp": "https://www.truefoundry.com/ai-gateway",
        "license": "https://llmtools.cc/tool/truefoundry/",
        "sdk_note": "https://www.truefoundry.com/ai-gateway",
        "url_docs": "https://docs.truefoundry.com/",
        "free_tier": "https://www.truefoundry.com/pricing",
        "air_gapped": "https://www.truefoundry.com/ai-gateway",
        "extra_fees": "https://www.truefoundry.com/pricing",
        "hq_country": "https://www.f6s.com/company/truefoundry",
        "url_status": "https://status.truefoundry.com",
        "weaknesses": "https://llmtools.cc/tool/truefoundry/",
        "cap_budgets": "https://www.truefoundry.com/ai-gateway",
        "model_count": "https://www.truefoundry.com/ai-gateway",
        "api_surfaces": "https://www.truefoundry.com/ai-gateway",
        "cap_failover": "https://www.truefoundry.com/ai-gateway",
        "founded_year": "https://www.f6s.com/company/truefoundry",
        "product_fate": "https://siliconangle.com/2026/06/25/truefoundry-acquires-mlops-pioneer-seldon-ai-accelerate-enterprise-agentic-ai/",
        "regions_note": "https://www.truefoundry.com/docs/platform/security-and-compliance",
        "vendor_status": "https://www.truefoundry.com/blog/announcing-our-19m-series-a-scaling-ai-deployment-with-autonomous-agents-on-autopilot",
        "byok_supported": "https://www.truefoundry.com/ai-gateway",
        "cap_embeddings": "https://www.truefoundry.com/ai-gateway",
        "cap_guardrails": "https://www.truefoundry.com/ai-gateway",
        "certifications": "https://www.truefoundry.com/docs/platform/security-and-compliance",
        "config_as_code": "https://www.truefoundry.com/docs/ai-gateway/virtual-model",
        "retention_days": "https://www.truefoundry.com/docs/platform/security-and-compliance",
        "sla_uptime_pct": "https://www.truefoundry.com/service-level-agreement",
        "cap_rate_limits": "https://www.truefoundry.com/ai-gateway",
        "logs_by_default": "https://www.truefoundry.com/docs/ai-gateway/request-logging",
        "cap_cache_simple": "https://www.truefoundry.com/ai-gateway",
        "model_count_note": "https://www.truefoundry.com/ai-gateway",
        "cap_observability": "https://www.truefoundry.com/ai-gateway",
        "cap_pii_redaction": "https://www.truefoundry.com/ai-gateway",
        "eu_residency_note": "https://www.truefoundry.com/ai-gateway",
        "last_release_date": "https://www.truefoundry.com/docs/changelog",
        "openai_compatible": "https://www.truefoundry.com/ai-gateway",
        "subprocessors_url": "https://www.truefoundry.com/docs/platform/security-and-compliance",
        "cap_cache_semantic": "https://www.truefoundry.com/ai-gateway",
        "cap_load_balancing": "https://www.truefoundry.com/ai-gateway",
        "guardrails_location": "https://www.truefoundry.com/docs/platform/security-and-compliance",
        "latency_overhead_ms": "https://www.truefoundry.com/ai-gateway",
        "latency_source_note": "https://llmtools.cc/tool/truefoundry/",
        "requires_vendor_sdk": "https://www.truefoundry.com/docs/ai-gateway/quick-start",
        "self_host_cost_note": "https://www.truefoundry.com/pricing",
        "truefoundry_pricing": "https://www.truefoundry.com/pricing",
        "cap_prompt_management": "https://www.truefoundry.com/ai-gateway",
        "data_export_available": "https://docs.truefoundry.com/ai-gateway/export-logstraces",
        "prompt_transits_vendor": "https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment",
        "cap_conditional_routing": "https://www.truefoundry.com/ai-gateway"
      },
      "verified_at": {
        "gdpr": "2026-08-29",
        "soc2": "2026-08-29",
        "cap_mcp": "2026-08-29",
        "license": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "free_tier": "2026-08-29",
        "air_gapped": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "weaknesses": "2026-08-29",
        "cap_budgets": "2026-08-29",
        "model_count": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "cap_failover": "2026-08-29",
        "founded_year": "2026-08-29",
        "byok_supported": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "cap_cache_simple": "2026-08-29",
        "model_count_note": "2026-08-29",
        "cap_observability": "2026-08-29",
        "cap_pii_redaction": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "cap_cache_semantic": "2026-08-29",
        "cap_load_balancing": "2026-08-29",
        "latency_overhead_ms": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "cap_prompt_management": "2026-08-29",
        "data_export_available": "2026-08-29",
        "cap_conditional_routing": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:47.93379+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed and expanding. Raised a $19M Series A led by Intel Capital in January 2026 and was itself the acquirer of MLOps vendor Seldon Technologies in June 2026. Note that the uptime commitment applies to Enterprise customers only; other plans get best-effort availability.",
      "last_release_date": "2026-08-21",
      "sla_uptime_pct": 99.9,
      "sla_url": "https://www.truefoundry.com/service-level-agreement",
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "depends",
      "prompt_path_note": "Three genuinely different data paths: managed SaaS where request and response data flows through TrueFoundry's control plane; gateway-in-your-infrastructure where all LLM traffic stays with you and TrueFoundry is not in the live request path; and fully self-hosted, where everything except the authentication and analytics servers runs on your infrastructure.",
      "logs_by_default": "full_content",
      "logs_note": "On hosted deployments logging happens unless you suppress it: absent or true in the logging-config header means logging occurs. You can control whether bodies are logged and redact patterns, and a deny rule always wins.",
      "retention_days": 90,
      "retention_note": "Ninety days for SaaS blob storage, configurable. In self-hosted modes the control plane does no long-term retention.",
      "zdr_status": "not_published",
      "trains_on_customer_data": "not_published",
      "training_note": "Neither the security page nor the deployment docs address training.",
      "subprocessors_url": "https://www.truefoundry.com/docs/platform/security-and-compliance",
      "regions_note": "You choose the region where the gateway is deployed and where logs and traces are stored, and you can bring your own S3, Azure Blob or GCS bucket. Gateway planes run active-active across multiple regions.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II programme for TrueFoundry SaaS",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "standard DPAs available",
          "status": "available"
        },
        "iso27001": {
          "detail": "asserted in deployment docs but absent from the security page's programme list",
          "status": "claimed"
        },
        "hipaa_baa": {
          "detail": "HIPAA programme maintained",
          "status": "certified"
        }
      },
      "guardrails_location": "both",
      "guardrails_note": "The gateway plane evaluates PII, prompt-injection and content-filtering policy in memory on the request path — and in self-hosted modes that plane is yours.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "platform_fee",
      "pricing_model_note": "Free developer tier + flat monthly platform fee + per-unit usage overage + enterprise quote + optional self-host infra cost. Developer $0/mo, Pro $499/mo, Pro Plus $2,999/mo, Enterprise custom. TrueFoundry does not resell inference.",
      "min_commitment": "None stated. Enterprise implied custom.",
      "overage_note": "Request overage priced in blocks: 2M requests and 5 API keys for additional $499/month; Pro Plus 'contact sales'. Self-hosting adds ~$600–$1,000/month of infrastructure cost when you self-host the Gateway or both the Control and Gateway planes. Log/trace retention overage and export cost not stated.",
      "cache_support": "hybrid",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": null,
      "cache_pricing_note": "Exact ('Simple caching') and semantic caching both listed as features; provider passthrough not stated. No cached-token pricing published. TrueFoundry charges per request/platform, not per token, so caching cuts your provider bill and you keep the saving — but cached requests presumably still consume the tier's request allowance, and the feature list does not mark which tier includes semantic caching.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "'Alerts' listed, tier unstated.",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "'Virtual models' are listed; per-tier API key counts (5 extra keys bundled with the $499 overage pack). Not branded as virtual keys explicitly.",
          "tier": null,
          "supported": true
        },
        "rate_limiting": {
          "note": "Listed, tier unstated.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "'RBAC on models' and 'RBAC on MCPs' are listed — allowlisting by role rather than a named allowlist.",
          "tier": null,
          "supported": true
        },
        "budget_caps_per_key": {
          "note": "Not stated as per-key.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "'Budget limiting' listed without scope or tier.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "'Cost per team/user/model/application' plus custom metadata and metadata filtering. Per-key, per-tag and per-customer not stated.",
      "cost_export": "'Export to custom storage buckets', 'Data Lake Export', 'Connect multiple storage bucket' and 'Export to other monitoring platforms'. CSV, cost API and webhook export not stated; export pricing not stated.",
      "product_kind_note": "TrueFoundry calls it the \"TrueFoundry AI Gateway\", \"the proxy layer that sits between your applications and the LLM providers and MCP Servers\", and \"an enterprise-grade platform that enables users to access 1000+ LLMs using a unified interface while taking care of observability and governance\", offering \"one endpoint with an OpenAI-compatible schema for every provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). It is a routing proxy with an observability/governance plane.",
      "deployment_modes": [
        "saas",
        "self_host",
        "hybrid_vpc",
        "on_prem"
      ],
      "deployment_note": "Four documented topologies with tier and cost: (1) \"AI Gateway SAAS only\" — Starter tier, $0 hosting, \"you don't need to deploy any infrastructure on your end\"; (2) \"SaaS AI Gateway with data storage on your own infrastructure\" — Starter, cost = \"storage cost in S3\"; (3) \"Gateway Plane only\" with TrueFoundry-hosted control plane — Enterprise, \"~$600/month\" hosting; (4) \"Control Plane + Gateway Plane\" on customer infrastructure — Enterprise, \"~$800-1000/month\" ([Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment)). Docker/Helm/Terraform specifics are not mentioned on the pages fetched: n.a. ([Self-hosted gateway](https://www.truefoundry.com/docs/ai-gateway/gateway-self-hosted)). Platform tiers are Developer $0, Pro $499/month, Pro Plus $2,999/month, Enterprise custom ([TrueFoundry pricing](https://www.truefoundry.com/pricing)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "yes",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap with the OpenAI SDK (`base_url=\"{GATEWAY_BASE_URL}\"`, or `OPENAI_BASE_URL`/`OPENAI_API_KEY` env vars), with ready snippets for OpenAI SDK, LangChain, LangGraph, LlamaIndex, Node.js, Google ADK and cURL; you need the Gateway Base URL, API key and Model ID ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)). The surface is broad: also `/rerank`, `/moderations`, `/files`, fine-tune, a Compaction API and `/proxy` for provider-native passthrough ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Streaming is marked supported for all listed chat providers (OpenAI, Azure OpenAI, Anthropic, Bedrock, Vertex, Cohere, Gemini, Groq, AI21, Cerebras, Wafer, SambaNova, Perplexity AI, Together AI, xAI, DeepInfra) with no documented caveats ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "model_count_low": 1000,
      "model_count_high": 1000,
      "model_count_source_note": "\"1000+ LLMs\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway), undated).",
      "provider_count_low": 27,
      "provider_count_high": 27,
      "provider_count_source_note": "n.a. as a stated total; the intro page enumerates 27 provider integrations (Gemini & Vertex AI, Google Gemini, AWS Bedrock, AWS SageMaker, Azure OpenAI, Azure AI Foundry, OpenAI, Cohere, Databricks, AI21, Anthropic, Together AI, xAI, DeepInfra, Perplexity AI, Mistral AI, Cloudera, Groq, ElevenLabs, Deepgram, Cartesia, Smallest AI, Snowflake Cortex, Self Hosted, OpenRouter, SambaNova, Cerebras) and adds \"if you don't see the provider you need, there is a high chance it will just work as self-hosted models or OpenAI provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway), undated).",
      "first_party_vs_partner": "All third-party or customer-run: TrueFoundry proxies provider APIs and customer-registered self-hosted models; the intro page does not claim any TrueFoundry-owned model hardware ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway); [Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)).",
      "custom_endpoint_support": "Yes: \"you can easily integrate any OpenAI-compatible self-hosted model\" by supplying \"the endpoint URL\" and optional \"Auth Data which will be used for Authentication of request to the model\"; documented targets include vLLM, Ollama, SGLang and Text Generation Inference running on \"your own infrastructure\", \"on-premises GPUs\", \"private cloud instances\", or \"any model server not deployed through TrueFoundry\", and registered models appear in the catalog next to cloud providers ([Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)).",
      "byok_mode": "byok_only",
      "merchant_of_record": "n.a. as an explicit statement about model usage; TrueFoundry invoices its platform tiers (Developer $0, Pro $499/month, Pro Plus $2,999/month, Enterprise custom, plus \"additional usage at transparent, per-unit rates\") while model calls run on the customer's own provider accounts ([TrueFoundry pricing](https://www.truefoundry.com/pricing); [Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)).",
      "key_management_note": "Upstream provider keys are held per Model Account, with separate accounts used to hold multiple keys for the same provider; self-hosted model registrations store optional \"Auth Data\" for upstream auth ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Self-hosted models](https://www.truefoundry.com/docs/ai-gateway/self-hosted-models)). Caller auth uses a Personal Access Token (for development) or Virtual Account Token (for applications) created in the platform's Access section, and the gateway offers \"RBAC and scoped keys for users, teams, and applications\" plus spend limits and per-user/per-model/per-application throttles ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Data handling varies by topology: on SaaS \"data is encrypted at rest and in transit\" on infrastructure that is \"SOC2, ISO27001, GDPR, and HIPAA compliant\"; with customer-managed storage, request-response data still flows through the TrueFoundry control plane, which \"might cache some of the data for faster queries, but doesn't do any long term retention of the data\" and requires bucket access (\"you will not be able to use this feature if you don't give access to TrueFoundry control plane access to your bucket\"); fully self-hosted keeps both planes on customer infrastructure ([Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "api_openai_chat_completions_note": "Yes: OpenAI-schema chat completions through the gateway base URL (`https://gateway.truefoundry.ai` for SaaS; self-hosted URLs come from the Playground code snippet), used with `OpenAI(api_key=..., base_url=\"{GATEWAY_BASE_URL}\")` ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start); [Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "api_anthropic_messages_note": "Yes as SDK compatibility: \"drop-in support for OpenAI, Anthropic, and other provider SDKs\", plus a `/proxy` API that can \"forward provider-native requests through the AI Gateway while keeping logging, rate limiting, and budget controls\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)). Explicit `/v1/messages` path: n.a.",
      "api_openai_responses_note": "n.a. (not documented on the pages fetched).",
      "api_embeddings_note": "Yes: \"Embedding (`/embeddings`)\", with a linked Embeddings API reference ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "api_images_note": "Yes: \"Image Generation (`/images/generations`)\", \"Image Edit (`/images/edits`)\", \"Image Variation (`/images/variations`)\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "api_audio_note": "Yes, both directions: Text To Speech and Speech to Text are documented, plus Audio Translation at `/audio/translations` and a Live/Realtime API ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "api_batch_note": "Yes: \"Batch (`/batches`)\" — \"run large workloads asynchronously at batch pricing\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway)).",
      "deployment_modes_note": "hosted SaaS (\"You can run the AI Gateway as fully managed SaaS… hosted in multiple regions across the world\"), self-host, hybrid/customer-cloud VPC, and on-prem: \"SaaS, hybrid, or fully self-hosted in your own VPC\" and \"Yes, the AI Gateway supports on-premise deployments on any infrastructure or cloud provider\" ([Introduction to AI Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway); [Deployment options](https://www.truefoundry.com/docs/ai-gateway/modes-of-deployment)). Air-gapped: n.a. (explicitly not mentioned on the deployment pages fetched).",
      "byok_mode_note": "byok_only: providers are added as Model Accounts — \"A Model Account represents one account of a model provider (e.g. OpenAI, Anthropic, AWS Bedrock)… You can add multiple accounts per provider, each with their own API keys\" — and models are added \"after providing API key\" ([Quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start)). No TrueFoundry-sold inference credits are documented; pricing is platform subscription plus usage ([TrueFoundry pricing](https://www.truefoundry.com/pricing)).",
      "access_meta": {
        "qualified": [
          "api_anthropic_messages",
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": "enumerated"
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` as a request-timeout key. What exists are SLA thresholds used for health, not for aborting a request: `sla_cutoff.time_to_first_token_ms` and `sla_cutoff.time_per_output_token_ms` ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).",
      "rel_retry_config": "config_file",
      "rel_retry_note": "`config_file` (also editable in the UI). `retry_config` with `attempts` (**default `0`**), `delay` (**default `100` ms**), and `on_status_codes` (**default `[\"429\",\"500\",\"502\",\"503\"]`**). Retries occur on the same target; backoff beyond the fixed `delay` is not documented ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).",
      "rel_retry_default_note": "`retry_config` with `attempts` (**default `0`**), `delay` (**default `100` ms**), and `on_status_codes` (**default `[\"429\",\"500\",\"502\",\"503\"]`**).",
      "rel_fallback_config": "config_file",
      "rel_fallback_note": "ORDERED for priority-based routing (`priority`, lower number = higher priority, `0` highest); fallback targets come from `load_balance_targets`, gated by `fallback_status_codes` (default `[\"401\",\"403\",\"404\",\"408\",\"429\",\"500\",\"502\",\"503\"]`) and `fallback_candidate` (default `true`). Ordering depends on strategy: latency-based puts lower-latency targets first; weight-based sticky routing tries remaining healthy targets in sequence and that order is explicitly \"not weight-based\" ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).",
      "rel_fallback_shape": "ordered",
      "rel_health_check_config": "config_file",
      "rel_health_check_note": "`config_file`, and it is passive health tracking plus cooldown ejection. The gateway \"continuously monitors every target\", marks failing targets unhealthy, moves them to the end of the list as last resort, and recovers automatically. Failure-based cooldown: 5xx/429/401/403, default threshold 2+ failures in a rolling 2-minute window. SLA-based cooldown (priority routing only) via `sla_cutoff.time_per_output_token_ms` / `time_to_first_token_ms`, averaged over a 3-minute rolling window using up to 10 samples with a minimum of 3; TTFT samples only come from streaming requests and TPOT samples require at least 30 output tokens ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). TrueFoundry's blog additionally describes classic circuit-breaker states (`closed`/`open`/`half_open`) as a pattern, but states no configuration keys for it ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).",
      "rel_multi_region_config": "config_file",
      "rel_multi_region_note": "SaaS requests are auto-tagged with `tfy_gateway_region` and `tfy_gateway_zone`, and you route on them with `metadata_match` (documented examples `US → azure-us/gpt-4o`, `EU → azure-eu/gpt-4o`, `IN → azure-in/gpt-4o`, and `metadata_match: {tfy_gateway_zone: SFO}`), with a catch-all `priority: 1` target as the cross-region fallback. Both are true here: TrueFoundry runs multiple SaaS gateway regions AND you configure the cross-region target chain yourself ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). Multiple gateways can also be deployed cross-region under one control plane ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).",
      "rel_load_balancing_config": "config_file",
      "rel_load_balancing_note": "Weights supported: `routing_config.type` of `weight-based-routing` (integer `weight` 0–100, must sum to 100), `priority-based-routing`, `latency-based-routing` (\"No weights needed\", selection by recent TPOT), and `complexity-based-routing` (`simple`/`medium`/`complex`). Targets are catalog identifiers (`azure/gpt-4o`, `openai/gpt-4o`, `bedrock/llama`, `onprem/llama`), with `metadata_match`, `headers_override` and `override_params` per target; rules are evaluated in order and the first match wins ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).",
      "architecture_class": "interpreted_proxy",
      "architecture_note": "`interpreted_proxy` (JS/edge runtime). \"The AI Gateway is written using the Hono framework\", described as \"ultra-fast, minimalistic, and is designed for the edge\" ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).",
      "self_host_data_plane": "yes",
      "self_host_artifact_note": "`yes`, but the artifact is not named on the fetched pages. Deployment is documented as SaaS, your VPC, on-prem or air-gapped, running as Kubernetes pods with readiness probes and optional `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` forward-proxy env vars; no Docker image name or Helm chart name is stated ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture), [TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).",
      "latency_claims": [
        {
          "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "+7",
          "metric": "not stated",
          "publisher": "vendor",
          "conditions": "200-220 RPS on 1 vCPU / 1 GB RAM, tracing disabled. Percentile, payload and cache state not stated."
        },
        {
          "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "+8",
          "metric": "not stated",
          "publisher": "vendor",
          "conditions": "Same hardware at 200-220 RPS with complete tracing enabled."
        },
        {
          "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture",
          "unit": "ms",
          "scope": "gateway_overhead",
          "value": "+12",
          "metric": "not stated",
          "publisher": "vendor",
          "conditions": "350-370 RPS with complete tracing — the only figure here showing overhead rising with load."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://www.truefoundry.com/docs/platform/gateway-plane-architecture",
          "unit": "RPS",
          "scope": "capacity",
          "value": "250",
          "metric": "sustained",
          "publisher": "vendor",
          "conditions": "Single pod, 1 CPU / 1 GB RAM, no degradation stated until about 350 RPS."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported, with the most precise streaming-fallback semantics in this set: the gateway waits for the \"first non-empty stream chunk\", and if Anthropic returns `overloaded_error` before that chunk it falls back to the next eligible target. Separate caveat: `prompt_version_fqn` \"does not work with agents (when using MCP/tools)\". Note also that TTFT SLA samples are only recorded on streaming requests ([TrueFoundry routing config](https://www.truefoundry.com/docs/ai-gateway/load-balancing-overview), [TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)).",
      "guard_pii": "sync_block",
      "guard_pii_note": "Built-in PII Detection and Azure AI Language PII/PHI guardrails ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview)). Custom plugins distinguish mutation from validation: a Presidio `POST /pii-redaction` plugin returns `transformed: true` with a full replacement `requestBody` to redact, whereas a Guardrails-AI `POST /pii-detection` plugin \"only detects and reports PII without redacting it\" ([Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))",
      "guard_injection": "sync_block",
      "guard_injection_note": "Azure Prompt Shield is a built-in guardrail integration; external options include Model Armor, Prisma AIRS, CrowdStrike, and NeMo ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview))",
      "guard_content": "sync_block",
      "guard_content_note": "Azure Content Safety moderation and OpenAI Moderations are supported; blocked requests return a configurable `custom_error_message` ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview), [Guardrails configuration](https://www.truefoundry.com/docs/ai-gateway/guardrails-configuration))",
      "guard_custom": "sync_block",
      "guard_custom_note": "Policy forms: built-in Regex and Secrets Detection rules, plus custom HTTP plugins bound to lifecycle hooks `llm_input`, `llm_output`, `mcp_tool_pre_invoke`, `mcp_tool_post_invoke`, returning either a validate verdict or a mutated request body ([Guardrails overview](https://www.truefoundry.com/docs/ai-gateway/guardrails-overview), [Guardrails configuration](https://www.truefoundry.com/docs/ai-gateway/guardrails-configuration), [Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Access control is applied at the model-account level, governing which users/teams and which PATs/VATs can reach which models ([Gateway access control](https://www.truefoundry.com/docs/ai-gateway/gateway-access-control))",
      "guard_execution_location": "both",
      "guard_third_party": [
        "Azure AI Language",
        "Azure Content Safety",
        "OpenAI Moderation",
        "Microsoft Presidio"
      ],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` as a global fail-open/fail-closed rule. For custom plugins, HTTP 4xx/5xx from the plugin signals a \"processing or dependency failure only; not used for 'PII found' policy outcomes\", which separates infrastructure failure from a policy verdict but does not state the resulting request disposition ([Custom guardrails](https://www.truefoundry.com/docs/ai-gateway/custom-guardrails))",
      "log_content": "configurable",
      "log_content_note": "Global logging mode is `HEADER_CONTROLLED` (default behaviour: log unless the header says otherwise), `ALWAYS`, or `NEVER`; logged requests, viewable under AI Gateway → Monitor → Requests, include request content, and sensitive values can be redacted per subject/model/metadata ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Set the global mode to `NEVER`, or send `X-TFY-LOGGING-CONFIG: {\"enabled\": false}` (stringified JSON) per request; metrics dashboards remain populated ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging), [Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))",
      "tracing_support": "otel",
      "tracing_note": "OpenTelemetry natively for both traces and metrics, with MCP tool and agent spans; trace storage is managed by TrueFoundry ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data))",
      "export_destinations": [
        "OpenTelemetry",
        "Datadog",
        "Prometheus",
        "Grafana",
        "S3",
        "Webhook"
      ],
      "export_destinations_note": "OTLP exporters to Grafana Cloud, Datadog (OTLP), and AWS CloudWatch, plus any OTLP endpoint ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data)); aggregated metrics download as CSV and per-datasource metrics APIs for model, MCP, guardrail, cache, routing, and agent metrics ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` — no feedback/score endpoint documented ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` on the fetched pages; the closest is `audit_mode_blocked`, a guardrail outcome that records what *would* have been blocked without blocking it ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics))",
      "observability_requires_saas": "Partially — the gateway is customer-deployed and telemetry can be exported to the customer's own stack, but \"Traces storage is managed by TrueFoundry\" ([Export OpenTelemetry data](https://www.truefoundry.com/docs/ai-gateway/export-opentelemetry-data))",
      "log_retention_note": "`n.a.` — no retention window stated ([Request logging](https://www.truefoundry.com/docs/ai-gateway/request-logging))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "Documented overhead table (metric: added latency in ms; hardware 1 vCPU / 1 GB RAM): at **200–220 RPS** `+7 ms` tracing disabled, `+7 ms` tracing enabled without request logs, `+8 ms` complete tracing; at **350–370 RPS** `+7 ms`, `+7 ms`, `+12 ms`. Percentile is not stated; payload and cache conditions are not stated ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)). The blog restates this as \"~7 ms overhead at 200 RPS even with tracing on\" and adds promotional figures **`~10ms`** (\"Built for Speed: ~10ms Latency, Even Under Load\") and **`~3–4 ms`**, neither with a stated percentile or conditions ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).",
        "multi_surface": {},
        "throughput_prose": "**250 RPS** on a single pod with 1 CPU / 1 GB RAM, scaling \"without any degradation in performance until about 350 RPS\" ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)); the blog states **350+ RPS** on 1 vCPU ([TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages)).",
        "benchmark_independence": "All vendor self-published. The docs table (with RPS, hardware and tracing conditions) is the credible artifact; the blog's `~10ms` and `~3–4 ms` figures carry no methodology and the `~3–4 ms` figure appears in a comparative context, so treat it as marketing rather than a measured result ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture), [TrueFoundry LLM failover blog](https://www.truefoundry.com/blog/llm-failover-load-balancing-provider-outages))."
      },
      "guard_pii_default": "choose_at_setup",
      "guard_injection_default": "choose_at_setup",
      "guard_content_default": "sync_block",
      "guard_custom_default": "choose_at_setup",
      "rel_notes": "Latency-based routing uses Time Per Output Token (total response time ÷ output tokens) over a 20-minute window with a 10-minute sticky epoch, chosen because TPOT folds TTFT and inter-token latency into one output-length-independent number ([TrueFoundry virtual models / fallback](https://www.truefoundry.com/docs/ai-gateway/fallback)). Control-plane/data-plane separation means gateway pods only take traffic after config sync (readiness probe) ([TrueFoundry gateway plane architecture](https://www.truefoundry.com/docs/platform/gateway-plane-architecture)).",
      "guard_notes": "TrueFoundry's guardrail analytics are the most enforcement-aware of any product here: outcomes are split into `allowed`, `blocked`, `mutated`, and `audit_mode_blocked`, with per-guardrail block/mutate rates and P50–P99 guardrail latency, so you can measure whether a control actually fires ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics)). One documentation inconsistency: the analytics page labels \"Total Flagged Requests\" as \"requests that were blocked by a guardrail\" ([Metrics dashboard](https://www.truefoundry.com/docs/ai-gateway/analytics)).",
      "fit_agent_features_note": "Through the OpenAI Agents SDK integration, TrueFoundry documents multi-agent workflows, handoffs, function tools, human-in-the-loop workflows, automatic tracing, session management, and conversation history across agent runs. The documentation does not state A2A support. ([TrueFoundry OpenAI Agents SDK integration](https://www.truefoundry.com/docs/ai-gateway/openai-agents-sdk))",
      "fit_byok_required": "yes",
      "fit_byok_required_note": "Effectively yes to have any model: \"Select the provider you want to add models from and add models after providing API Key,\" though the page does not state a first-call rule explicitly ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))",
      "fit_client_sdk_langs": [
        "Python",
        "Node.js",
        "Go",
        "Rust",
        "Java"
      ],
      "fit_client_sdks_note": "Documented client and integration formats include OpenAI SDK, LangChain, LangGraph, Google ADK, LlamaIndex, Go-OpenAI, Rust-OpenAI, LangChain4j, Node.js, cURL, REST, and streaming APIs. Official OpenAI Agents SDK packages are `openai-agents` for Python and `@openai/agents` for JavaScript/TypeScript; documented classes include `Agent`, `Runner`, `OpenAIChatCompletionsModel`, and `AsyncOpenAI`. ([TrueFoundry OpenAI Agents SDK integration](https://www.truefoundry.com/docs/ai-gateway/openai-agents-sdk))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "Yes for platform-level cloud access: TrueFoundry documents AWS IAM Roles for Service Accounts (IRSA, `sts:AssumeRoleWithWebIdentity` with the `eks.amazonaws.com/role-arn` annotation) and GCP GKE Workload Identity; Azure is listed as \"Coming Soon\". ([TrueFoundry docs - access cloud services with IAM](https://www.truefoundry.com/docs/access-data-from-s3-or-other-clouds-services))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated; pricing lists a \"$ 0 / month\" \"Developer\" tier, with no trial-credit or credit-card wording ([Pricing - TrueFoundry](https://www.truefoundry.com/pricing))",
      "fit_ecosystem_notes": "TrueFoundry supports fully managed SaaS, hybrid, private-VPC, on-premises, and fully self-hosted deployments, including use on any infrastructure or cloud provider and as a standalone gateway. The self-hosted Gateway Plane is stateless and connects securely to the TrueFoundry control plane over WebSockets; the gateway supports providers including OpenAI, Anthropic, Azure, and 1000+ LLMs overall. ([TrueFoundry introduction to the LLM Gateway](https://www.truefoundry.com/docs/ai-gateway/intro-to-llm-gateway))",
      "fit_eval_prereqs": "A TrueFoundry account (created in step 1) with a verified email, and a model-provider account plus API key to add models. Credit card, cloud account and cluster are not mentioned ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))",
      "fit_eval_shape": "cloud_console",
      "fit_eval_snippet": "partial",
      "fit_eval_snippet_note": "`client = OpenAI(api_key=\"your_truefoundry_api_key\", base_url=\"{GATEWAY_BASE_URL}\")` — but no completed request call is shown on the page; snippets live in the Playground's Code Snippets tab ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "tfy-llm-gateway for the Gateway Plane, truefoundry for the full control plane plus gateway plane",
      "fit_kubernetes": "official_helm",
      "fit_kubernetes_note": "The AI Gateway can be deployed on Kubernetes using the `tfy-llm-gateway` Helm chart at `https://github.com/truefoundry/infra-charts/tree/main/charts/tfy-llm-gateway`. A separate full TrueFoundry control-plane and gateway-plane installation uses the `truefoundry` chart from `https://github.com/truefoundry/infra-charts/tree/main/charts/truefoundry`. ([TrueFoundry deployment overview](https://www.truefoundry.com/docs/platform/deployment-overview))",
      "fit_mcp": "mcp_gateway",
      "fit_mcp_note": "TrueFoundry documents an MCP Gateway that centralizes access to MCP servers, with dynamic tool discovery and invocation, a curated registry, and support for remote, virtual, OpenAPI, and stdio MCP servers over streamable HTTP. It also documents centralized authentication and combining tools from multiple MCP servers through Virtual MCP Servers. ([TrueFoundry MCP Gateway overview](https://www.truefoundry.com/docs/ai-gateway/mcp/mcp-overview))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers",
          "fit_gateway_platform",
          "fit_model_gate",
          "fit_vercel_ai_sdk"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented) on the quickstart, self-hosting or pricing pages ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))",
      "fit_onboarding_notes": "No install commands are given in the quickstart at all; auth uses a PAT or VAT from the Access section, and multiple provider accounts/keys can be added. Self-hosted PostgreSQL 17+ requires disabling SSL (`force_ssl=0` on AWS, `require_secure_transport=false` on Azure) ([Quick Start Guide: Setup & Integration - TrueFoundry Docs](https://docs.truefoundry.com/gateway/quick-start), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. TrueFoundry documents OpenAI-compatible drop-in usage by changing the client `base_url` and API key; the documented SaaS base URL is `https://gateway.truefoundry.ai`, while self-hosted examples use `{GATEWAY_BASE_URL}`. ([TrueFoundry AI Gateway quick start](https://www.truefoundry.com/docs/ai-gateway/quick-start))",
      "fit_prod_prereqs": "An organization registered on TrueFoundry; Kubernetes 1.27+ with nodes having at least 100 GB ephemeral disk; dynamic PVC provisioning; an ingress controller or Istio service mesh; a domain and TLS certificate; outbound HTTPS access to TrueFoundry endpoints; PostgreSQL >= 13 (managed recommended); S3/GCS/Azure Blob or other S3-compatible blob storage; and TrueFoundry-provided Tenant Name, License Key (`TFY_API_KEY`) and Image Pull Secret. Sizing: medium tier ≥3 AI Gateway replicas (~500 req/s), large tier ≥10 replicas (~2,000 req/s) ([Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "infra_rollout",
      "fit_python_frameworks": [
        "LangChain",
        "LangGraph",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "TrueFoundry documents ready-to-use LangChain and LlamaIndex snippets. The LangChain integration uses `pip install langchain-openai` and the `ChatOpenAI` class; LangGraph examples use `StateGraph`, `MessagesState`, and related workflow APIs. ([TrueFoundry LangChain integration](https://www.truefoundry.com/docs/ai-gateway/langchain))",
      "fit_quickstart_steps": 4,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.truefoundry.com/gateway/quick-start",
      "fit_self_host_docs": "documented_no_command",
      "fit_self_host_install": "No literal install command is published on the self-hosting pages; they document topologies and prerequisites only ([Self-hosting of Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/ai-gateway/gateway-self-hosted), [Deploy AI Gateway - TrueFoundry Docs](https://www.truefoundry.com/docs/platform/deploy-control-plane-and-gateway-plane))",
      "fit_terraform": "shipped_code",
      "fit_terraform_note": "TrueFoundry states that its software ships with OpenTofu/Terraform code, but no official Terraform provider, module, or Terraform Registry name is documented. ([TrueFoundry deployment overview](https://www.truefoundry.com/docs/platform/deployment-overview))",
      "fit_terraform_ref": "the software ships with OpenTofu/Terraform code; no provider, module or Registry name is documented",
      "fit_third_party_documented": {},
      "fit_vendor_time_claim": null,
      "fit_vercel_ai_sdk": "not_documented",
      "fit_vercel_ai_sdk_note": "n.a. (not documented)",
      "fit_vercel_ai_sdk_package": null
    },
    {
      "slug": "fireworks-ai",
      "name": "Fireworks AI",
      "company": "Fireworks AI",
      "tagline": "Inference provider serving open-weight models on its own stack, with fine-tuning and dedicated deployments.",
      "category": "inference_provider",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2022,
      "hq_country": "United States",
      "url_home": "https://fireworks.ai/",
      "url_docs": "https://docs.fireworks.ai/getting-started/introduction",
      "url_pricing": "https://fireworks.ai/pricing",
      "url_status": "https://status.fireworks.ai/",
      "url_github": "https://github.com/fw-ai/cookbook",
      "url_changelog": null,
      "url_models_api": "https://api.fireworks.ai/v1/accounts/{account_id}/models",
      "github_stars": null,
      "model_count": 100,
      "model_count_note": "Docs state \"100+ Supported Models\" across text, vision, audio, image and embeddings; the list-models API returns a totalSize with a maximum pageSize of 200.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": null,
      "free_tier": "$1 in free credits on signup, then postpaid billing.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Region-restricted deployments",
          "amount": "1.5x premium over standard deployment pricing"
        },
        {
          "label": "On-demand GPUs",
          "amount": "H100/H200 $8.00/hour, B200 $13.00, B300 $15.00, GB300 $20.00"
        },
        {
          "label": "Managed LoRA supervised fine-tuning",
          "amount": "$0.50-$10.00 per 1M tokens depending on model size"
        },
        {
          "label": "Embeddings",
          "amount": "$0.008-$0.10 per 1M input tokens"
        }
      ],
      "self_host_cost_note": "No self-host option; Fireworks sets its own per-token serverless prices rather than marking up another vendor's list price, with DEFAULT, PRIORITY and FAST service tiers resolved live per model. Dedicated capacity is billed per GPU-hour, and batch inference plus supervised and reinforcement fine-tuning are priced separately.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": true,
      "config_as_code": null,
      "portability_note": "Export covers rated usage and cost via firectl, not request logs.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Batch inference API",
        "Fine-tuning API (SFT and RFT)",
        "Dedicated deployments"
      ],
      "sdk_note": "Documented as a drop-in replacement using the same API, so OpenAI SDKs work by changing the base URL; the service_tier parameter is Fireworks-specific and does not port to other providers.",
      "cap_failover": null,
      "cap_load_balancing": null,
      "cap_conditional_routing": null,
      "cap_cache_simple": null,
      "cap_cache_semantic": null,
      "cap_observability": null,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": null,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": "Not a proxy, so there is no gateway overhead figure. Third-party measurements: a Telnyx-run streamed GLM-5.2 benchmark (240 successful requests, 10 runs per provider, June 25 2026) recorded a 1.28 s median time-to-first-token for Fireworks versus 0.79 s for Together, while Artificial Analysis' live GLM-5.1 and Kimi K3 trackers placed Fireworks near the top for output speed among ten tracked providers; a separate Python harness on Llama 3.1 8B reported about 320 tokens/sec median for Fireworks versus 580 for Groq and 250 for Together.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": "Region-restricted deployments are offered at a 1.5x price premium, but no specific EU region list or residency commitment was found in vendor documentation.",
      "zdr": null,
      "zdr_note": "Zero data retention is the default posture, not an upgrade.",
      "air_gapped": null,
      "best_for": "Teams wanting fast serving of open-weight models with tiered latency options and both supervised and reinforcement fine-tuning.",
      "strengths": [
        "100+ open-weight models with a drop-in OpenAI-compatible API",
        "Selectable DEFAULT, PRIORITY and FAST service tiers plus batch inference for cost control",
        "Both supervised and reinforcement fine-tuning, with managed LoRA priced from $0.50 per 1M tokens",
        "States SOC 2 Type II and HIPAA compliance, with a public trust center and documented CMEK encryption states",
        "Region-restricted deployments available when workloads must stay in a specific geography"
      ],
      "weaknesses": [
        "Single-source provider: no closed frontier models, no independent catalog, and no cross-provider routing or failover",
        "Only a one-time $1 evaluation credit, which is too small for realistic load testing",
        "GDPR, EU residency specifics and zero-data-retention are not documented on vendor pages found",
        "Serverless is multi-tenant and priority tiers cost more, so predictable latency effectively requires paying up or going dedicated"
      ],
      "avoid_if": "You need frontier closed models, multi-provider failover, or documented EU data residency and zero retention.",
      "reviews": [
        {
          "url": "https://workos.com/blog/fireworks-ai-the-pytorch-teams-bet-on-inference-as-the-new-runtime",
          "why": "Zack Proser's deep dive into Fireworks' FireAttention stack, serverless vs on-demand deployment model and where its inference economics come from.",
          "date": "2026-01-08",
          "type": "written",
          "title": "Fireworks.ai: The PyTorch Team's Bet on Inference as the New Runtime",
          "publisher": "WorkOS"
        },
        {
          "url": "https://aishwaryasrinivasan.substack.com/p/running-llama-4-maverick-full-1-m",
          "why": "Hands-on account of pushing Llama 4 Maverick's full 1M-token context through the Fireworks API, with the scripts and practical limits described.",
          "date": "2025-04-29",
          "type": "written",
          "title": "Running LLaMA 4 Maverick Full 1M-Token Context on Fireworks",
          "publisher": "AI with Aish (Substack)"
        },
        {
          "url": "https://www.youtube.com/watch?v=1w8iEY3z4DE",
          "why": "theCUBE's hosts press Fireworks' CEO on inference economics and platform strategy in a third-party-produced conference interview.",
          "date": "2026-07-09",
          "type": "video",
          "title": "Lin Qiao, Fireworks AI | RAISE Summit 2026",
          "publisher": "YouTube - SiliconANGLE theCUBE"
        },
        {
          "url": "https://www.youtube.com/watch?v=i__vPJ9bW6I",
          "why": "Research-podcast hosts interview a Fireworks co-founder on GPU sourcing, kernels, routing and cross-region training that underpin the service.",
          "date": "2026-06-24",
          "type": "video",
          "title": "Infrastructure for AI at Scale - With Benny Chen (Fireworks AI)",
          "publisher": "YouTube - The Information Bottleneck"
        },
        {
          "url": "https://www.reddit.com/r/speechtech/comments/1pkq9pv/fireworksai_ast_critical_issues_stay_away_until/",
          "why": "Founder documents a 35-day Fireworks speech-to-text defect with dates, connection-count symptoms, cost impact and the workaround used, plus corroborating replies.",
          "date": "2025-12-12",
          "type": "community",
          "title": "Fireworks.ai AST critical issues (stay away until they fix them)",
          "publisher": "Reddit - r/speechtech"
        },
        {
          "url": "https://www.reddit.com/r/LocalLLaMA/comments/1jti1j4/llama_4_behaving_differently_on_groq_vs_fireworks/",
          "why": "Head-to-head thread on identical-model output differences between the two providers, with notes on Fireworks' fine-tune hosting and tier/rate-limit behaviour.",
          "date": "2025-04-07",
          "type": "community",
          "title": "LLaMa 4 behaving differently on Groq vs Fireworks AI",
          "publisher": "Reddit - r/LocalLLaMA"
        }
      ],
      "sources": {
        "name": "https://fireworks.ai/pricing",
        "soc2": "https://docs.fireworks.ai/faq/enterprise/compliance/certifications",
        "company": "https://en.wikipedia.org/wiki/Fireworks_AI",
        "tagline": "https://docs.fireworks.ai/getting-started/introduction",
        "sdk_note": "https://docs.fireworks.ai/getting-started/introduction",
        "url_docs": "https://docs.fireworks.ai/getting-started/introduction",
        "url_home": "https://fireworks.ai/pricing",
        "cap_audio": "https://docs.fireworks.ai/getting-started/introduction",
        "cap_batch": "https://docs.fireworks.ai/guides/batch-inference",
        "free_tier": "https://fireworks.ai/pricing",
        "hipaa_baa": "https://docs.fireworks.ai/faq/enterprise/compliance/certifications",
        "extra_fees": "https://fireworks.ai/pricing",
        "hq_country": "https://en.wikipedia.org/wiki/Fireworks_AI",
        "url_github": "https://docs.fireworks.ai/getting-started/introduction",
        "url_status": "https://docs.fireworks.ai/getting-started/introduction",
        "zdr_status": "https://docs.fireworks.ai/guides/security_compliance/data_handling",
        "model_count": "https://docs.fireworks.ai/getting-started/introduction",
        "url_pricing": "https://fireworks.ai/pricing",
        "api_surfaces": "https://docs.fireworks.ai/guides/batch-inference",
        "founded_year": "https://en.wikipedia.org/wiki/Fireworks_AI",
        "product_fate": "https://docs.fireworks.ai/faq/deployment/serverless/service-levels",
        "regions_note": "https://docs.fireworks.ai/deployments/regions",
        "cap_image_gen": "https://docs.fireworks.ai/getting-started/introduction",
        "vendor_status": "https://www.reuters.com/technology/nvidia-backed-startup-fireworks-valued-175-billion-latest-funding-2026-07-16/",
        "cap_embeddings": "https://fireworks.ai/pricing",
        "certifications": "https://docs.fireworks.ai/guides/security_compliance/data_security",
        "retention_days": "https://docs.fireworks.ai/guides/security_compliance/data_handling",
        "url_models_api": "https://docs.fireworks.ai/api-reference/list-models",
        "logs_by_default": "https://docs.fireworks.ai/guides/security_compliance/data_handling",
        "model_count_note": "https://docs.fireworks.ai/api-reference/list-models",
        "eu_residency_note": "https://fireworks.ai/pricing",
        "fireworks_pricing": "https://fireworks.ai/pricing",
        "last_release_date": "https://docs.fireworks.ai/updates/changelog",
        "openai_compatible": "https://docs.fireworks.ai/tools-sdks/openai-compatibility",
        "guardrails_location": "https://docs.fireworks.ai/faq/security/infrastructure/model-security",
        "latency_source_note": "https://markaicode.com/vs/together-ai-vs-fireworks-ai/",
        "requires_vendor_sdk": "https://docs.fireworks.ai/tools-sdks/openai-compatibility",
        "self_host_cost_note": "https://docs.fireworks.ai/api-reference/list-models",
        "data_export_available": "https://docs.fireworks.ai/accounts/exporting-usage-costs",
        "prompt_transits_vendor": "https://docs.fireworks.ai/deployments/regions",
        "trains_on_customer_data": "https://docs.fireworks.ai/guides/security_compliance/data_handling",
        "fireworks_cost_structure_faq": "https://docs.fireworks.ai/faq/billing-pricing-usage/pricing/cost-structure",
        "fireworks_caching_billing_faq": "https://docs.fireworks.ai/faq-new/billing-pricing/is-prompt-caching-billed-differently"
      },
      "verified_at": {
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_batch": "2026-08-29",
        "free_tier": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_github": "2026-08-29",
        "url_status": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "founded_year": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "url_models_api": "2026-08-29",
        "model_count_note": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29",
        "data_export_available": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:39.701913+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed with near-daily changelog activity. Raised a $1.51B Series D at a $17.5B valuation in July 2026. Note that the serverless tier carries no latency or availability guarantee at all.",
      "last_release_date": "2026-08-27",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Inference runs on Fireworks' own global hardware fleet.",
      "logs_by_default": "none",
      "logs_note": "No prompt or generation data is logged or stored for any open model without explicit opt-in — it exists only in volatile memory for the life of the request. Metadata such as token counts is logged as needed to run the service.",
      "retention_days": 0,
      "retention_note": "Zero by default. The Responses API is the exception: storage defaults to on, held 30 days, and you must pass store=False to opt out.",
      "zdr_status": "yes",
      "trains_on_customer_data": "opt_in",
      "training_note": "Nothing is stored without explicit opt-in, and opting in enables features such as FireOptimizer. Worth noting what is absent: no blanket “we never train on your data” statement was found.",
      "subprocessors_url": null,
      "regions_note": "Region groupings for global (the default), US, Europe and Asia-Pacific, plus around 19 pinnable single regions including Iowa, Frankfurt, Iceland and Tokyo.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "Type II",
          "status": "certified"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "controls mapped to GDPR and CCPA; no executable DPA page retrieved",
          "status": "claimed"
        },
        "iso27001": {
          "detail": "achieved, alongside 27701 and 42001",
          "status": "certified"
        },
        "hipaa_baa": {
          "detail": "described as HIPAA-certified and compliant; no BAA document retrieved",
          "status": "claimed"
        }
      },
      "guardrails_location": "none",
      "guardrails_note": "No guardrails are applied to models by default; Fireworks expects customers to implement their own.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference (Fireworks is the provider). Per-token serverless, per-GPU-second on-demand, per-token/GPU-hour training, enterprise quote. No separate gateway or platform fee — routing layer is $0.",
      "min_commitment": "None stated for inference. Fine-tuning has no stated minimum on the pricing page.",
      "overage_note": "None stated. Serverless is postpaid, so cost scales with tokens. Note: 'Checkpoint storage for serverless models is included during private preview' implies a future charge.",
      "cache_support": "prefix",
      "cache_read_discount_pct": 50,
      "cache_write_premium_pct": 0,
      "cache_pricing_note": "Prompt caching with discounted cached tokens (prefix-style caching for serverless; mechanism not labelled exact-match or semantic). Default 50% discount, but exact discount varies by model. Pricing page separately lists 'Cached Prefill / 1M' rates for training that are 80% below prefill (e.g. Qwen 3.5 9B $0.132 vs $0.66; Kimi K3 $2.17 vs $10.87). No cache-write charge documented — close to a free win, but only serverless models are covered.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "'High rate limits' on serverless; higher on Enterprise. Platform-set, not a configurable spend control.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated.",
      "cost_export": "Not stated.",
      "product_kind_note": "\"Fireworks AI is the fastest platform for building with open source AI models\" ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)), exposing a \"Fireworks AI REST API\" for \"various language, image and embedding models using an API Key\" that \"also lets you automate management of models, deployments, datasets, and more\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). It is a first-party inference and training platform, not a router.",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "Serverless models are \"pre-deployed by the Fireworks team for the community to use\", shared, with \"no SLA guarantees for up-time or latency\", best-effort availability and at least two weeks' deprecation notice; \"neither custom base models nor LoRA addons are supported for serverless inference\" and \"all user-provided models, including trained models, require a dedicated deployment\" ([Models overview](https://docs.fireworks.ai/models/overview)). Dedicated deployments are \"private to you\", give access to \"a wide array of hardware\", support base models and LoRA addons, and are billed by GPU-second; deployments are addressed as `accounts/<ACCOUNT_ID>/deployments/<DEPLOYMENT_ID>` or `accounts/fireworks/models/gpt-oss-120b#<DEPLOYMENT_NAME>` ([Models overview](https://docs.fireworks.ai/models/overview); [On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). Region-restricted deployments carry a 1.5x premium ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap with the OpenAI SDK (`https://api.fireworks.ai/inference/v1`) or curl; a first-party Fireworks Python SDK also exists but is not required ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). All requests need `Authorization: Bearer <API_KEY>` and `Content-Type: application/json` ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). Operational caveat: a deployment scaled to zero returns \"a `503` error immediately\" while it scales up, so \"applications should implement retry logic\" ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)). Models must be deployed before use — \"a model must be deployed before it can be used for inference\" ([Models overview](https://docs.fireworks.ai/models/overview)).",
      "model_count_low": 100,
      "model_count_high": 100,
      "model_count_source_note": "\"100+ Supported Models\" spanning \"text, vision, audio, image, and embeddings\" ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction), undated). The pricing page names specific models (GLM 5.3, Qwen3 8B, Qwen3-235B, gpt-oss-120B, DeepSeek V3, Kimi K2, Qwen 3.5 9B, Qwen 3.6 27B, Kimi K3) without a total ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party only: Fireworks serves models on its own platform (\"the fastest platform for building with open source AI models\") and does not route to third-party provider APIs, so no provider count exists ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction); [Models overview](https://docs.fireworks.ai/models/overview)).",
      "first_party_vs_partner": "Vendor-operated hardware serving open-weight and customer models: deployments provide \"dedicated GPUs for your models\" and preemptible deployments borrow \"idle reserved GPUs\"; Fireworks maintains \"a library of common base models\" and customers can add their own ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments); [Models overview](https://docs.fireworks.ai/models/overview)). Whether that hardware is Fireworks-owned or leased from clouds is not stated: n.a. ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "custom_endpoint_support": "Bring-your-own-weights rather than bring-your-own-endpoint: \"upload your own models (for supported architectures) from Hugging Face or elsewhere\", plus LoRA addon upload/training, all requiring a dedicated deployment ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments); [Models overview](https://docs.fireworks.ai/models/overview)). Registering an external self-hosted endpoint (vLLM, Ollama, SageMaker, private URL): n.a.",
      "byok_mode": "platform_credits_only",
      "merchant_of_record": "Fireworks: it bills serverless usage per token (postpaid), dedicated/on-demand per GPU-second with \"no extra charges for start-up times\", fine-tuning per 1M training tokens and reinforcement fine-tuning per GPU hour billed per second, with preemptible capacity carrying \"no charge to hold dedicated capacity\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "key_management_note": "Single-vendor key model: API keys are created with `firectl api-key create` or in the dashboard at `https://app.fireworks.ai/settings/users/api-keys`, and every request carries `Authorization: Bearer <API_KEY>`; account-scoped quota APIs (List/Get/Update Quota) exist ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction)). No third-party provider secrets are stored, and inference necessarily runs on Fireworks infrastructure, so Fireworks processes request content; the models overview asserts \"your data is yours\" ([Models overview](https://docs.fireworks.ai/models/overview)). Encryption/KMS specifics: n.a.",
      "api_openai_chat_completions_note": "Yes: `POST https://api.fireworks.ai/inference/v1/chat/completions`, configured as an OpenAI SDK `base_url`/`baseURL` of `https://api.fireworks.ai/inference/v1` ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "api_anthropic_messages_note": "n.a. (not documented on the pages fetched).",
      "api_openai_responses_note": "n.a.",
      "api_embeddings_note": "Yes: embedding models are part of the REST API and are priced per 1M input tokens by base-model size (up to 150M: $0.008; 150M–350M: $0.016; Qwen3 8B: $0.10), with \"embeddings & reranking\" positioned for \"search & context retrieval\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction); [Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)).",
      "api_images_note": "Yes as a supported model family: the REST API covers \"language, image and embedding models\" and the catalog spans \"text, vision, audio, image, and embeddings\" ([Fireworks API reference introduction](https://docs.fireworks.ai/api-reference/introduction); [Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)). Endpoint path: n.a.",
      "api_audio_note": "Yes as a supported modality (\"text, vision, audio, image, and embeddings\"), but no STT/TTS endpoint path is documented on the pages fetched ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)).",
      "api_batch_note": "Yes: \"Batch Inference — run async inference jobs at scale, faster and cheaper\" ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction)). Endpoint path: n.a.",
      "deployment_modes_note": "hosted cloud service in two shapes: serverless inference (\"use popular models instantly with pay-per-token pricing\", pre-deployed by Fireworks and shared across users) and dedicated GPU deployments (\"deploy with high performance on dedicated GPUs with fast autoscaling and minimal cold starts\"), plus training ([Fireworks introduction](https://docs.fireworks.ai/getting-started/introduction); [Models overview](https://docs.fireworks.ai/models/overview)). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. — the models overview page explicitly does not mention any of them ([Models overview](https://docs.fireworks.ai/models/overview)). Enterprise deployments are quote-based: \"contact us for enterprise deployments with faster speeds, lower costs, and higher rate limits\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22).",
      "byok_mode_note": "platform_credits_only in the sense that all usage is billed by Fireworks on its own API key: serverless is \"pay per token\" with postpaid billing and \"$1 in free credits\", on-demand is \"pay per GPU second\" ([Fireworks pricing](https://fireworks.ai/pricing), 2026-05-22; [Models overview](https://docs.fireworks.ai/models/overview)). There are no upstream provider keys to bring; requests authenticate with `FIREWORKS_API_KEY` ([On-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "access_meta": {
        "qualified": [
          "api_images",
          "api_audio"
        ],
        "model_count_kind": "vendor_total",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` for inference requests. The only documented timeout is control-plane: `--wait-timeout` on `firectl deployment create` with `--wait`, **default 1 hour** ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented` as a platform feature; it is pushed to the client: when a deployment is scaled to zero, requests return `503` immediately and \"Your application should implement retry logic to handle this\". Default count and backoff: `n.a.` ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "No model or provider fallback list ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "`not_documented` as configurable. Deployment health is observable via `state` (`CREATING`, `READY`, `UPDATING`, `DELETING`, `DELETED`, `FAILED`) and replica counts (`min_replica_count`, `ready_replica_count`, `desired_replica_count`, `max_replica_count`), with guidance to \"Verify it's healthy\" before cutting traffic over — this is deployment status, not upstream ejection or circuit breaking ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "rel_multi_region_config": "config_file",
      "rel_multi_region_note": "`config_file` at creation time, and it is genuinely a capacity-failover control: `--region` accepts `GLOBAL`, `US`, `EUROPE`, `APAC`; `GLOBAL` is recommended for production workloads needing \"geographic availability or capacity failover\". Without `--region` a deployment is \"pinned to a single datacenter at creation time and will not be automatically migrated later\", and placement \"cannot be changed in place\" — you create a replacement deployment ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "rel_load_balancing_config": "not_documented",
      "rel_load_balancing_note": "Deployment shapes can be optimized for speed, throughput or cost, but no load-balancing policy or weights ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "`vendor_saas` — an inference provider, not a gateway; the serving runtime is described by engine name (FireAttention V4) rather than a disclosed process runtime, and the control surface is the `firectl` CLI plus `https://api.fireworks.ai/inference/v1` ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200), [Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "Fireworks distributes `firectl` binaries (Homebrew, macOS arm64/x86_64, Linux x86_64, Windows 64-bit) and an open-source benchmarking tool, but not a self-hostable serving plane ([firectl CLI](https://docs.fireworks.ai/tools-sdks/firectl/firectl), [Fireworks performance benchmarking](https://docs.fireworks.ai/deployments/benchmarking)).",
      "latency_claims": [],
      "throughput_claims": [
        {
          "url": "https://fireworks.ai/blog/fireattention-v4",
          "unit": "tokens/s",
          "scope": "generation_speed",
          "value": ">250",
          "metric": "output speed",
          "publisher": "vendor",
          "conditions": "DeepSeek V3 0324, FireAttention V4 at FP4, NVIDIA B200, 8 GPUs with NVLink, speculation disabled."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Streaming is available through the OpenAI-compatible API surface; the fetched pages state no streaming-specific caveat, but they do state hard availability caveats: \"A preemptible deployment can be preempted mid-request and disappear with no warning\", \"Never point production or latency-sensitive traffic at a preemptible deployment\", and scaled-to-zero deployments return `503` immediately ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "No PII guardrail feature; the data-handling posture is zero retention rather than detection ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "guard_injection": "not_documented",
      "guard_injection_note": "`not_documented` as a platform feature ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "guard_content": "not_documented",
      "guard_content_note": "`not_documented` — no inline moderation filter is documented on the fetched pages; the data-handling and account-policy docs describe retention and model access only ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling), [Model access policy](https://docs.fireworks.ai/accounts/model-access-policy))",
      "guard_custom": "not_documented",
      "guard_custom_note": "`not_documented` ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "guard_model_policy": "yes",
      "guard_model_policy_note": "Enterprise account admins set a Model access policy controlling which models can be used for inference, deployments, and training; each model is allowed or denied per surface (serverless, serverless fast, dedicated, training), allowlist or denylist style, keyed on resource names like `accounts/fireworks/models/qwen3-235b-a22b`. It is account-wide: \"There is no per-user or per-group model access control today\", and updating requires the Admin role ([Model access policy](https://docs.fireworks.ai/accounts/model-access-policy))",
      "guard_execution_location": "not_documented",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "log_content": "metadata_only",
      "log_content_note": "Zero data retention applies by default and prompts/generations are not logged; the exception is the Responses API, where `store=True` is the default and stored responses are deleted after 30 days ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`store=False` on Responses API calls prevents storage; ZDR is the default elsewhere ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "tracing_support": "proprietary",
      "tracing_note": "Via integrations rather than natively: Weights & Biases Weave and MLflow tracing are documented ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))",
      "export_destinations": [],
      "export_destinations_note": "A Prometheus-compatible metrics endpoint at `https://api.fireworks.ai/v1/accounts/<account_id>/metrics`, scrapeable by Prometheus, the OpenTelemetry Collector, the Datadog Agent, Vector, Grafana, and New Relic ([Exporting metrics](https://docs.fireworks.ai/deployments/exporting-metrics)); W&B and MLflow for traces ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` — evaluation happens in W&B/MLflow, not against Fireworks-stored traffic (there is none by default) ([MLOps and observability integrations](https://docs.fireworks.ai/ecosystem/integrations/mlops-observability), [Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "observability_requires_saas": null,
      "log_retention_note": "No prompt/generation retention by default; Responses API stored objects are deleted after 30 days ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "**\">250 tokens/second\"** on DeepSeek V3 0324 with FireAttention V4 at FP4 (NVFP4) on NVIDIA B200, 8 GPUs with NVLink, speculation (MTP) disabled. This is a generation-speed figure, not gateway overhead; the post also names TTFT and end-to-end response time as metrics without giving numbers, and compares against H200 FP8 on SGLang and B200 FP4 on TRT-LLM. Related engine claims: NVFP4 has \"2x FLOPs throughput\" and needs \"~1.5x-2x less memory reads\" than MXFP6/MXFP8, and Hopper→Blackwell FP8 end-to-end improves \"~1.5-1.7x\" ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200)).",
        "multi_surface": {},
        "throughput_prose": "Same \">250 tokens/second\" figure and hardware. Accelerator options are `NVIDIA_A100_80GB`, `NVIDIA_H100_80GB`, `NVIDIA_H200_141GB`, with the caveat that \"scaling is sub-linear (2x GPUs ≠ 2x performance)\". Fireworks publishes a self-service harness (`python benchmark.py --model ... --num-requests 1000 --concurrency 10`) rather than headline RPS numbers ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200), [Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments), [Fireworks performance benchmarking](https://docs.fireworks.ai/deployments/benchmarking)).",
        "benchmark_independence": "Vendor self-published, and **explicitly comparative against rival stacks** (SGLang on H200 FP8, TRT-LLM on B200 FP4). Fireworks itself flags the comparison as incomplete: \"We were not able to fully validate TRT-LLM FP4 performance on B200 due to setup complexity\", plus \"Speculation success being very prompt-dependent makes it hard to compare across implementations\" ([Fireworks FireAttention V4](https://fireworks.ai/blog/fireattention-v4-fp4-b200)). Fireworks also appears as a compared provider in Groq's cited ArtificialAnalysis.ai benchmark — a third-party figure published by a competitor ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "not_documented",
      "guard_custom_default": "not_documented",
      "rel_notes": "Reliability here is deployment topology, not routing config: choose `--region GLOBAL` at creation, avoid preemptible for production, keep `min_replica_count` above zero to avoid cold `503`s, and implement your own retries. Region placement being immutable is the biggest planning constraint ([Fireworks on-demand deployments](https://docs.fireworks.ai/guides/ondemand-deployments)).",
      "guard_notes": "Because nothing is retained, there is nothing to run guardrails or evals against after the fact — the trade-off is maximal privacy and minimal auditability ([Data handling](https://docs.fireworks.ai/guides/security_compliance/data_handling)). The user-management page documents no logging, tracing, or guardrail controls at all ([Managing users](https://docs.fireworks.ai/accounts/users)).",
      "fit_agent_features_note": "The Fireworks Responses API supports tool calls, `max_tool_calls`, parallel tool calls, `tool_choice`, streaming, and conversational continuation with `previous_response_id`. Fireworks also describes server-side handling of the agentic loop—reasoning, tool selection, and execution—and server-side state for long conversations. ([Fireworks Responses API reference](https://docs.fireworks.ai/api-reference/post-responses))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. A Fireworks API key is used ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))",
      "fit_client_sdk_langs": [
        "Python",
        "TypeScript"
      ],
      "fit_client_sdks_note": "Documented clients include the Fireworks Python SDK, package `fireworks-ai` with class `Fireworks`; the OpenAI Python package `openai` with class `OpenAI`; the Anthropic Python package `anthropic` with class `anthropic.Anthropic`; the OpenAI JavaScript/TypeScript package `openai` with class `OpenAI`; and the Anthropic JavaScript/TypeScript package `@anthropic-ai/sdk` with class `Anthropic`. Fireworks also documents the `@ai-sdk/fireworks` JavaScript/TypeScript provider and `curl`. ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))",
      "fit_cloud_iam": "native_iam",
      "fit_cloud_iam_note": "For BYOC on AWS, Fireworks documents AWS IAM roles for the cluster and support for `AWS_PROFILE` when `firectl` uploads model files to S3. BYOC API examples use `api_key=\"unused\"` because the API key is not used, while airgapped deployment stores a metering key in AWS Secrets Manager. ([Fireworks EKS BYOC documentation](https://docs.fireworks.ai/ecosystem/integrations/eks-bring-your-own-cloud))",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_required",
      "fit_credit_card_note": "Not stated as required: \"Get started with $1 in free credits.\" ([Fireworks - Pricing](https://fireworks.ai/pricing))",
      "fit_ecosystem_notes": "Fireworks documents hosted serverless API endpoints and BYOC deployments, including private-preview deployments in customer-managed cloud, managed Kubernetes, GPU-cloud, or on-premises environments. Airgapped EKS deployments keep images, models, and dependencies in the organization’s infrastructure and cannot be managed through the Fireworks web app or `firectl`; the Fireworks Python SDK is documented as alpha and requires `--pre` for the latest version. ([Fireworks BYOC setup](https://docs.fireworks.ai/ecosystem/integrations/byoc/how-setup-works))",
      "fit_eval_prereqs": "A Fireworks API key created in the dashboard and exported as `FIREWORKS_API_KEY`. Account, credit card, cloud account, cluster and provider key are not mentioned as prerequisites ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))",
      "fit_eval_shape": "sdk_install",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `client.chat.completions.create(model=\"accounts/fireworks/models/deepseek-v3p1\", messages=[{\"role\":\"user\",\"content\":\"Say hello in Spanish\"}])`, with OpenAI, Anthropic, JavaScript and curl variants; example response `\"¡Hola!\"` ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))",
      "fit_gateway_platform": "third_party_documented",
      "fit_gateway_platform_note": "Not documented by Fireworks, but API7's APISIX-based AI gateway documents a `fireworks-ai` catalog provider that connects through the `openai` adapter with `api_base` = `https://api.fireworks.ai/inference/v1`. ([API7 AI Gateway - Fireworks AI provider](https://docs.api7.ai/ai-gateway/providers/fireworks-ai))",
      "fit_helm_chart": "Helm charts and templates for EKS BYOC clusters; no chart name or repo stated",
      "fit_kubernetes": "helm_unnamed",
      "fit_kubernetes_note": "Fireworks documents Kubernetes and Amazon EKS deployments, including deployment with Helm charts or Helm deployment templates. No official chart name, repository, or operator name is stated. ([Fireworks airgapped EKS deployment](https://docs.fireworks.ai/ecosystem/integrations/eks/airgapped))",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Fireworks supports MCP tools in its Responses API; the `tools` field accepts MCP tools and response tools support the `mcp` type. Fireworks also documents MCP server usage with an example SSE server URL such as `https://gitmcp.io/docs`. ([Fireworks Responses API reference](https://docs.fireworks.ai/api-reference/post-responses))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloudflare_workers"
        ]
      },
      "fit_model_gate": "none",
      "fit_model_gate_note": "n.a. (not documented). No approval, enablement, quota or region gate on either page ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart), [Fireworks - Pricing](https://fireworks.ai/pricing))",
      "fit_onboarding_notes": "The Python SDK is in alpha, so installs need `--pre` (`pip install --pre fireworks-ai`, `poetry add --pre fireworks-ai`, `uv add --pre fireworks-ai`); OpenAI/Anthropic-compatible paths avoid the alpha SDK (`pip install openai`, `pip install anthropic`, `npm install openai`, `npm install @anthropic-ai/sdk`). Priority tier is selected with `service_tier: \"priority\"` ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. For hosted inference, Fireworks documents the OpenAI SDK with `base_url=\"https://api.fireworks.ai/inference/v1\"`; BYOC examples use `base_url=\"http://<YOUR-LOAD-BALANCER-HOSTNAME>/v1\"` and `api_key=\"unused\"`. ([Fireworks OpenAI compatibility docs](https://docs.fireworks.ai/tools-sdks/openai-compatibility))",
      "fit_prod_prereqs": "n.a. (not documented). No database, Redis, cluster, load balancer, control plane or licence requirement is stated ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart), [Fireworks - Pricing](https://fireworks.ai/pricing))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "sdk_install",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex",
        "CrewAI",
        "PydanticAI",
        "Strands"
      ],
      "fit_python_frameworks_note": "Fireworks documents integrations with LangChain and LlamaIndex, and also lists CrewAI, PydanticAI, Strands Agents, and AWS AgentCore as supported agent frameworks. The LangChain JavaScript integration uses package `@langchain/fireworks` and class `Fireworks`; package and class names for the other documented integrations are not stated. ([Fireworks agent-framework integrations](https://docs.fireworks.ai/ecosystem/integrations/agent-frameworks))",
      "fit_quickstart_steps": 2,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.fireworks.ai/getting-started/quickstart",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "official_modules",
      "fit_terraform_note": "Fireworks documents the official Terraform module `fw-ai-external/aws-cluster/fireworksai` at `registry.terraform.io/modules/fw-ai-external/aws-cluster/fireworksai/latest`, with example version `0.1.2`. ([Fireworks EKS BYOC documentation](https://docs.fireworks.ai/ecosystem/integrations/eks-bring-your-own-cloud))",
      "fit_terraform_ref": "fw-ai-external/aws-cluster/fireworksai on the Registry, example version 0.1.2",
      "fit_third_party_documented": {
        "gateway_platform": "API7"
      },
      "fit_vendor_time_claim": "Vendor claims, verbatim: \"Make your first Serverless API call in minutes\" and \"help you make your first API call in minutes\" ([Fireworks quickstart](https://docs.fireworks.ai/getting-started/quickstart)); pricing page adds \"Get started in seconds with per token pricing, zero setup and no cold starts\" ([Fireworks - Pricing](https://fireworks.ai/pricing))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Fireworks has an official Vercel AI SDK provider package, `@ai-sdk/fireworks`, with provider instances `fireworks` and `createFireworks`. ([Vercel AI SDK Fireworks provider](https://ai-sdk.dev/providers/ai-sdk-providers/fireworks))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/fireworks"
    },
    {
      "slug": "groq",
      "name": "Groq",
      "company": "Groq",
      "tagline": "Inference provider running open-weight models on its own LPU hardware for very high output speed.",
      "category": "inference_provider",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2016,
      "hq_country": "United States",
      "url_home": "https://groq.com/",
      "url_docs": "https://console.groq.com/docs/models",
      "url_pricing": "https://groq.com/pricing",
      "url_status": "https://groqstatus.com/",
      "url_github": null,
      "url_changelog": null,
      "url_models_api": "https://api.groq.com/openai/v1/models",
      "github_stars": null,
      "model_count": null,
      "model_count_note": "Groq publishes a model catalog in its docs and a live /openai/v1/models endpoint, but no total model count was found on vendor pages.",
      "provider_count": null,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": null,
      "free_tier": "Free tier with per-model rate limits (for example gpt-oss-120b at 30 requests/min, 1,000 requests/day, 8,000 tokens/min); the paid Developer plan raises this to about 1,000 RPM and 250,000 TPM and adds Batch and Flex processing.",
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Speech-to-text (whisper-large-v3)",
          "amount": "$0.111 per hour of audio ($0.04/hour for v3-turbo)"
        },
        {
          "label": "Text-to-speech (Orpheus)",
          "amount": "$22-$40 per 1M characters"
        }
      ],
      "self_host_cost_note": "No self-host option; Groq sets its own per-token prices on its own LPU/LPX hardware rather than marking up another vendor's list price. Representative rates: gpt-oss-120b $0.15 input / $0.60 output per 1M tokens, gpt-oss-20b $0.075/$0.30, Qwen3.6-27B $0.60/$3.00.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": null,
      "config_as_code": false,
      "portability_note": "Invoice downloads only; no documented usage or log export.",
      "api_surfaces": [
        "OpenAI Chat Completions (https://api.groq.com/openai/v1)",
        "OpenAI Responses API",
        "Batch API (Developer plan)",
        "Speech-to-text and text-to-speech"
      ],
      "sdk_note": "Described as mostly compatible with OpenAI client libraries: base URL https://api.groq.com/openai/v1, with image input and function calling supported, but logprobs, logit_bias, top_logprobs, N>1 and vtt/srt outputs unsupported.",
      "cap_failover": null,
      "cap_load_balancing": null,
      "cap_conditional_routing": null,
      "cap_cache_simple": null,
      "cap_cache_semantic": null,
      "cap_observability": null,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": null,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": true,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": null,
      "cap_image_gen": null,
      "cap_audio": true,
      "cap_video": null,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": "Groq is a direct provider, not a proxy, so the figure below is output speed rather than proxy overhead. Groq's own model docs publish per-model output speeds in tokens/sec: 1,000 t/s for GPT-OSS-20B, 500 t/s for GPT-OSS-120B and Qwen3.6-27B, 560 t/s for Llama 3.1 8B, 450 t/s for groq/compound, 280 t/s for Llama 3.3 70B. Independently, Artificial Analysis measured 3,400 tokens/sec on Gemma 4 31B at 100K-token context on Nvidia LPX racks built on Groq 3 LPUs, versus 882 t/s for Cerebras; The Register notes that test is a best case for the architecture. A third-party Python harness on Llama 3.1 8B measured about 580 tokens/sec median for Groq with sub-100 ms time-to-first-token.",
      "throughput_rps": 1000,
      "soc2": true,
      "hipaa_baa": null,
      "gdpr": null,
      "eu_residency": false,
      "eu_residency_note": "Groq states that all retained data is held in Google Cloud buckets in the United States, with Standard Contractual Clauses used for transfers; no EU residency option is documented.",
      "zdr": true,
      "zdr_note": "The most straightforward ZDR here: any customer can enable it in Data Controls, globally or per feature, with no request process.",
      "air_gapped": null,
      "best_for": "Latency-sensitive, user-facing applications on open-weight models where output speed matters more than catalog breadth.",
      "strengths": [
        "Highest published output speeds of the group, up to 1,000 tokens/sec on GPT-OSS-20B on its own LPU hardware",
        "Zero Data Retention is self-serve for every customer, globally or per feature, with no retention of inference data by default",
        "Very low per-token prices, for example gpt-oss-120b at $0.15 input / $0.60 output per 1M tokens",
        "Mostly OpenAI-compatible API plus speech-to-text and text-to-speech endpoints",
        "SOC 2 Type II maintained with reports published for 2024 and 2025"
      ],
      "weaknesses": [
        "Smallest catalog here: open-weight models only, with no frontier closed models and no published model count",
        "Single-source hardware provider — no cross-provider routing, load balancing or failover, so an outage has no in-platform fallback",
        "All retained data sits in US Google Cloud buckets, with no EU residency option and no documented HIPAA BAA",
        "Free-tier rate limits are tight (about 30 requests/min on gpt-oss-120b) and some OpenAI parameters are unsupported"
      ],
      "avoid_if": "You need a broad multi-vendor model catalog, EU data residency, or a signed HIPAA BAA.",
      "reviews": [
        {
          "url": "https://www.tomshardware.com/tech-industry/semiconductors/nvidia-presents-groq-3-lpx-architecture-and-unveils-its-first-third-party-inference-benchmark",
          "why": "Luke James reports the first third-party benchmark numbers for the Groq 3 LPX rack and LP30 chip that power Groq's inference service, against Rubin and Cerebras.",
          "date": "2026-08-26",
          "type": "written",
          "title": "Hot Chips 2026: Nvidia presents Groq 3 LPX architecture and unveils its first third-party inference benchmark",
          "publisher": "Tom's Hardware"
        },
        {
          "url": "https://newsletter.semianalysis.com/p/groq-inference-tokenomics-speed-but",
          "why": "SemiAnalysis models the per-token economics behind GroqCloud's headline latency, comparing its LPU deployment cost against GPU-based serving of Mixtral.",
          "date": "2024-02-21",
          "type": "written",
          "title": "Groq Inference Tokenomics: Speed, But At What Cost?",
          "publisher": "SemiAnalysis"
        },
        {
          "url": "https://www.youtube.com/watch?v=S53BanCP14c",
          "why": "Hands-on Groq API tutorial on an established LLM channel: keys, Python/Colab usage, streaming, stop sequences and a Streamlit chatbot built on Mixtral and Llama 2 70B.",
          "date": "2024-02-29",
          "type": "video",
          "title": "Getting Started with Groq API | Making Near Real Time Chatting with LLMs Possible",
          "publisher": "YouTube - Prompt Engineering"
        },
        {
          "url": "https://www.youtube.com/watch?v=WB1WLlKmlcQ",
          "why": "theCUBE interviews Groq's CEO at SC23 with a live latency demo and pointed questions about LPU economics versus GPUs and tokens-as-a-service.",
          "date": "2023-11-14",
          "type": "video",
          "title": "Jonathan Ross, Groq | SC23",
          "publisher": "YouTube - SiliconANGLE theCUBE"
        },
        {
          "url": "https://www.reddit.com/r/LLMDevs/comments/1ufz1g0/groq_alternatives_for_production_apps_rate_limits/",
          "why": "7-comment thread on hitting Groq's rate limits and the closed developer plan when moving an app to production, with alternatives weighed.",
          "date": "2026-06-26",
          "type": "community",
          "title": "Groq alternatives for production apps? (Rate limits + Dev plan closed)",
          "publisher": "Reddit - r/LLMDevs"
        },
        {
          "url": "https://news.ycombinator.com/item?id=46379183",
          "why": "Thread where practitioners discuss what the deal means for GroqCloud continuity, with concrete reliability comparisons against Cerebras and latency requirements for voice.",
          "date": "2025-12-24",
          "type": "community",
          "title": "Nvidia to buy assets from Groq for $20B cash",
          "publisher": "Hacker News"
        }
      ],
      "sources": {
        "zdr": "https://console.groq.com/docs/your-data",
        "name": "https://groq.com/company",
        "soc2": "https://trust.groq.com/",
        "company": "https://en.wikipedia.org/wiki/Groq",
        "tagline": "https://groq.com/pricing",
        "sdk_note": "https://console.groq.com/docs/openai",
        "url_docs": "https://console.groq.com/docs/models",
        "url_home": "https://groq.com/company",
        "zdr_note": "https://console.groq.com/docs/your-data",
        "cap_audio": "https://console.groq.com/docs/models",
        "cap_batch": "https://console.groq.com/docs/rate-limits",
        "free_tier": "https://console.groq.com/docs/rate-limits",
        "extra_fees": "https://console.groq.com/docs/models",
        "hq_country": "https://en.wikipedia.org/wiki/Groq",
        "url_status": "https://groqstatus.com/",
        "zdr_status": "https://console.groq.com/docs/your-data",
        "url_pricing": "https://groq.com/pricing",
        "api_surfaces": "https://console.groq.com/docs/openai",
        "eu_residency": "https://console.groq.com/docs/your-data",
        "founded_year": "https://en.wikipedia.org/wiki/Groq",
        "product_fate": "https://groq.com/newsroom/groq-and-nvidia-enter-non-exclusive-inference-technology-licensing-agreement-to-accelerate-ai-inference-at-global-scale",
        "regions_note": "https://groq.com/platform",
        "vendor_status": "https://groq.com/newsroom/groq-closes-usd350-million-series-a-building-the-world-s-leading-ai-inference-cloud",
        "certifications": "https://console.groq.com/docs/legal/customer-data-processing-addendum",
        "config_as_code": "https://console.groq.com/docs/spend-limits",
        "retention_days": "https://console.groq.com/docs/your-data",
        "throughput_rps": "https://console.groq.com/docs/models",
        "url_models_api": "https://console.groq.com/docs/models",
        "cap_rate_limits": "https://console.groq.com/docs/rate-limits",
        "logs_by_default": "https://console.groq.com/docs/your-data",
        "model_count_note": "https://console.groq.com/docs/models",
        "eu_residency_note": "https://console.groq.com/docs/your-data",
        "groq_billing_faqs": "https://console.groq.com/docs/billing-faqs",
        "last_release_date": "https://console.groq.com/docs/changelog",
        "openai_compatible": "https://console.groq.com/docs/openai",
        "subprocessors_url": "https://console.groq.com/docs/legal/customer-data-processing-addendum",
        "groq_billing_plans": "https://console.groq.com/settings/billing/plans",
        "groq_prompt_caching": "https://console.groq.com/docs/prompt-caching",
        "guardrails_location": "https://console.groq.com/docs/your-data",
        "latency_source_note": "https://console.groq.com/docs/models",
        "requires_vendor_sdk": "https://console.groq.com/docs/openai",
        "self_host_cost_note": "https://console.groq.com/docs/models",
        "groq_pricing_marketing": "https://groq.com/pricing",
        "prompt_transits_vendor": "https://console.groq.com/docs/your-data",
        "trains_on_customer_data": "https://groq.com/privacy-policy"
      },
      "verified_at": {
        "zdr": "2026-08-29",
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "zdr_note": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_batch": "2026-08-29",
        "free_tier": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "eu_residency": "2026-08-29",
        "founded_year": "2026-08-29",
        "config_as_code": "2026-08-29",
        "throughput_rps": "2026-08-29",
        "url_models_api": "2026-08-29",
        "cap_rate_limits": "2026-08-29",
        "model_count_note": "2026-08-29",
        "eu_residency_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:40.755599+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Groq has not been acquired and says it will continue to operate as an independent company, having closed a $350M Series A in August 2026. But the shape of the company changed: in December 2025 Nvidia entered a non-exclusive inference-technology licensing agreement and hired founder Jonathan Ross, the president and other staff, with a new CEO taking over. Groq says GroqCloud will operate without interruption, though its public changelog has not been updated since April 2026.",
      "last_release_date": "2026-04-18",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": "Most recent GroqCloud changelog entry. Four months without a public update is an outlier among the actively developed products here.",
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Hosted inference. Where retention does occur, the data sits in Google Cloud buckets in the United States.",
      "logs_by_default": "none",
      "logs_note": "Groq does not retain customer data for inference requests by default. Usage metadata is collected for all users and contains no inputs or outputs. Temporary logging happens only to troubleshoot reliability-degrading errors or investigate suspected abuse.",
      "retention_days": 0,
      "retention_note": "Zero by default. Exceptions: up to 30 days for the reliability and abuse logs, 30 days for batch inputs and outputs, and fine-tuning artifacts held until you delete them.",
      "zdr_status": "yes",
      "trains_on_customer_data": "not_published",
      "training_note": "The data page is silent, and the privacy policy explicitly routes the question to the services agreement and data-processing addendum. Read that as silence, not a no.",
      "subprocessors_url": "https://trust.groq.com/subprocessors",
      "regions_note": "Retained data is US-only, in Google Cloud buckets. Inference itself spans 13 data centres on four continents including Finland, the UK, Saudi Arabia and Australia, but no customer region pinning is documented.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "the DPA commits to annual Type II audits; no report or portal retrieved",
          "status": "claimed"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "detail": "DPA published, with standard contractual clauses",
          "status": "certified"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "status": "not_published"
        }
      },
      "guardrails_location": "none",
      "guardrails_note": "No Groq-operated guardrail or PII-redaction feature was found.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference on a free/Developer/Enterprise ladder; pay-as-you-go per token with progressive billing. No separate gateway or platform fee — routing/API layer is $0.",
      "min_commitment": "None stated. Bills in arrears; progressive billing thresholds at $1, $10, $100, $500 and $1,000 (India: $1, $10, then recurring $100). $0.50 minimum before billing.",
      "overage_note": "None stated. Downgrading stops usage charges immediately and returns the account to Free tier rate limits and restrictions.",
      "cache_support": "prefix",
      "cache_read_discount_pct": 50,
      "cache_write_premium_pct": 0,
      "cache_pricing_note": "Prefix prompt caching, free to use. 50% discount on cached input tokens; pricing discount applies only on successful cache hits. No cache-write charge. Clean win for the customer — no feature fee, no write premium — with one caveat: batch requests already receive a 50% discount on all tokens, no additional discount applied to cached tokens in batch requests. The two discounts do not stack.",
      "cost_controls": {
        "enforcement": "mixed",
        "spend_alerts": {
          "note": "Usage alerts against budget thresholds.",
          "tier": null,
          "supported": true
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Developer tier gives 'significantly increased rate limits for production workloads'. Platform-set; configurable limits not stated.",
          "tier": "developer",
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated as per-key.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Account-level Spend Limits supported: automated spending limits and proactive usage alerts as you approach budget thresholds. Listed among Developer-tier benefits.",
          "tier": null,
          "supported": true
        }
      },
      "cost_attribution": "Not stated.",
      "cost_export": "Not stated.",
      "product_kind_note": "Groq presents GroqCloud as \"fast LLM inference, OpenAI-compatible\", where \"hosted models are directly accessible through the GroqCloud Models API endpoint\" ([Groq docs overview](https://console.groq.com/docs/overview); [Supported models](https://console.groq.com/docs/models)). It is a raw first-party inference API on Groq's own silicon — \"we pioneered the LPU\" ([Groq pricing](https://groq.com/pricing)).",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "No customer-installed component; usage is against `https://api.groq.com/openai/v1` with a Groq API key ([Groq docs overview](https://console.groq.com/docs/overview)). Capacity is Groq-operated: \"we're building hundreds of megawatts of capacity, with many more on the way\", and \"with LPX, it works alongside NVIDIA's next-generation GPUs\" ([Groq pricing](https://groq.com/pricing)). Tier gating shows up as rate limits under a \"DEVELOPER PLAN\" column, with some models' prices and limits marked \"Contact Sales\" ([Supported models](https://console.groq.com/docs/models)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "yes",
      "api_embeddings": "not_documented",
      "api_images": "not_documented",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap: \"We designed Groq API to be mostly compatible with OpenAI's client libraries\" — pass the Groq key as `api_key` and set `base_url` to `https://api.groq.com/openai/v1`; Groq also ships \"our own Groq Python and Groq TypeScript libraries that we encourage you to use\" ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)). Documented compatibility gaps: `logprobs`, `logit_bias`, `top_logprobs` and `messages[].name` are unsupported and \"will result in a 400 error\"; if `N` is supplied it must equal 1; `temperature` of 0 \"will be converted to `1e-8`\" (a float32 value > 0 and <= 2 is recommended); and audio `vtt` and `srt` response formats are not supported ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)). Model discovery is `GET https://api.groq.com/openai/v1/models`, returning \"a JSON list of all active models\" ([Supported models](https://console.groq.com/docs/models)).",
      "model_count_low": 6,
      "model_count_high": 16,
      "model_count_source_note": "No total is stated; the catalog lists 6 production models (`llama-3.1-8b-instant`, `llama-3.3-70b-versatile`, `openai/gpt-oss-120b`, `openai/gpt-oss-20b`, `whisper-large-v3`, `whisper-large-v3-turbo`), 2 production systems (`groq/compound`, `groq/compound-mini`) and 8 preview models, where preview models are \"for evaluation purposes only\", \"should not be used in production environments\", and \"may be discontinued at short notice\" ([Supported models](https://console.groq.com/docs/models), undated).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party only: Groq runs its own inference on its own hardware (\"we pioneered the LPU\") and does not route to third-party provider APIs, so no provider count is published ([Groq pricing](https://groq.com/pricing); [Supported models](https://console.groq.com/docs/models)).",
      "first_party_vs_partner": "Vendor hardware serving open-weight third-party models: Groq hosts models such as Llama, GPT-OSS, Whisper, Qwen and MiniMax on its own LPU/LPX infrastructure and exposes them through its API; there is no third-party routing ([Supported models](https://console.groq.com/docs/models); [Groq pricing](https://groq.com/pricing)).",
      "custom_endpoint_support": "n.a. (no customer endpoint registration or custom-model upload documented on the pages fetched) ([Supported models](https://console.groq.com/docs/models)).",
      "byok_mode": "platform_credits_only",
      "merchant_of_record": "Groq: it publishes and charges the token, character and hourly prices for its hosted models (e.g. \"$0.15 input $0.60 output\" per 1M tokens, \"$40.00 per 1M characters\", \"$0.111 per hour\"), with batch at a 50% discount and enterprise items shown as \"Contact Sales\" ([Supported models](https://console.groq.com/docs/models); [Groq Batch API](https://console.groq.com/docs/batch)). The pricing page itself does not state markup, BYOK or credit mechanics: n.a. ([Groq pricing](https://groq.com/pricing)).",
      "key_management_note": "Single-vendor key model: requests carry a Groq API key in the `Authorization: Bearer` header, sourced from `GROQ_API_KEY` and created in the Groq Console (\"you can find your API key here\") ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Supported models](https://console.groq.com/docs/models)). No third-party provider secrets exist. Because inference is hosted, Groq necessarily processes request content; explicit retention or vault/KMS statements: n.a. on the pages fetched.",
      "api_openai_chat_completions_note": "Yes: OpenAI-compatible base URL `https://api.groq.com/openai/v1`, with `/v1/chat/completions` used as a batch endpoint target too ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch)).",
      "api_anthropic_messages_note": "n.a. (not documented on the pages fetched).",
      "api_openai_responses_note": "Yes: `POST https://api.groq.com/openai/v1/responses`, described as \"a more advanced interface for generating model responses\" supporting text and image inputs, stateful conversations using previous responses as context, and function calling ([Groq docs overview](https://console.groq.com/docs/overview); [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "api_embeddings_note": "n.a. (no embeddings endpoint documented on the pages fetched).",
      "api_images_note": "n.a. as generation; image inputs are supported by the Responses API and vision chat requests ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch)).",
      "api_audio_note": "Yes: \"Audio Transcription and Translation\" (STT) via `/v1/audio/transcriptions` and `/v1/audio/translations`, served by `whisper-large-v3` / `whisper-large-v3-turbo`; TTS-style models appear in the catalog (`canopylabs/orpheus-v1-english`, priced per 1M characters) ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Groq Batch API](https://console.groq.com/docs/batch); [Supported models](https://console.groq.com/docs/models)).",
      "api_batch_note": "Yes: the Groq Batch API takes a JSONL file where \"each line in this file represents a single API call\", `POST` only, supporting `/v1/chat/completions`, `/v1/audio/transcriptions` and `/v1/audio/translations`, up to 50,000 lines and 200MB per file, up to 200 batch IDs per status query, multiple concurrent jobs, priced \"at a 50% cost discount compared to synchronous API pricing\", charging only successfully completed requests, and with rate limits separate from synchronous per-model limits ([Groq Batch API](https://console.groq.com/docs/batch)).",
      "deployment_modes_note": "hosted SaaS only (\"hosted models are directly accessible through the GroqCloud Models API endpoint\") ([Supported models](https://console.groq.com/docs/models)). Self-host, hybrid/VPC, on-prem, air-gapped: n.a. on the pages fetched.",
      "byok_mode_note": "platform_credits_only in effect: usage is authenticated with a Groq API key (`GROQ_API_KEY`, `Authorization: Bearer $GROQ_API_KEY`) and priced per 1M tokens / per 1M characters / per hour by Groq, with no upstream provider keys to supply ([Groq OpenAI compatibility](https://console.groq.com/docs/openai); [Supported models](https://console.groq.com/docs/models)).",
      "access_meta": {
        "qualified": [
          "api_images"
        ],
        "model_count_kind": "enumerated",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "`not_documented` on the fetched Groq docs ([Groq text generation docs](https://console.groq.com/docs/text-chat)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([Groq text generation docs](https://console.groq.com/docs/text-chat)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "`not_documented` ([Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "rel_load_balancing_config": "not_documented",
      "rel_load_balancing_note": "`not_documented` ([Groq text generation docs](https://console.groq.com/docs/text-chat)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "`vendor_saas` on the software side, with disclosed custom hardware: the LPU Inference Engine served through GroqCloud and the Groq API. No proxy runtime is disclosed ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "Only client SDKs are distributed (`pip install groq`, Groq Python and Groq TypeScript libraries) ([Groq text generation docs](https://console.groq.com/docs/text-chat), [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "latency_claims": [
        {
          "url": "https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubling-down-on-performance",
          "unit": "s",
          "scope": "end_to_end",
          "value": "0.8",
          "metric": "total for 100 output tokens",
          "publisher": "independent",
          "conditions": "Llama 2 Chat 70B, ~100-token prompts generating ~200 tokens; estimated from latency and throughput, refreshed every three hours."
        }
      ],
      "throughput_claims": [
        {
          "url": "https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubling-down-on-performance",
          "unit": "tokens/s",
          "scope": "generation_speed",
          "value": "241",
          "metric": "output speed",
          "publisher": "independent",
          "conditions": "Llama 2 Chat 70B, measured by ArtificialAnalysis.ai. Groq separately claims 300 tokens/s internally."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported via `stream=True`, returning \"an iterator of completion deltas rather than a single, full completion\"; async streaming supported with `AsyncGroq`. Non-streaming caveats on the OpenAI-compatible surface: `logprobs`, `logit_bias`, `top_logprobs` and `messages[].name` are unsupported and return `400`; `N` must equal `1`; `temperature: 0` is converted to `1e-8`; audio transcription/translation do not support `vtt` or `srt` ([Groq text generation docs](https://console.groq.com/docs/text-chat), [Groq OpenAI compatibility](https://console.groq.com/docs/openai)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "`not_documented` — no PII guardrail feature ([Your data](https://console.groq.com/docs/your-data))",
      "guard_injection": "not_documented",
      "guard_injection_note": "Llama Prompt Guard 2 is offered as a model you call yourself ([Content moderation](https://console.groq.com/docs/content-moderation))",
      "guard_content": "not_documented",
      "guard_content_note": "`not_documented` as an inline platform control — moderation is available only as callable models: GPT-OSS-Safeguard 20B, Llama Guard 4, and Llama Prompt Guard 2. The application must make a second call and act on the verdict; Groq does not filter the request path ([Content moderation](https://console.groq.com/docs/content-moderation))",
      "guard_custom": "not_documented",
      "guard_custom_note": "GPT-OSS-Safeguard accepts a custom policy in its prompt, but this is a model capability rather than a gateway policy engine ([Content moderation](https://console.groq.com/docs/content-moderation))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages ([Your data](https://console.groq.com/docs/your-data))",
      "guard_execution_location": "not_documented",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([Content moderation](https://console.groq.com/docs/content-moderation))",
      "log_content": "metadata_only",
      "log_content_note": "`metadata_only` — request metadata is always retained; customer prompt/completion data is not retained by default, though abuse-detection logs may be held for up to 30 days ([Your data](https://console.groq.com/docs/your-data))",
      "log_opt_out": "yes",
      "log_opt_out_note": "Zero Data Retention is available through Data Controls; metadata retention is not optional ([Your data](https://console.groq.com/docs/your-data))",
      "tracing_support": "otel",
      "tracing_note": "Via third-party OTel instrumentation: Arize Phoenix tracing is documented for Groq calls ([Arize + Groq](https://console.groq.com/docs/arize))",
      "export_destinations": [],
      "export_destinations_note": "Observability integrations are client-side: Arize, MLflow, and LangSmith ([Integrations](https://console.groq.com/docs/integrations)); Arize Phoenix specifically for OTel traces ([Arize + Groq](https://console.groq.com/docs/arize))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Integrations](https://console.groq.com/docs/integrations))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` from Groq; evaluation happens in the connected observability platform ([Integrations](https://console.groq.com/docs/integrations))",
      "observability_requires_saas": null,
      "log_retention_note": "Metadata retained (duration unspecified); abuse-related logs up to 30 days; no customer content retention by default ([Your data](https://console.groq.com/docs/your-data))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "**0.8 seconds** total response time to receive 100 output tokens, \"estimated based on latency and throughput\", where latency is defined as \"Time to first tokens chunk received, in seconds, after API request sent\". Conditions: Llama 2 Chat (70B), unique prompts ~100 tokens generating ~200 output tokens, benchmark refreshed every three hours ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results), [Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).",
        "multi_surface": {},
        "throughput_prose": "**241 tokens per second** on the Llama 2 Chat (70B) API as measured by ArtificialAnalysis.ai, \"more than double the speed of other hosting providers\"; throughput defined as tokens/sec received after the first chunk. Groq additionally states its internal benchmarks \"consistently\" reached **300 tokens per second**, and elsewhere claims customers run \"at 10x the speed\" with no value or conditions ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark), [Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results)).",
        "benchmark_independence": "Mixed, and the distinction matters: the 241 tok/s and 0.8 s figures originate from **independent third party ArtificialAnalysis.ai** but are reported on Groq's own pages; the 300 tok/s figure is **vendor internal**. The same third-party benchmark **includes competitors** (Microsoft Azure, Amazon Bedrock, Perplexity, Together.ai, Anyscale, Deepinfra, Fireworks, Lepton) — those rival numbers reach you via Groq's retelling, so verify against each rival before quoting ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "not_documented",
      "guard_custom_default": "not_documented",
      "rel_notes": "Groq is an inference provider with essentially no published reliability control surface: timeouts, retries, fallback, health and balancing are all your responsibility (or your gateway's). Also note the benchmark figures are dated Llama 2 70B-era results ([Groq LPU independent benchmark](https://groq.com/newsroom/groq-lpu-inference-engine-leads-in-first-independent-llm-benchmark)).",
      "guard_notes": "Groq is an inference provider, not a policy gateway: everything guardrail-shaped is a model in the catalogue and every observability integration is instrumentation you add in your own client ([Content moderation](https://console.groq.com/docs/content-moderation), [Integrations](https://console.groq.com/docs/integrations)).",
      "fit_agent_features_note": "The Responses API documents function calling, `code_interpreter`, `browser_search`, MCP tools, multi-step orchestration, exposed reasoning, tool discovery, and MCP approval flows using `mcp_approval_request` and `mcp_approval_response`. Groq’s `compound-beta` agentic tools system also supports code generation and execution, API calls, and webpage data retrieval; sessions, threads, and A2A are not documented. ([Groq remote MCP docs](https://console.groq.com/docs/tool-use/remote-mcp))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. The first call uses `GROQ_API_KEY` ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))",
      "fit_client_sdk_langs": [
        "Python",
        "TypeScript"
      ],
      "fit_client_sdks_note": "Groq provides official Python and JavaScript/TypeScript libraries: Python package `groq` with the `Groq` class, and JavaScript/TypeScript package `groq-sdk` with the `Groq` class; both offer synchronous and asynchronous clients. Groq is also compatible with OpenAI’s client libraries, and its documentation shows Vercel AI SDK packages `ai` and `@ai-sdk/groq`. ([Groq client libraries docs](https://console.groq.com/docs/libraries))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "third_party_documented",
      "fit_cloudflare_workers_note": "Documented by Cloudflare, not Groq: Groq is a supported Cloudflare AI Gateway provider, with base URL `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/groq` replacing `https://api.groq.com/openai/v1`; no Workers binding is documented on that page. ([Cloudflare AI Gateway - Groq provider](https://developers.cloudflare.com/ai-gateway/usage/providers/groq/))",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented). No credit-card, free-tier or trial-credit wording appears on the quickstart, rate-limits, or pricing pages ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart), [Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits), [Groq On-Demand Pricing](https://groq.com/pricing))",
      "fit_ecosystem_notes": "Groq’s documentation lists Google Workspace Connectors for Gmail, Google Calendar, and Google Drive, along with integrations for external services. Its ecosystem also includes Vercel deployment documentation, the Groq developer community and API Cookbook, and MCP integrations with clients such as Claude Desktop, Cursor, and Windsurf. ([Groq platform overview](https://console.groq.com/docs/overview))",
      "fit_eval_prereqs": "A Groq API key, recommended as the `GROQ_API_KEY` environment variable. No account, credit card, cloud account, cluster or provider key is mentioned ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))",
      "fit_eval_shape": "sdk_install",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `client.chat.completions.create(messages=[{\"role\":\"user\",\"content\":\"Explain the importance of fast language models\"}], model=\"llama-3.3-70b-versatile\")` ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": null,
      "fit_kubernetes": "not_documented",
      "fit_kubernetes_note": "n.a. (not documented)",
      "fit_mcp": "mcp_tools_in_api",
      "fit_mcp_note": "Yes. Groq’s Responses API supports remote MCP over HTTPS using tools such as `type: \"mcp\"`, `server_label`, `server_url`, `headers`, `require_approval`, and `allowed_tools`; remote MCP is currently in beta. Groq also documents a `groq-mcp` MCP server package usable with Claude Desktop and other MCP clients. ([Groq remote MCP docs](https://console.groq.com/docs/tool-use/remote-mcp))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_credit_card",
          "fit_gateway_platform",
          "fit_kubernetes",
          "fit_model_gate",
          "fit_terraform"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented) ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart), [Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits))",
      "fit_onboarding_notes": "Rate limits are the practical gate rather than approvals; the Developer plan unlocks higher limits plus Batch and Flex processing ([Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits)). The quickstart also points to an onboarding video, the Playground, the developer community and the Groq API Cookbook ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Groq supports OpenAI’s client libraries when configured with the base URL `https://api.groq.com/openai/v1` and a Groq API key. ([Groq OpenAI compatibility docs](https://console.groq.com/docs/openai))",
      "fit_prod_prereqs": "n.a. (not documented) beyond rate-limit tiering: \"Need higher rate limits? Upgrade to Developer plan to access higher limits, Batch and Flex processing, and more.\" ([Rate Limits - GroqDocs](https://console.groq.com/docs/rate-limits))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "sdk_install",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Groq documents LangChain integration through `langchain-groq` and the `ChatGroq` class. LlamaIndex integration is also documented for Python and JavaScript, with the LlamaIndex page additionally documenting installation of LlamaIndex. ([Groq LangChain integration docs](https://console.groq.com/docs/langchain))",
      "fit_quickstart_steps": 0,
      "fit_quickstart_steps_caveat": "The quickstart is prose plus a code sample with no numbered procedure.",
      "fit_quickstart_url": "https://console.groq.com/docs/quickstart",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "not_documented",
      "fit_terraform_note": "n.a. (not documented)",
      "fit_terraform_ref": null,
      "fit_third_party_documented": {
        "cloudflare_workers": "Cloudflare"
      },
      "fit_vendor_time_claim": "Vendor claim, verbatim: \"Get up and running with the Groq API in a few minutes\" ([Quickstart - GroqDocs](https://console.groq.com/docs/quickstart))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Yes. Groq provides Vercel AI SDK integration through `@ai-sdk/groq` and `ai`; the provider can be imported from `@ai-sdk/groq` and used as `groq('llama-3.3-70b-versatile')`. ([Groq AI SDK integration docs](https://console.groq.com/docs/ai-sdk/))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/groq"
    },
    {
      "slug": "together-ai",
      "name": "Together AI",
      "company": "Together AI",
      "tagline": "Inference provider running open-weight models on its own GPUs, with fine-tuning and dedicated endpoints.",
      "category": "inference_provider",
      "deployment": "managed_only",
      "license": "Proprietary",
      "founded_year": 2022,
      "hq_country": "United States",
      "url_home": "https://www.together.ai/",
      "url_docs": "https://docs.together.ai/docs/introduction",
      "url_pricing": "https://www.together.ai/pricing",
      "url_status": "https://status.together.ai/",
      "url_github": null,
      "url_changelog": null,
      "url_models_api": "https://api.together.ai/v1/models",
      "github_stars": null,
      "model_count": 200,
      "model_count_note": "Vendor model directory shows \"200+ models\" across chat, image, vision, video, audio, embedding, rerank and moderation categories; rerank is not offered on serverless.",
      "provider_count": 40,
      "token_markup_pct": null,
      "credit_fee_pct": null,
      "credit_fee_min_usd": null,
      "byok_supported": null,
      "byok_fee_note": null,
      "seat_fee_usd_month": null,
      "free_tier": null,
      "enterprise_from_usd_year": null,
      "extra_fees": [
        {
          "label": "Fine-tuning",
          "amount": "$0.48-$2.90 per 1M tokens with a $4.00 minimum per job"
        },
        {
          "label": "Dedicated GPU endpoints",
          "amount": "H100 $5.49/hour, B200 $8.99/hour"
        },
        {
          "label": "GPU clusters (on-demand)",
          "amount": "H100 $3.99, H200 $5.99, B200 $8.19 per GPU-hour"
        },
        {
          "label": "Code Sandbox / Code Interpreter",
          "amount": "$0.0446 per vCPU, $0.03 per interpreter session"
        },
        {
          "label": "Shared filesystem",
          "amount": "$0.16 per GiB per month"
        }
      ],
      "self_host_cost_note": "No self-host option; Together sets its own per-token prices rather than marking up another vendor's list price. Representative serverless rates include MiniMax M3 at $0.30 per 1M input, gpt-oss-120B at $0.15, DeepSeek V4 Flash at $0.14, and Kimi K3 at $3.00 input / $15.00 output. Discounted batch rates and provisioned throughput sold in PTUs are also published, plus embeddings from $0.02 per 1M tokens.",
      "openai_compatible": true,
      "requires_vendor_sdk": false,
      "data_export_available": null,
      "config_as_code": false,
      "portability_note": "Dashboard cost analytics only; no documented export.",
      "api_surfaces": [
        "OpenAI Chat Completions",
        "Together Batch API",
        "Fine-tuning API",
        "Dedicated endpoints"
      ],
      "sdk_note": "Documented as an OpenAI-compatible API, so OpenAI SDKs work by changing the base URL; the model list endpoint returns id, type, organization, license, context_length and a pricing object.",
      "cap_failover": null,
      "cap_load_balancing": null,
      "cap_conditional_routing": null,
      "cap_cache_simple": null,
      "cap_cache_semantic": null,
      "cap_observability": null,
      "cap_prompt_management": null,
      "cap_evals": null,
      "cap_guardrails": true,
      "cap_pii_redaction": null,
      "cap_budgets": null,
      "cap_rate_limits": null,
      "cap_virtual_keys": null,
      "cap_mcp": null,
      "cap_embeddings": true,
      "cap_image_gen": true,
      "cap_audio": true,
      "cap_video": true,
      "cap_batch": true,
      "latency_overhead_ms": null,
      "latency_source_note": "Not a proxy, so there is no gateway overhead figure. Third-party measurements: a Telnyx-run streamed benchmark on GLM-5.2 (240 successful requests, 10 runs per provider, June 25 2026) recorded a 0.79 s median time-to-first-token for Together versus 1.28 s for Fireworks; a separate Python harness on Llama 3.1 8B reported roughly 250 tokens/sec median for Together versus 580 for Groq and 320 for Fireworks, with the author noting run-to-run variance.",
      "throughput_rps": null,
      "soc2": true,
      "hipaa_baa": true,
      "gdpr": null,
      "eu_residency": null,
      "eu_residency_note": null,
      "zdr": null,
      "zdr_note": "On by default and user-controlled, but note the boundary: ZDR applies only from the moment you enable it and does nothing about data already processed.",
      "air_gapped": null,
      "best_for": "Teams that want a broad open-weight catalog plus cheap fine-tuning and the option to move to dedicated GPUs on one vendor.",
      "strengths": [
        "200+ open-weight models across text, vision, image, video, audio and embeddings in one OpenAI-compatible API",
        "Clear path from serverless to dedicated endpoints, GPU clusters and provisioned throughput as volume grows",
        "Inexpensive fine-tuning from $0.48 per 1M tokens with a $4 job minimum",
        "SOC 2 Type 2 completed and a public trust center, with HIPAA BAAs in place with partners",
        "Model list endpoint publishes per-model pricing, licence and context length for programmatic catalog checks"
      ],
      "weaknesses": [
        "Single-source provider: no independent catalog of closed frontier models and no cross-provider routing or failover",
        "No published free tier, and third-party reporting says new accounts need a prepaid minimum balance",
        "GDPR posture, EU data residency and zero-data-retention are not documented on any vendor page found",
        "Serverless is multi-tenant, so throughput can vary with neighbouring load and model IDs can be rotated or deprecated"
      ],
      "avoid_if": "You need closed frontier models, cross-provider failover, or contractual EU data residency.",
      "reviews": [
        {
          "url": "https://techcrunch.com/2026/07/01/neocloud-together-ai-raises-800m-leaps-to-8-3b-valuation/",
          "why": "TechCrunch situates Together's inference business in the neocloud market, explaining how its GPU-cluster economics and model-serving demand differ from routing platforms.",
          "date": "2026-07-01",
          "type": "written",
          "title": "Neocloud Together AI raises $800M, leaps to $8.3B valuation",
          "publisher": "TechCrunch"
        },
        {
          "url": "https://www.youtube.com/watch?v=FybjHMBQV8M",
          "why": "Craig Smith interviews Together's VP of Research about the inference stack and infrastructure choices behind the hosted service.",
          "date": "2025-08-25",
          "type": "video",
          "title": "Inside Together AI: The Research Driving Next-Gen Open-Source Models",
          "publisher": "YouTube - Eye on AI"
        },
        {
          "url": "https://www.youtube.com/watch?v=_GQfj3jhXVM",
          "why": "sentdex builds against Together's inference API hands-on - keys, streaming, prompt formatting, model parameters - and ships a TermGPT demo on top of it.",
          "date": "2023-12-24",
          "type": "video",
          "title": "Open Source AI Inference API w/ Together",
          "publisher": "YouTube - sentdex"
        },
        {
          "url": "https://www.reddit.com/r/AI_Agents/comments/1ugzoe2/anyone_moved_away_from_togetherai_looking_for/",
          "why": "37-comment migration thread where users report latency inconsistency and failed model hosting on Together, and compare Featherless, Groq, Cerebras, DeepInfra and Fireworks.",
          "date": "2026-06-27",
          "type": "community",
          "title": "Anyone moved away from Together.ai? Looking for alternatives",
          "publisher": "Reddit - r/AI_Agents"
        },
        {
          "url": "https://www.reddit.com/r/LocalLLaMA/comments/1qozrne/for_those_using_hosted_inference_providers/",
          "why": "27-comment thread with real spend and failure detail (e.g. ~$250/month pod bills, lost pod storage) comparing Together against other hosted inference providers.",
          "date": "2026-01-28",
          "type": "community",
          "title": "For those using hosted inference providers (Together, Fireworks, Baseten, RunPod, Modal) - what do you love and hate?",
          "publisher": "Reddit - r/LocalLLaMA"
        }
      ],
      "sources": {
        "name": "https://www.together.ai/pricing",
        "soc2": "https://www.together.ai/blog/soc-2-compliance",
        "company": "https://www.together.ai/about-us",
        "tagline": "https://docs.together.ai/docs/introduction",
        "sdk_note": "https://docs.together.ai/reference/models-1",
        "url_docs": "https://docs.together.ai/docs/introduction",
        "url_home": "https://www.together.ai/",
        "cap_audio": "https://docs.together.ai/docs/serverless-models",
        "cap_batch": "https://www.together.ai/pricing",
        "cap_video": "https://docs.together.ai/docs/serverless-models",
        "hipaa_baa": "https://www.together.ai/blog/soc-2-compliance",
        "extra_fees": "https://www.together.ai/pricing",
        "hq_country": "https://www.forbes.com/companies/together-ai/",
        "url_status": "https://status.together.ai/",
        "zdr_status": "https://www.together.ai/privacy",
        "model_count": "https://www.together.ai/models",
        "url_pricing": "https://www.together.ai/pricing",
        "api_surfaces": "https://docs.together.ai/docs/introduction",
        "founded_year": "https://www.forbes.com/companies/together-ai/",
        "product_fate": "https://www.together.ai/terms-of-service",
        "regions_note": "https://support.together.ai/articles/8079447813-eu-data-centers-and-dedicated-model-deployment",
        "cap_image_gen": "https://docs.together.ai/docs/serverless-models",
        "vendor_status": "https://www.reuters.com/legal/transactional/together-ai-raises-800-million-83-billion-valuation-2026-07-01/",
        "cap_embeddings": "https://www.together.ai/pricing",
        "cap_guardrails": "https://docs.together.ai/docs/serverless-models",
        "certifications": "https://www.together.ai/blog/soc-2-compliance",
        "config_as_code": "https://docs.together.ai/docs/billing-credits",
        "provider_count": "https://www.together.ai/models",
        "retention_days": "https://docs.together.ai/docs/privacy-and-security",
        "url_models_api": "https://docs.together.ai/reference/models-1",
        "logs_by_default": "https://docs.together.ai/docs/privacy-and-security",
        "model_count_note": "https://docs.together.ai/docs/serverless-models",
        "together_pricing": "https://www.together.ai/pricing",
        "last_release_date": "https://docs.together.ai/docs/changelog",
        "openai_compatible": "https://docs.together.ai/docs/introduction",
        "guardrails_location": "https://docs.together.ai/docs/privacy-and-security",
        "latency_source_note": "https://markaicode.com/vs/together-ai-vs-fireworks-ai/",
        "requires_vendor_sdk": "https://docs.together.ai/docs/inference/openai-compatibility",
        "self_host_cost_note": "https://www.together.ai/pricing",
        "prompt_transits_vendor": "https://docs.together.ai/docs/privacy-and-security",
        "trains_on_customer_data": "https://docs.together.ai/docs/privacy-and-security",
        "together_serverless_overview": "https://docs.together.ai/docs/serverless/overview.md"
      },
      "verified_at": {
        "name": "2026-08-29",
        "soc2": "2026-08-29",
        "company": "2026-08-29",
        "tagline": "2026-08-29",
        "sdk_note": "2026-08-29",
        "url_docs": "2026-08-29",
        "url_home": "2026-08-29",
        "cap_audio": "2026-08-29",
        "cap_batch": "2026-08-29",
        "cap_video": "2026-08-29",
        "hipaa_baa": "2026-08-29",
        "extra_fees": "2026-08-29",
        "hq_country": "2026-08-29",
        "url_status": "2026-08-29",
        "model_count": "2026-08-29",
        "url_pricing": "2026-08-29",
        "api_surfaces": "2026-08-29",
        "founded_year": "2026-08-29",
        "cap_image_gen": "2026-08-29",
        "cap_embeddings": "2026-08-29",
        "cap_guardrails": "2026-08-29",
        "config_as_code": "2026-08-29",
        "provider_count": "2026-08-29",
        "url_models_api": "2026-08-29",
        "model_count_note": "2026-08-29",
        "openai_compatible": "2026-08-29",
        "latency_source_note": "2026-08-29",
        "requires_vendor_sdk": "2026-08-29",
        "self_host_cost_note": "2026-08-29"
      },
      "locked_fields": [],
      "published": true,
      "updated_at": "2026-08-31T04:37:46.577984+00:00",
      "vendor_status": "independent",
      "acquired_by": null,
      "acquired_date": null,
      "acquisition_terms": null,
      "product_fate": "Actively developed with near-daily platform changes. Raised an $800M Series C at an $8.3B valuation in July 2026. Its terms of service explicitly disclaim any uptime guarantee unless separately agreed in an order form.",
      "last_release_date": "2026-08-28",
      "sla_uptime_pct": null,
      "sla_url": null,
      "claim_basis": {},
      "last_release_note": null,
      "prompt_transits_vendor": "yes",
      "prompt_path_note": "Third-party models hosted on Together run on its own infrastructure. Some models are offered as passthrough, meaning Together forwards your prompts and responses straight to the upstream provider — leave that toggle off if you do not want traffic leaving Together.",
      "logs_by_default": "none",
      "logs_note": "Inputs and outputs are not stored by default. Temporary caching may be used to improve performance unless configured otherwise.",
      "retention_days": 0,
      "retention_note": "Zero by default. No retention window is published for the opt-in storage path.",
      "zdr_status": "yes",
      "trains_on_customer_data": "opt_in",
      "training_note": "Data sharing for training is opt-in and not enabled by default; Together states it does not train on your data without explicit consent.",
      "subprocessors_url": null,
      "regions_note": "US by default. EU data centres exist for inference and storage inside the EU but only on Scale and Enterprise plans, serverless endpoints offer no region selection, and private networking and VPC-based deployments are supported.",
      "certifications": {
        "itar": {
          "status": "not_published"
        },
        "soc2": {
          "detail": "a blog post reports a completed Type 2 examination; the trust center lists only Type 1",
          "status": "conflicting"
        },
        "fedramp": {
          "status": "not_published"
        },
        "gdpr_dpa": {
          "status": "not_published"
        },
        "iso27001": {
          "status": "not_published"
        },
        "hipaa_baa": {
          "detail": "adheres to HIPAA with BAAs signed with its partners, not offered to customers",
          "status": "alignment_only"
        }
      },
      "guardrails_location": "none",
      "guardrails_note": "Deliberately none: models are hosted at full precision, and Together states it does not distil them, force system prompts or layer censorship on top.",
      "acquired_date_basis": null,
      "security_incidents": [],
      "pricing_model": "bundled_inference",
      "pricing_model_note": "Pay-for-own-inference across serverless per-token, Batch API, Provisioned Throughput (PTUs), dedicated single-tenant endpoints, GPU clusters, sandbox and storage. Enterprise quote for some SKUs. No separate gateway/platform fee — routing layer is $0.",
      "min_commitment": "None for on-demand serverless. Fine-tuning has a $4.00 minimum charge per job. PTU savings estimates assume continuous 24/7 provisioning (~43,800 min/mo) but no commitment is required.",
      "overage_note": "None stated. Cost risks are structural — dedicated endpoints and PTUs bill for provisioned capacity whether or not you use it; storage is billed per GiB/month.",
      "cache_support": "prefix",
      "cache_read_discount_pct": null,
      "cache_write_premium_pct": 0,
      "cache_pricing_note": "Automatic prefix caching on select serverless models, best-effort, plus per-replica caching on dedicated endpoints. Prefix-based: only the longest matching prefix counts. Serverless cache is shared across the fleet and entries are evicted as traffic shifts — cache hits aren't guaranteed and there's no configurable retention window. Docs say 'a steep discount', pricing table publishes numbers: MiniMax M3 $0.06 vs $0.30 (80% off), Kimi K3 $0.30 vs $3.00 (90% off), DeepSeek V4 Flash $0.03 vs $0.14 (~79% off), GLM-5.3 $0.26 vs $1.40 (~81% off), Qwen3.8-2.4T-A95B $0.25 vs $2.00 (87.5% off). No cache-write charge. Only dedicated endpoints make the saving reliable.",
      "cost_controls": {
        "enforcement": "not_stated",
        "spend_alerts": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "virtual_keys": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "rate_limiting": {
          "note": "Exists on serverless models but values and configurability not stated.",
          "tier": null,
          "supported": true
        },
        "spend_webhooks": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "model_allowlists": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_key": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        },
        "budget_caps_per_team": {
          "note": "Not stated.",
          "tier": null,
          "supported": null
        }
      },
      "cost_attribution": "Not stated.",
      "cost_export": "Not stated.",
      "product_kind_note": "Together AI's docs say \"Run, train, and serve open-source AI models on Together AI\" via \"our OpenAI-compatible API\", also described as the \"Together AI OpenAI compatibility layer\" ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)). It is a first-party inference, fine-tuning and GPU platform, not a router.",
      "deployment_modes": [
        "saas"
      ],
      "deployment_note": "Serverless requires no provisioning and is charged on \"the tokens you use and the size of the model\"; dedicated endpoints serve \"a single model on hardware reserved only for you\", bill \"per-minute by hardware while the endpoint is running, regardless of your model or request volume\", bill each replica independently and stop billing \"as soon as they are scaled down\" ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)). Hardware options listed for dedicated inference are NVIDIA HGX H100, H200, B200, B300, GB200 NVL72 and GB300 NVL72, with GPU clusters \"pay as you go… on an hourly basis\" ([Together AI pricing](https://www.together.ai/pricing)). Migration is code-free: \"prototype with serverless, then switch to dedicated endpoints without changing your application code\", since \"dedicated endpoints use the same inference APIs as serverless models\" ([Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).",
      "api_openai_chat_completions": "yes",
      "api_anthropic_messages": "not_documented",
      "api_openai_responses": "not_documented",
      "api_embeddings": "yes",
      "api_images": "yes",
      "api_audio": "yes",
      "api_batch": "yes",
      "api_surface_note": "Drop-in base-URL swap: existing OpenAI Python/TypeScript or curl code moves over with \"two changes: the API key and base URL\" (`https://api.together.ai/v1`); first-party SDKs (`together`, `together-ai`) exist but are optional ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)). Documented gaps: `assistants.*`/`threads.*`/`runs.*` not supported (build agent loops with chat completions and function calling), OpenAI-shaped `fine_tuning.jobs.*` not supported (use Together's native fine-tuning API), `files.*` only partial (Together has its own Files API), `batches.*` not supported (native Batch API), `moderations.create` not supported (use Llama Guard through chat completions); `GET /v1/models` is supported for listing/retrieval ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).",
      "model_count_low": 19,
      "model_count_high": 19,
      "model_count_source_note": "n.a.: no aggregate total is stated; the overview lists 7 chat, 6 audio, 3 image and 3 vision models by name (e.g. Kimi K3, DeepSeek V4 Flash 0731, GLM-5.2, MiniMax M3, GPT OSS 120B, Whisper Large v3, FLUX.2 [max]) ([Together AI docs overview](https://docs.together.ai/intro), undated; [Serverless models](https://docs.together.ai/docs/serverless/models)).",
      "provider_count_low": null,
      "provider_count_high": null,
      "provider_count_source_note": "First-party only: \"Together AI hosts many popular models\" on its own serverless and dedicated infrastructure and does not route to third-party provider APIs, so no provider count is published ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).",
      "first_party_vs_partner": "Vendor-operated hardware serving open-weight third-party models: models are \"available via serverless or dedicated model inference\", a dedicated endpoint serves \"a single model on hardware reserved only for you\", and the \"Available models\" list is described as \"Together-hosted models you can deploy on dedicated endpoints\" ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)). Whether the GPUs are owned or leased is not stated: n.a. ([Together AI pricing](https://www.together.ai/pricing)).",
      "custom_endpoint_support": "Bring-your-own-weights, not bring-your-own-endpoint: \"Bring your own model — upload weights and serve them via the Together AI API\", and dedicated endpoints can serve \"custom fine-tuned models\" / uploaded \"own model weights\"; registering an external private URL, vLLM or SageMaker endpoint is not documented ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints)).",
      "byok_mode": "platform_credits_only",
      "merchant_of_record": "Together AI: serverless usage is charged on \"the tokens you use and the size of the model\", dedicated inference on GPU hours / per-minute hardware time, GPU clusters hourly, with separate published prices for audio minutes, TTS characters, batch, fine-tuning, sandboxes and storage ([Together AI docs overview](https://docs.together.ai/intro); [Dedicated endpoints](https://docs.together.ai/docs/dedicated-endpoints); [Together AI pricing](https://www.together.ai/pricing)).",
      "key_management_note": "Single-vendor key model: requests authenticate with `Authorization: Bearer $TOGETHER_API_KEY` obtained from Together's hosted API settings; no third-party provider secrets are stored ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)). Because inference is hosted, Together processes request content; explicit statements on retention, encryption, KMS or vendor visibility: n.a. on the pages fetched.",
      "api_openai_chat_completions_note": "Yes: `POST https://api.together.ai/v1/chat/completions`, including vision inputs, tools/function calling and `response_format` structured outputs ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)).",
      "api_anthropic_messages_note": "n.a. (explicitly not mentioned in the OpenAI-compatibility matrix) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).",
      "api_openai_responses_note": "n.a. (not mentioned) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).",
      "api_embeddings_note": "Yes: `POST /v1/embeddings` ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).",
      "api_images_note": "Yes: `POST /v1/images/generations` (with streaming image generation, e.g. \"stream real-time image generations with FLUX Schnell\") ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI docs overview](https://docs.together.ai/intro)).",
      "api_audio_note": "Yes, both directions: `POST /v1/audio/speech` (TTS), `POST /v1/audio/transcriptions` and `POST /v1/audio/translations` (STT), with per-audio-minute ASR pricing (Nemotron 3 ASR Streaming 0.6B, Parakeet TDT 0.6B v3, Nemotron 3.5 ASR, Whisper Large v3 and its streaming variant) and per-1M-character TTS pricing (Parakeet TDT 0.6B V3 Realtime, Cartesia Sonic-3, Orpheus TTS, Kokoro-82M TTS, Cartesia Sonic-2) ([OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI pricing](https://www.together.ai/pricing)).",
      "api_batch_note": "Yes, but native rather than OpenAI-shaped: \"queue async generations and fetch results later\" via \"Together's native Batch API\", while `batches.*` in the OpenAI shape is \"not supported\"; batch prices are listed as \"Batch API price\" on the pricing page ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility); [Together AI pricing](https://www.together.ai/pricing)).",
      "deployment_modes_note": "hosted cloud service in several shapes: serverless inference, dedicated single-tenant model inference (\"provision single-tenant GPUs for predictable, isolated latency\"), dedicated containers \"on managed GPUs with autoscaling and observability\", GPU clusters (\"spin up H100 and B200 clusters with attached storage for training or large batch jobs\"), plus Sandbox VM environments ([Together AI docs overview](https://docs.together.ai/intro); [Together AI pricing](https://www.together.ai/pricing)). Self-host in customer infrastructure, hybrid/VPC, on-prem, air-gapped: n.a. on the pages fetched.",
      "byok_mode_note": "platform_credits_only in effect: all usage runs on a Together API key (`Authorization: Bearer $TOGETHER_API_KEY`) and is billed by Together; there are no upstream provider keys to supply ([Together AI docs overview](https://docs.together.ai/intro); [OpenAI compatibility](https://docs.together.ai/docs/inference/openai-compatibility)).",
      "access_meta": {
        "qualified": [
          "api_audio",
          "api_batch"
        ],
        "model_count_kind": "enumerated",
        "provider_count_kind": null
      },
      "rel_timeout_config": "not_documented",
      "rel_timeout_note": "Timeout appears only as a documented response code: `504 Timeout` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "rel_retry_config": "not_documented",
      "rel_retry_note": "`not_documented`; default count and backoff: `n.a.` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "rel_retry_default_note": null,
      "rel_fallback_config": "not_documented",
      "rel_fallback_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "rel_fallback_shape": "not_documented",
      "rel_health_check_config": "not_documented",
      "rel_health_check_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "rel_multi_region_config": "not_documented",
      "rel_multi_region_note": "Dedicated endpoints on Together Cloud are documented, but no regional placement or cross-region failover ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).",
      "rel_load_balancing_config": "not_documented",
      "rel_load_balancing_note": "`not_documented` ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "architecture_class": "vendor_saas",
      "architecture_note": "Serverless and dedicated endpoints at `api.together.ai`; no runtime disclosed beyond the named inference engine ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).",
      "self_host_data_plane": "no",
      "self_host_artifact_note": "Only client SDKs (`from together import Together`; `import Together from \"together-ai\"`) ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "latency_claims": [],
      "throughput_claims": [
        {
          "url": "https://www.together.ai/blog/together-inference-engine-2",
          "unit": "tokens/s",
          "scope": "generation_speed",
          "value": "over 400",
          "metric": "decoding throughput",
          "publisher": "vendor",
          "conditions": "Meta Llama 3 8B on the Together Inference Engine 2.0."
        }
      ],
      "streaming_support": "yes",
      "streaming_note": "Supported. `stream` (boolean): when true it will \"stream tokens as Server-Sent Events as the model generates them\", and \"The stream terminates with `data: [DONE]`\"; the `200` response supports both `application/json` and `text/event-stream`, with schema `ChatCompletionStream` = `ChatCompletionEvent` | `StreamSentinel` and chunk type `chat.completion.chunk`. No streaming caveat is stated ([Together chat completions reference](https://docs.together.ai/reference/chat-completions-1)).",
      "guard_pii": "not_documented",
      "guard_pii_note": "`not_documented` ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))",
      "guard_injection": "not_documented",
      "guard_injection_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))",
      "guard_content": "sync_block",
      "guard_content_note": "`sync_block`-style filtering via an inline request parameter — adding `\"safety_model\": \"Meta-Llama/Llama-Guard-7b\"` to a chat completion makes Together run the safety model and filter the response before returning it. The control is per request, chosen by the caller, not an administrator policy ([Safety models](https://www.together.ai/blog/safety-models))",
      "guard_custom": "not_documented",
      "guard_custom_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))",
      "guard_model_policy": "not_documented",
      "guard_model_policy_note": "`not_documented` on the fetched pages ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))",
      "guard_execution_location": "vendor_saas",
      "guard_third_party": [],
      "guard_failure_mode": "not_documented",
      "guard_failure_mode_note": "`not_documented` ([Safety models](https://www.together.ai/blog/safety-models))",
      "log_content": "metadata_only",
      "log_content_note": "`metadata_only` — zero data retention is the default; organisation-level toggles exist for storing prompts, allowing training, and provider passthrough ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))",
      "log_opt_out": "yes",
      "log_opt_out_note": "`yes` — prompt storage is an opt-in organisation setting, so the default already keeps metrics without bodies ([Privacy and security](https://docs.together.ai/docs/privacy-and-security))",
      "tracing_support": "otel",
      "tracing_note": "`n.a.` — no OpenTelemetry or agent tracing documented; the endpoint feed is an audit event stream, not request traces ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))",
      "export_destinations": [],
      "export_destinations_note": "A Prometheus-compatible metrics endpoint, `GET https://o11y-de2-metrics.cloud.together.ai/organizations/{org_id}/metrics`, working with Prometheus, Grafana Agent, the Datadog OpenMetrics integration, and Vector; it is in beta and \"access may need to be enabled for your organization\". Events are also readable via `tg beta endpoints events` with `--json` ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))",
      "feedback_capture": "no",
      "feedback_capture_note": "`n.a.` ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))",
      "eval_hooks": "no",
      "eval_hooks_note": "`n.a.` — the API exposes `logprobs` for scoring outputs client-side, but no eval service on logged traffic ([Logprobs](https://docs.together.ai/docs/logprobs))",
      "observability_requires_saas": null,
      "log_retention_note": "No prompt retention by default; endpoint metrics/events retention is not stated ([Privacy and security](https://docs.together.ai/docs/privacy-and-security), [Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring))",
      "production_meta": {
        "sources": {},
        "qualified": [],
        "latency_prose": "`n.a.` No latency-metric figure (no TTFT, p50, p99 or overhead) is published on the fetched pages; all published numbers are throughput/speedup ratios ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).",
        "multi_surface": {},
        "throughput_prose": "All from the engine announcement, metric = decoding throughput (tokens/sec) unless noted: - \"over **400 tokens per second** on Meta Llama 3 8B\" - \"decoding throughput **4x faster** than open-source vLLM\" - Together Turbo: \"up to **4.5x** performance improvement over vLLM (version 0.5.1)\" on Llama-3-8B-Instruct and Llama-3-70B-Instruct; **2.8x–4.5x** and **2.6x–4.3x** decoding speedup respectively, with Llama-3-8B-Instruct on a single H100 and Llama-3-70B-Instruct on 8×H100 - Together Turbo: \"up to **7x the capacity** of vLLM (version 0.5.1)\" and up to 7x cost reduction; on 4×H100 Turbo lands \"within 8%\" of Together Reference on 8×H100 - Together Lite: \"**12x reduction in cost**\" vs vLLM, and on two A100s it \"outperforms vLLM FP16 and FP8 running on eight H100 GPUs by up to **30%**\" - Together Reference: \"**4x speedup** over the state-of-the-art inference engine vLLM across normal serving regimes\" Payload/prompt length, concurrency and cache conditions are not stated for any of these ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)).",
        "benchmark_independence": "All vendor self-published, and **heavily comparative against rivals**: vLLM 0.5.1 (open source), and the claim that the engine \"outperforms commercial solutions including Amazon Bedrock, Azure AI, Fireworks, and Octo AI by 1.3x to 2.5x\", with Together Reference \"over 2x faster tokens per second than Amazon Bedrock, Microsoft Azure, or Octo AI; and over 30% faster than Fireworks FP8 models\". Those competitor figures are vendor claims about rivals and are not verifiable from the rivals' own docs ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2)). Together also appears as a compared provider in the third-party ArtificialAnalysis.ai benchmark reported by Groq ([Groq LPU benchmark blog](https://groq.com/blog/artificialanalysis-ai-llm-benchmark-doubles-axis-to-fit-new-groq-lpu-inference-engine-performance-results))."
      },
      "guard_pii_default": "not_documented",
      "guard_injection_default": "not_documented",
      "guard_content_default": "choose_at_setup",
      "guard_custom_default": "not_documented",
      "rel_notes": "Quality/speed tiering (Turbo / Lite / Reference) is the main reliability-adjacent lever, and the published comparisons are ratios against vLLM rather than absolute latency, which makes them hard to translate into an SLO. Streaming ASR models are listed separately in the serverless catalog ([Together Inference Engine 2.0](https://www.together.ai/blog/together-inference-engine-2), [Together serverless models](https://docs.together.ai/docs/serverless-models)).",
      "guard_notes": "Together's observability is infrastructure-shaped, not request-shaped: edge/router/worker metric layers, latency histograms, and `_total` counters for dedicated endpoints, with no per-request log of prompts to inspect ([Monitor endpoints and deployments](https://docs.together.ai/docs/dedicated-endpoints/monitoring)).",
      "fit_agent_features_note": "Together supports function calling through `chat.completions.create` with `tools` and `tool_choice`. `assistants.*`, `threads.*`, and `runs.*` are not supported; the documentation instructs users to build agent loops on top of chat completions and function calling. ([Together OpenAI compatibility docs](https://docs.together.ai/docs/inference/openai-compatibility))",
      "fit_byok_required": "no",
      "fit_byok_required_note": "No. The first call uses `TOGETHER_API_KEY` ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))",
      "fit_client_sdk_langs": [
        "Python",
        "TypeScript"
      ],
      "fit_client_sdks_note": "Together documents official Python and TypeScript SDKs: Python package `together` and TypeScript npm package `together-ai`. Its integration documentation also covers Hugging Face with `huggingface_hub` and `InferenceClient` or `@huggingface/inference` and `HfInference`, LangChain with `langchain-together` and `ChatTogether`, and LlamaIndex with `llama-index` and `OpenAILike`. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))",
      "fit_cloud_iam": "not_documented",
      "fit_cloud_iam_note": "n.a. (not documented)",
      "fit_cloudflare_workers": "not_documented",
      "fit_cloudflare_workers_note": "n.a. (not documented)",
      "fit_credit_card": "not_documented",
      "fit_credit_card_note": "n.a. (not documented). Neither the quickstart nor the pricing page states a credit-card requirement, free tier, or trial credits ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))",
      "fit_ecosystem_notes": "Together documents a REST API at `https://api.together.ai/v1`, an OpenAI SDK-compatible interface, dedicated model inference, fine-tuning, and GPU clusters. Its integration documentation lists Hugging Face, Vercel AI SDK, LangChain, LlamaIndex, Helicone, CrewAI, LangGraph, DSPy, PydanticAI, AutoGen (AG2), Agno, Composio, and RAG integrations; Together-native endpoints include video generation, image edits and inpainting beyond `images.generate`, reasoning controls and `reasoning_content`, and the logprobs surface that are not exposed by the OpenAI SDKs. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))",
      "fit_eval_prereqs": "None stated separately; account registration is folded into step 1. No credit card, cloud account, cluster or provider key is mentioned ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))",
      "fit_eval_shape": "sdk_install",
      "fit_eval_snippet": "yes",
      "fit_eval_snippet_note": "Yes: `client.chat.completions.create(model=\"MiniMaxAI/MiniMax-M3\", messages=[{\"role\":\"user\",\"content\":\"What are the top 3 things to do in New York?\"}])`, run from a saved file or directly via cURL ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))",
      "fit_gateway_platform": "not_documented",
      "fit_gateway_platform_note": "n.a. (not documented)",
      "fit_helm_chart": "only a third-party Volcano scheduler chart on Together GPU clusters",
      "fit_kubernetes": "generic_platform_only",
      "fit_kubernetes_note": "No Helm chart or operator is documented for Together's inference gateway itself; Together's Kubernetes documentation covers installing the third-party Volcano scheduler chart (`volcano-sh/volcano` from `https://volcano-sh.github.io/helm-charts`) on Together GPU clusters. ([Together docs - Volcano on GPU clusters](https://docs.together.ai/docs/volcano-on-gpu-clusters))",
      "fit_mcp": "docs_mcp_only",
      "fit_mcp_note": "Together AI documents a docs MCP server at `https://docs.together.ai/mcp` (server ids `together-docs` / `TogetherAIDocs`) that lets coding agents query Together's documentation; an MCP gateway/registry or tool proxying for inference traffic is not documented there. ([Together AI docs - agent skills / MCP server](https://docs.together.ai/docs/agent-skills))",
      "fit_meta": {
        "not_documented_fields": [
          "fit_cloud_iam",
          "fit_cloudflare_workers",
          "fit_credit_card",
          "fit_gateway_platform",
          "fit_model_gate",
          "fit_terraform"
        ]
      },
      "fit_model_gate": "not_documented",
      "fit_model_gate_note": "n.a. (not documented) ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))",
      "fit_onboarding_notes": "\"New API keys are shown only once; save the value safely.\" Optional `uv init --no-workspace` scaffolding is shown, and \"After a few seconds, you should see the response printed to your terminal.\" Python and TypeScript SDKs plus OpenAI-SDK and REST paths are offered ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))",
      "fit_openai_dropin": "yes",
      "fit_openai_dropin_note": "Yes. Together documents OpenAI SDK compatibility by changing the API key and setting the base URL to `https://api.together.ai/v1`; the rest of the code can remain the same. ([Together OpenAI compatibility docs](https://docs.together.ai/docs/inference/openai-compatibility))",
      "fit_prod_prereqs": "n.a. (not documented) ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart), [Pricing | Together AI](https://www.together.ai/pricing))",
      "fit_prod_selfhost_shape": null,
      "fit_prod_shape": "sdk_install",
      "fit_python_frameworks": [
        "LangChain",
        "LlamaIndex"
      ],
      "fit_python_frameworks_note": "Together documents integrations for LangChain using the `langchain-together` package and `ChatTogether`, and for LlamaIndex using `llama-index`, `OpenAILike`, and dedicated Together embedding classes. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))",
      "fit_quickstart_steps": 3,
      "fit_quickstart_steps_caveat": null,
      "fit_quickstart_url": "https://docs.together.ai/docs/quickstart",
      "fit_self_host_docs": "no_self_host",
      "fit_self_host_install": null,
      "fit_terraform": "not_documented",
      "fit_terraform_note": "n.a. (not documented)",
      "fit_terraform_ref": null,
      "fit_third_party_documented": {
        "kubernetes": "the Volcano project"
      },
      "fit_vendor_time_claim": "Vendor claim, verbatim: \"Make your first request to Together AI in a few minutes.\" ([Quickstart - Together AI docs](https://docs.together.ai/docs/quickstart))",
      "fit_vercel_ai_sdk": "official_provider",
      "fit_vercel_ai_sdk_note": "Together documents an official Vercel AI SDK provider package, `@ai-sdk/togetherai`, installable with `npm i ai @ai-sdk/togetherai`. ([Together SDK integrations](https://docs.together.ai/docs/inference/sdk-integrations))",
      "fit_vercel_ai_sdk_package": "@ai-sdk/togetherai"
    }
  ]
}