From 16e2cd8a74058815a66d4ef55553718c16fd3284 Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Fri, 19 Sep 2025 16:04:16 +0200 Subject: [PATCH 1/8] update: add autoscaling content for Kafka Diskless Topics --- .../diskless/howto/enable-autoscaling.md | 224 ++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 docs/products/diskless/howto/enable-autoscaling.md diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md new file mode 100644 index 000000000..33bbfa8d5 --- /dev/null +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -0,0 +1,224 @@ +--- +title: Enable autoscaling for Aiven for Apache Kafka® Diskless Topics +limited: true +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import ConsoleLabel from "@site/src/components/ConsoleIcons" +import {ConsoleIcon} from "@site/src/components/ConsoleIcons" +import RelatedPages from "@site/src/components/RelatedPages"; + +Autoscaling automatically adjusts the service capacity for your Aiven for Apache Kafka® deployment based on CPU usage. +It helps maintain performance during traffic spikes and reduces resource use when +demand is low. Autoscaling is available only for services using Diskless Topics in +Bring Your Own Cloud (BYOC) and must be enabled when the service is created. + +:::important +To use autoscaling, you must create a Diskless Topics service in BYOC with autoscaling +enabled. You cannot enable autoscaling on an existing service. +::: + +:::note[Availability] +Autoscaling for Diskless Topics is in **Limited Availability (LA)**. Contact +[Aiven support team](mailto:support@aiven.io) to request access. +::: + +## Why use autoscaling + +- **Maintain performance**: Services scale up during sustained high CPU load. +- **Control costs**: Services scale down when demand decreases. +- **Stay flexible**: Customers set minimum and maximum plans; scaling happens within + those limits. +- **Get notified**: You receive an email whenever a scaling event occurs. + +## How autoscaling works + +Autoscaling monitors CPU usage across your Kafka service and adjusts the service plan +based on predefined thresholds. It supports plan-based scaling only for services that +use Diskless Topics in BYOC. + +- **Scale up**: When CPU usage stays above 85% for several minutes. +- **Scale down**: When CPU usage stays below 40% for several minutes. +- **Plan switching**: The service scales by switching to the next larger or smaller + autoscaling plan. +- **Scaling in groups**: Nodes are added or removed in fixed groups (for example, + three nodes at a time), not individually. +- **Cooldown period**: After each scaling action, the autoscaler waits at least 10 + minutes before evaluating again. +- **Health check**: Scaling occurs only when all brokers are healthy and in the + **RUNNING** state. + +## Key considerations and limitations + +- You must enable autoscaling during service creation. You cannot enable it later. +- Autoscaling is available only for services that use Diskless Topics in BYOC. +- Manual plan changes are disabled when autoscaling is enabled. +- Billing is based on autoscaling plans (`autoscaling-*`). Each scaling event appears + as a separate line item in your invoice. + +## Prerequisites + +- An Aiven for Apache Kafka® service running with **Diskless Topics (BYOC)**. +- The service must be created with autoscaling enabled. +- Autoscaling must be enabled for your project by Aiven support. + You cannot convert an existing non-autoscaling service into autoscaling. +- Access to one of the following: + - [Aiven Console](https://console.aiven.io/) + - [Aiven API](https://api.aiven.io/doc/) + - [Aiven CLI client](/docs/tools/cli) + - [Terraform](https://registry.terraform.io/providers/aiven/aiven/latest) + +## Enable autoscaling + +To enable autoscaling for Kafka with Diskless Topics in BYOC, create an +autoscaler integration endpoint and link it to your service. + + + + +1. Log in to [Aiven Console](https://console.aiven.io/) and go to your project. +1. In the left sidebar, click . +1. Click **Aiven Service Autoscaler** > **Add new endpoint**. +1. Enter an endpoint name, for example `kafka-autoscaler`, and click **Add endpoint**. + +Integrate the autoscaler with a Kafka service: + +1. In the sidebar, click and open your Kafka + service. +1. Click . +1. Under **Endpoint integrations**, click **Aiven Service Autoscaler**. +1. Select the autoscaler endpoint you created. +1. Set the scaling limits: + - **Minimum plan:** The lowest plan your service is allowed to scale down to. + - **Maximum plan:** The highest plan your service is allowed to scale up to. +1. Click **Enable**. + + + + +Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): + +1. Create an autoscaler integration endpoint with `endpoint_type` set to `autoscaler_service`: + +```bash +curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "endpoint_name": "kafka-autoscaler", + "endpoint_type": "autoscaler_service" + }' + ``` + +1. Link your Kafka service to the new endpoint by calling `ServiceIntegrationCreate`: + +```bash +curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "source_service": "SERVICE_NAME", + "integration_type": "autoscaler_service", + "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", + "user_config": { + "autoscaling": { + "min_plan": "autoscaling-smallvm-3x", + "max_plan": "autoscaling-smallvm-6x" + } + } + }' +``` + +Parameters: + +- `source_service`: The Kafka service to autoscale. +- `dest_endpoint_id`: The ID of the autoscaler endpoint you created. +- `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. +- `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. + + + + + +Enable autoscaling with the [Aiven CLI](/docs/tools/cli): + +1. Create an autoscaler integration endpoint: + +```bash +avn service integration-endpoint-create \ + --project YOUR_PROJECT_NAME \ + --endpoint-name kafka-autoscaler \ + --endpoint-type autoscaler_service +``` + +1. Get the endpoint ID: + +```bash +avn service integration-endpoint-list --project YOUR_PROJECT_NAME +``` + +1. Link your Kafka service to the autoscaler endpoint: + +```bash +avn service integration-create \ + --project YOUR_PROJECT_NAME \ + --integration-type autoscaler_service \ + --dest-endpoint-id ENDPOINT_ID \ + --source-service YOUR_SERVICE_NAME \ + --user-config-json '{"autoscaling":{"min_plan":"autoscaling-smallvm-3x","max_plan":"autoscaling-smallvm-6x"}}' +``` + +Parameters: + +- `autoscaling.min_plan`: The smallest plan the service can scale down to. +- `autoscaling.max_plan`: The largest plan the service can scale up to. + + + + +Use the +[`aiven_service_integration_endpoint`](https://registry.terraform.io/providers/aiven/aiven/latest/docs/resources/service_integration_endpoint) +and +[`aiven_service_integration`](https://registry.terraform.io/providers/aiven/aiven/latest/docs/resources/service_integration) +resources to enable autoscaling. + +```hcl +resource "aiven_service_integration_endpoint" "autoscaler" { + project = var.project_name + endpoint_name = "kafka-autoscaler" + endpoint_type = "autoscaler_service" +} + +resource "aiven_service_integration" "autoscaling" { + project = var.project_name + integration_type = "autoscaler_service" + source_service_name = aiven_kafka.kafka.service_name + dest_endpoint_id = aiven_service_integration_endpoint.autoscaler.id + + user_config = { + autoscaling = { + min_plan = "autoscaling-smallvm-3x" + max_plan = "autoscaling-smallvm-6x" + } + } +} +``` + +Parameters: + +- `min_plan`: The smallest plan the service can scale down to. +- `max_plan`: The largest plan the service can scale up to. + +:::note[Terraform limitation] +With the current Terraform design, you must first create your Kafka service with a plan. +After the service exists, create the autoscaler endpoint and integration in a +separate `terraform apply` step. + +Defining the autoscaler directly in the `aiven_kafka` resource is under discussion and +may be supported in future versions of the Terraform provider. +::: + + From ed2f10f1ea4f74ef7259b154b9de8d8d1bf3d104 Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Fri, 19 Sep 2025 16:15:56 +0200 Subject: [PATCH 2/8] update: content and TOC --- .../config/vocabularies/Aiven/accept.txt | 2 + .../diskless/howto/enable-autoscaling.md | 112 +++++++++--------- sidebars.ts | 5 +- 3 files changed, 63 insertions(+), 56 deletions(-) diff --git a/.github/vale/styles/config/vocabularies/Aiven/accept.txt b/.github/vale/styles/config/vocabularies/Aiven/accept.txt index cd8f39d28..5fe0918fb 100644 --- a/.github/vale/styles/config/vocabularies/Aiven/accept.txt +++ b/.github/vale/styles/config/vocabularies/Aiven/accept.txt @@ -23,6 +23,7 @@ Astacus australia Auth0 autojoin +autoscale autoscaler autoscaling autovacuum @@ -54,6 +55,7 @@ config Config Confluent connect +Cooldown Coralogix Couchbase CPU/MS diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index 33bbfa8d5..2e8c7ce8e 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -1,5 +1,6 @@ --- title: Enable autoscaling for Aiven for Apache Kafka® Diskless Topics +sidebar_label: Enable autoscaling limited: true --- @@ -67,7 +68,7 @@ use Diskless Topics in BYOC. - [Aiven Console](https://console.aiven.io/) - [Aiven API](https://api.aiven.io/doc/) - [Aiven CLI client](/docs/tools/cli) - - [Terraform](https://registry.terraform.io/providers/aiven/aiven/latest) + - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest) ## Enable autoscaling @@ -101,43 +102,43 @@ Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): 1. Create an autoscaler integration endpoint with `endpoint_type` set to `autoscaler_service`: -```bash -curl --request POST \ - --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ - --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ - --header 'content-type: application/json' \ - --data '{ - "endpoint_name": "kafka-autoscaler", - "endpoint_type": "autoscaler_service" - }' - ``` + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "endpoint_name": "kafka-autoscaler", + "endpoint_type": "autoscaler_service" + }' + ``` 1. Link your Kafka service to the new endpoint by calling `ServiceIntegrationCreate`: -```bash -curl --request POST \ - --url https://api.aiven.io/v1/project/{project_name}/integration \ - --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ - --header 'content-type: application/json' \ - --data '{ - "source_service": "SERVICE_NAME", - "integration_type": "autoscaler_service", - "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", - "user_config": { - "autoscaling": { - "min_plan": "autoscaling-smallvm-3x", - "max_plan": "autoscaling-smallvm-6x" - } - } - }' -``` - -Parameters: - -- `source_service`: The Kafka service to autoscale. -- `dest_endpoint_id`: The ID of the autoscaler endpoint you created. -- `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. -- `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "source_service": "SERVICE_NAME", + "integration_type": "autoscaler_service", + "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", + "user_config": { + "autoscaling": { + "min_plan": "autoscaling-smallvm-3x", + "max_plan": "autoscaling-smallvm-6x" + } + } + }' + ``` + + Parameters: + + - `source_service`: The Kafka service to autoscale. + - `dest_endpoint_id`: The ID of the autoscaler endpoint you created. + - `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. + - `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. @@ -147,34 +148,34 @@ Enable autoscaling with the [Aiven CLI](/docs/tools/cli): 1. Create an autoscaler integration endpoint: -```bash -avn service integration-endpoint-create \ - --project YOUR_PROJECT_NAME \ - --endpoint-name kafka-autoscaler \ - --endpoint-type autoscaler_service -``` + ```bash + avn service integration-endpoint-create \ + --project YOUR_PROJECT_NAME \ + --endpoint-name kafka-autoscaler \ + --endpoint-type autoscaler_service + ``` 1. Get the endpoint ID: -```bash -avn service integration-endpoint-list --project YOUR_PROJECT_NAME -``` + ```bash + avn service integration-endpoint-list --project YOUR_PROJECT_NAME + ``` 1. Link your Kafka service to the autoscaler endpoint: -```bash -avn service integration-create \ - --project YOUR_PROJECT_NAME \ - --integration-type autoscaler_service \ - --dest-endpoint-id ENDPOINT_ID \ - --source-service YOUR_SERVICE_NAME \ - --user-config-json '{"autoscaling":{"min_plan":"autoscaling-smallvm-3x","max_plan":"autoscaling-smallvm-6x"}}' -``` + ```bash + avn service integration-create \ + --project YOUR_PROJECT_NAME \ + --integration-type autoscaler_service \ + --dest-endpoint-id ENDPOINT_ID \ + --source-service YOUR_SERVICE_NAME \ + --user-config-json '{"autoscaling":{"min_plan":"autoscaling-smallvm-3x","max_plan":"autoscaling-smallvm-6x"}}' + ``` -Parameters: + Parameters: -- `autoscaling.min_plan`: The smallest plan the service can scale down to. -- `autoscaling.max_plan`: The largest plan the service can scale up to. + - `autoscaling.min_plan`: The smallest plan the service can scale down to. + - `autoscaling.max_plan`: The largest plan the service can scale up to. @@ -222,3 +223,4 @@ may be supported in future versions of the Terraform provider. ::: + diff --git a/sidebars.ts b/sidebars.ts index 999faeba6..f9fb7f3a7 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -1233,7 +1233,10 @@ const sidebars: SidebarsConfig = { { type: 'category', label: 'How to', - items: ['products/diskless/howto/create-diskless-topic'], + items: [ + 'products/diskless/howto/create-diskless-topic', + 'products/diskless/howto/enable-autoscaling', + ], }, ], }, From 511f143433b69668c8a4100b7a07f2d68f8578ac Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Mon, 22 Sep 2025 10:56:47 +0200 Subject: [PATCH 3/8] update: content based on feedback --- .../diskless/howto/enable-autoscaling.md | 172 ++++++++++-------- 1 file changed, 100 insertions(+), 72 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index 2e8c7ce8e..ffc77aca9 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -10,7 +10,7 @@ import ConsoleLabel from "@site/src/components/ConsoleIcons" import {ConsoleIcon} from "@site/src/components/ConsoleIcons" import RelatedPages from "@site/src/components/RelatedPages"; -Autoscaling automatically adjusts the service capacity for your Aiven for Apache Kafka® deployment based on CPU usage. +Autoscaling automatically adjusts the capacity of your Aiven for Apache Kafka® service based on CPU usage. It helps maintain performance during traffic spikes and reduces resource use when demand is low. Autoscaling is available only for services using Diskless Topics in Bring Your Own Cloud (BYOC) and must be enabled when the service is created. @@ -31,7 +31,8 @@ Autoscaling for Diskless Topics is in **Limited Availability (LA)**. Contact - **Control costs**: Services scale down when demand decreases. - **Stay flexible**: Customers set minimum and maximum plans; scaling happens within those limits. -- **Get notified**: You receive an email whenever a scaling event occurs. +- **Get notified**: You receive an email notification whenever autoscaling scales the + service up or down. ## How autoscaling works @@ -39,8 +40,8 @@ Autoscaling monitors CPU usage across your Kafka service and adjusts the service based on predefined thresholds. It supports plan-based scaling only for services that use Diskless Topics in BYOC. -- **Scale up**: When CPU usage stays above 85% for several minutes. -- **Scale down**: When CPU usage stays below 40% for several minutes. +- **Scale up**: When CPU usage is consistently high. +- **Scale down**: When CPU usage is consistently low. - **Plan switching**: The service scales by switching to the next larger or smaller autoscaling plan. - **Scaling in groups**: Nodes are added or removed in fixed groups (for example, @@ -54,7 +55,8 @@ use Diskless Topics in BYOC. - You must enable autoscaling during service creation. You cannot enable it later. - Autoscaling is available only for services that use Diskless Topics in BYOC. -- Manual plan changes are disabled when autoscaling is enabled. +- When autoscaling is enabled, you cannot change the service plan manually. To change + the plan, first disable autoscaling. - Billing is based on autoscaling plans (`autoscaling-*`). Each scaling event appears as a separate line item in your invoice. @@ -65,83 +67,21 @@ use Diskless Topics in BYOC. - Autoscaling must be enabled for your project by Aiven support. You cannot convert an existing non-autoscaling service into autoscaling. - Access to one of the following: - - [Aiven Console](https://console.aiven.io/) - [Aiven API](https://api.aiven.io/doc/) - [Aiven CLI client](/docs/tools/cli) - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest) +:::note +Autoscaling is not yet available in the Aiven Console. Use the CLI, Terraform, or API to +enable or disable autoscaling. +::: + ## Enable autoscaling To enable autoscaling for Kafka with Diskless Topics in BYOC, create an autoscaler integration endpoint and link it to your service. - - -1. Log in to [Aiven Console](https://console.aiven.io/) and go to your project. -1. In the left sidebar, click . -1. Click **Aiven Service Autoscaler** > **Add new endpoint**. -1. Enter an endpoint name, for example `kafka-autoscaler`, and click **Add endpoint**. - -Integrate the autoscaler with a Kafka service: - -1. In the sidebar, click and open your Kafka - service. -1. Click . -1. Under **Endpoint integrations**, click **Aiven Service Autoscaler**. -1. Select the autoscaler endpoint you created. -1. Set the scaling limits: - - **Minimum plan:** The lowest plan your service is allowed to scale down to. - - **Maximum plan:** The highest plan your service is allowed to scale up to. -1. Click **Enable**. - - - - -Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): - -1. Create an autoscaler integration endpoint with `endpoint_type` set to `autoscaler_service`: - - ```bash - curl --request POST \ - --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ - --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ - --header 'content-type: application/json' \ - --data '{ - "endpoint_name": "kafka-autoscaler", - "endpoint_type": "autoscaler_service" - }' - ``` - -1. Link your Kafka service to the new endpoint by calling `ServiceIntegrationCreate`: - - ```bash - curl --request POST \ - --url https://api.aiven.io/v1/project/{project_name}/integration \ - --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ - --header 'content-type: application/json' \ - --data '{ - "source_service": "SERVICE_NAME", - "integration_type": "autoscaler_service", - "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", - "user_config": { - "autoscaling": { - "min_plan": "autoscaling-smallvm-3x", - "max_plan": "autoscaling-smallvm-6x" - } - } - }' - ``` - - Parameters: - - - `source_service`: The Kafka service to autoscale. - - `dest_endpoint_id`: The ID of the autoscaler endpoint you created. - - `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. - - `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. - - - Enable autoscaling with the [Aiven CLI](/docs/tools/cli): @@ -222,5 +162,93 @@ Defining the autoscaler directly in the `aiven_kafka` resource is under discussi may be supported in future versions of the Terraform provider. ::: + + + +Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): + +1. Create an autoscaler integration endpoint with `endpoint_type` set to `autoscaler_service`: + + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration_endpoint \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "endpoint_name": "kafka-autoscaler", + "endpoint_type": "autoscaler_service" + }' + ``` + +1. Link your Kafka service to the new endpoint by calling `ServiceIntegrationCreate`: + + ```bash + curl --request POST \ + --url https://api.aiven.io/v1/project/{project_name}/integration \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ + --header 'content-type: application/json' \ + --data '{ + "source_service": "SERVICE_NAME", + "integration_type": "autoscaler_service", + "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", + "user_config": { + "autoscaling": { + "min_plan": "autoscaling-smallvm-3x", + "max_plan": "autoscaling-smallvm-6x" + } + } + }' + ``` + + Parameters: + + - `source_service`: The Kafka service to autoscale. + - `dest_endpoint_id`: The ID of the autoscaler endpoint you created. + - `user_config.autoscaling.min_plan`: The smallest plan the service can scale down to. + - `user_config.autoscaling.max_plan`: The largest plan the service can scale up to. + + + + +## Disable autoscaling + +To disable autoscaling for Kafka with Diskless Topics in BYOC, remove the autoscaler +integration from your service. + + + + +List integrations to find the autoscaler integration ID: + +```bash +avn service integration-list --project PROJECT_NAME SERVICE_NAME +``` + +Then delete it: + +```bash +avn service integration-delete --project PROJECT_NAME INTEGRATION_ID +``` + + + + +Remove the autoscaler integration resource from your Terraform configuration and run: + +```bash +terraform apply +``` + + + + +Delete the service integration that links your Kafka service to the autoscaler endpoint: + +```bash +curl --request DELETE \ + --url https://api.aiven.io/v1/project/{project_name}/integration/{integration_id} \ + --header 'Authorization: Bearer REPLACE_WITH_TOKEN' +``` + From a63507b0943a7a3320e4df4ab1a48a4399796ad2 Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Mon, 22 Sep 2025 16:05:38 +0200 Subject: [PATCH 4/8] update: links --- docs/products/diskless/howto/enable-autoscaling.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index ffc77aca9..7bb4d8ab1 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -13,11 +13,12 @@ import RelatedPages from "@site/src/components/RelatedPages"; Autoscaling automatically adjusts the capacity of your Aiven for Apache Kafka® service based on CPU usage. It helps maintain performance during traffic spikes and reduces resource use when demand is low. Autoscaling is available only for services using Diskless Topics in -Bring Your Own Cloud (BYOC) and must be enabled when the service is created. +Bring Your Own Cloud (BYOC). :::important -To use autoscaling, you must create a Diskless Topics service in BYOC with autoscaling -enabled. You cannot enable autoscaling on an existing service. +Autoscaling must be enabled when you +[create a Diskless Topics service in BYOC](/docs/products/diskless/howto/create-diskless-topic). +You cannot enable it later on an existing service. ::: :::note[Availability] From 7d2590384277375781d0394ce8367dee36c7ac5d Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Tue, 23 Sep 2025 08:49:54 +0200 Subject: [PATCH 5/8] update: Feedback --- docs/products/diskless/howto/enable-autoscaling.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index 7bb4d8ab1..2118908d1 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -30,7 +30,7 @@ Autoscaling for Diskless Topics is in **Limited Availability (LA)**. Contact - **Maintain performance**: Services scale up during sustained high CPU load. - **Control costs**: Services scale down when demand decreases. -- **Stay flexible**: Customers set minimum and maximum plans; scaling happens within +- **Stay flexible**: You can set minimum and maximum plans; scaling happens within those limits. - **Get notified**: You receive an email notification whenever autoscaling scales the service up or down. @@ -54,8 +54,6 @@ use Diskless Topics in BYOC. ## Key considerations and limitations -- You must enable autoscaling during service creation. You cannot enable it later. -- Autoscaling is available only for services that use Diskless Topics in BYOC. - When autoscaling is enabled, you cannot change the service plan manually. To change the plan, first disable autoscaling. - Billing is based on autoscaling plans (`autoscaling-*`). Each scaling event appears @@ -158,9 +156,6 @@ Parameters: With the current Terraform design, you must first create your Kafka service with a plan. After the service exists, create the autoscaler endpoint and integration in a separate `terraform apply` step. - -Defining the autoscaler directly in the `aiven_kafka` resource is under discussion and -may be supported in future versions of the Terraform provider. ::: @@ -253,3 +248,7 @@ curl --request DELETE \ + + + +- [Get started with Diskless Topics](/docs/products/diskless/get-started) From c5ce9c64edce861b3a9f2f0b3557fd1a1c9204a3 Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Tue, 23 Sep 2025 11:30:49 +0200 Subject: [PATCH 6/8] update: content --- .../diskless/howto/enable-autoscaling.md | 84 +++++++++---------- 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index 2118908d1..b94c61e8f 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -10,27 +10,17 @@ import ConsoleLabel from "@site/src/components/ConsoleIcons" import {ConsoleIcon} from "@site/src/components/ConsoleIcons" import RelatedPages from "@site/src/components/RelatedPages"; -Autoscaling automatically adjusts the capacity of your Aiven for Apache Kafka® service based on CPU usage. +Autoscaling adjusts the capacity of your Aiven for Apache Kafka® service based on CPU usage. It helps maintain performance during traffic spikes and reduces resource use when -demand is low. Autoscaling is available only for services using Diskless Topics in -Bring Your Own Cloud (BYOC). - -:::important -Autoscaling must be enabled when you -[create a Diskless Topics service in BYOC](/docs/products/diskless/howto/create-diskless-topic). -You cannot enable it later on an existing service. -::: - -:::note[Availability] -Autoscaling for Diskless Topics is in **Limited Availability (LA)**. Contact -[Aiven support team](mailto:support@aiven.io) to request access. -::: +demand is low. Autoscaling is available only for Diskless Topics services deployed in +Bring Your Own Cloud (BYOC) using a plan that is part of the supported autoscaling +plan set. ## Why use autoscaling - **Maintain performance**: Services scale up during sustained high CPU load. - **Control costs**: Services scale down when demand decreases. -- **Stay flexible**: You can set minimum and maximum plans; scaling happens within +- **Stay flexible**: You can set minimum and maximum plans. Scaling happens within those limits. - **Get notified**: You receive an email notification whenever autoscaling scales the service up or down. @@ -47,38 +37,42 @@ use Diskless Topics in BYOC. autoscaling plan. - **Scaling in groups**: Nodes are added or removed in fixed groups (for example, three nodes at a time), not individually. -- **Cooldown period**: After each scaling action, the autoscaler waits at least 10 - minutes before evaluating again. +- **Cooldown period**: The autoscaler waits at least 10 minutes after each scaling action + before evaluating the service again. - **Health check**: Scaling occurs only when all brokers are healthy and in the **RUNNING** state. ## Key considerations and limitations -- When autoscaling is enabled, you cannot change the service plan manually. To change - the plan, first disable autoscaling. -- Billing is based on autoscaling plans (`autoscaling-*`). Each scaling event appears - as a separate line item in your invoice. +- **Service compatibility**: Autoscaling is supported only for Diskless Topics services + in BYOC that are created with a supported autoscaling plan. If the service is not + using a plan from the autoscaling plan set, autoscaling is not available. +- **Limited availability**: Contact the [Aiven support team](mailto:support@aiven.io) to + request access and enable autoscaling for your project. +- **Enable only at service creation**: You must enable autoscaling when you + [create a Diskless Topics service in BYOC](/docs/products/diskless/howto/create-diskless-topic). + You cannot enable autoscaling for existing services. +- **Aiven Console limitation**: Autoscaling is not available in the Aiven Console. Use + the CLI, Terraform, or API. +- **Plan changes not allowed with autoscaling**: You cannot manually change the service + plan while autoscaling is enabled. To change the plan, first disable autoscaling. +- **Billing**: Autoscaling uses special autoscaling plans. Each scaling event is billed + separately and appears as a line item in your invoice. ## Prerequisites -- An Aiven for Apache Kafka® service running with **Diskless Topics (BYOC)**. -- The service must be created with autoscaling enabled. -- Autoscaling must be enabled for your project by Aiven support. - You cannot convert an existing non-autoscaling service into autoscaling. +- An Aiven for Apache Kafka® service using Diskless Topics (BYOC) +- Autoscaling enabled when the service was created +- Autoscaling access granted for your project by Aiven support - Access to one of the following: - [Aiven API](https://api.aiven.io/doc/) - [Aiven CLI client](/docs/tools/cli) - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest) -:::note -Autoscaling is not yet available in the Aiven Console. Use the CLI, Terraform, or API to -enable or disable autoscaling. -::: - ## Enable autoscaling -To enable autoscaling for Kafka with Diskless Topics in BYOC, create an -autoscaler integration endpoint and link it to your service. +To enable autoscaling, create an autoscaler integration endpoint and link it to your +Kafka service with Diskless Topics in BYOC. @@ -108,7 +102,7 @@ Enable autoscaling with the [Aiven CLI](/docs/tools/cli): --integration-type autoscaler_service \ --dest-endpoint-id ENDPOINT_ID \ --source-service YOUR_SERVICE_NAME \ - --user-config-json '{"autoscaling":{"min_plan":"autoscaling-smallvm-3x","max_plan":"autoscaling-smallvm-6x"}}' + --user-config-json '{"autoscaling":{"min_plan":"","max_plan":""}}' ``` Parameters: @@ -140,8 +134,8 @@ resource "aiven_service_integration" "autoscaling" { user_config = { autoscaling = { - min_plan = "autoscaling-smallvm-3x" - max_plan = "autoscaling-smallvm-6x" + min_plan = "" + max_plan = "" } } } @@ -184,16 +178,16 @@ Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): --header 'Authorization: Bearer REPLACE_WITH_TOKEN' \ --header 'content-type: application/json' \ --data '{ - "source_service": "SERVICE_NAME", - "integration_type": "autoscaler_service", - "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", - "user_config": { - "autoscaling": { - "min_plan": "autoscaling-smallvm-3x", - "max_plan": "autoscaling-smallvm-6x" + "source_service": "SERVICE_NAME", + "integration_type": "autoscaler_service", + "dest_endpoint_id": "NEW_AUTOSCALER_ENDPOINT_ID", + "user_config": { + "autoscaling": { + "min_plan": "", + "max_plan": "" } } - }' + }' ``` Parameters: @@ -208,8 +202,8 @@ Enable autoscaling with the [Aiven API](https://api.aiven.io/doc/): ## Disable autoscaling -To disable autoscaling for Kafka with Diskless Topics in BYOC, remove the autoscaler -integration from your service. +To disable autoscaling, remove the autoscaler integration from your Kafka service +with Diskless Topics in BYOC. From 186ae06c731691aece86493079798a9e2086a3be Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Mon, 29 Sep 2025 14:44:25 +0200 Subject: [PATCH 7/8] update: content --- docs/products/diskless/howto/enable-autoscaling.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index b94c61e8f..b574f98c2 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -223,11 +223,9 @@ avn service integration-delete --project PROJECT_NAME INTEGRATION_ID -Remove the autoscaler integration resource from your Terraform configuration and run: - -```bash -terraform apply -``` +1. Open your Terraform configuration. +1. Remove the `aiven_service_integration` resource for the autoscaler. +1. Apply the changes. From e160a65e2f430151a08292e65b44db82796b53c8 Mon Sep 17 00:00:00 2001 From: Harshini Rangaswamy Date: Wed, 1 Oct 2025 11:23:32 +0200 Subject: [PATCH 8/8] update: content --- .../diskless/howto/enable-autoscaling.md | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/products/diskless/howto/enable-autoscaling.md b/docs/products/diskless/howto/enable-autoscaling.md index b574f98c2..02c4f4c18 100644 --- a/docs/products/diskless/howto/enable-autoscaling.md +++ b/docs/products/diskless/howto/enable-autoscaling.md @@ -44,9 +44,10 @@ use Diskless Topics in BYOC. ## Key considerations and limitations -- **Service compatibility**: Autoscaling is supported only for Diskless Topics services - in BYOC that are created with a supported autoscaling plan. If the service is not - using a plan from the autoscaling plan set, autoscaling is not available. +- **Service compatibility**: Autoscaling is only available for Kafka services using + Diskless Topics in BYOC that are created with a supported autoscaling plan. Only + specific plans are eligible for autoscaling. You must select one of these supported + plans when creating the service. - **Limited availability**: Contact the [Aiven support team](mailto:support@aiven.io) to request access and enable autoscaling for your project. - **Enable only at service creation**: You must enable autoscaling when you @@ -54,20 +55,40 @@ use Diskless Topics in BYOC. You cannot enable autoscaling for existing services. - **Aiven Console limitation**: Autoscaling is not available in the Aiven Console. Use the CLI, Terraform, or API. -- **Plan changes not allowed with autoscaling**: You cannot manually change the service - plan while autoscaling is enabled. To change the plan, first disable autoscaling. -- **Billing**: Autoscaling uses special autoscaling plans. Each scaling event is billed - separately and appears as a line item in your invoice. +- **Plan changes**: While autoscaling is enabled, you cannot change the service plan + manually. To move the service to another plan (for example, `business-*`), first + disable autoscaling. To re-enable autoscaling, the service must be on an autoscaling + plan. +- **Thresholds**: Autoscaling is based on predefined CPU thresholds. You cannot configure + these values. The only configurable limits are `min_plan` and `max_plan`. +- **Billing**: Autoscaling uses special autoscaling plans. When a service scales, you + are billed according to the new plan. Each plan change appears as a separate line item + on your invoice. ## Prerequisites -- An Aiven for Apache Kafka® service using Diskless Topics (BYOC) -- Autoscaling enabled when the service was created - Autoscaling access granted for your project by Aiven support - Access to one of the following: - [Aiven API](https://api.aiven.io/doc/) - [Aiven CLI client](/docs/tools/cli) - [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest) +- An Aiven for Apache Kafka® Diskless Topics (BYOC) service created with an autoscaling + plan. For example: + + - `` + - `` + + Example using the CLI: + + ```bash + avn service create kafka-autoscale-demo \ + --service-type kafka \ + --cloud google-europe-west3 \ + --plan + ``` + + This creates a Kafka service named `kafka-autoscale-demo` in the `google-europe-west3` + region using the `` plan. ## Enable autoscaling