diff --git a/docs-data/property-overrides.json b/docs-data/property-overrides.json index 43744c347b..5ece5d67e0 100644 --- a/docs-data/property-overrides.json +++ b/docs-data/property-overrides.json @@ -468,12 +468,29 @@ "description": "Configure the addressing style that controls how Redpanda formats bucket URLs for S3-compatible object storage.\n\nLeave this property unset (`null`) to use automatic configuration:\n\n* For AWS S3: Redpanda attempts `virtual_host` addressing first, then falls back to `path` style if needed\n* For MinIO: Redpanda automatically uses `path` style regardless of `MINIO_DOMAIN` configuration\n\nSet this property explicitly to override automatic configuration, ensure consistent behavior across deployments, or when using S3-compatible storage that requires a specific URL format.\n\nCAUTION: AWS requires virtual-hosted addressing for buckets created after September 30, 2020. If you use AWS S3 with buckets created after this date, use `virtual_host` addressing.\n\nNOTE: For MinIO deployments, Redpanda defaults to `path` style when this property is unset. To use `virtual_host` addressing with a configured `MINIO_DOMAIN`, set this property explicitly to `virtual_host`. For other S3-compatible storage backends, consult your provider's documentation to determine the required URL style.", "config_scope": "cluster" }, + "cloud_topics_compaction_interval_ms": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_compaction_key_map_memory": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_compaction_max_object_size": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_disable_level_zero_gc_for_tests": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_disable_metastore_flush_loop_for_tests": { + "version": "v26.1.1-rc2" + }, "cloud_topics_disable_reconciliation_loop": { - "exclude_from_docs": true, "config_scope": "cluster" }, "cloud_topics_enabled": { - "exclude_from_docs": true, + "description": "Enable Cloud Topics for the cluster. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics.", + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]" + ], "config_scope": "cluster" }, "cloud_topics_epoch_service_epoch_increment_interval": { @@ -484,26 +501,68 @@ "description": "The duration, in milliseconds, for which a cluster-wide epoch is cached locally on each broker.\n\nCaching the epoch locally reduces the need for frequent coordination with the controller. This property controls how long each broker can use a cached epoch value before fetching the latest value.\n\nIncrease this value to reduce coordination overhead in clusters with stable workloads. Decrease it if you need brokers to react more quickly to epoch changes in Tiered Storage.", "version": "v25.3.3" }, + "cloud_topics_fetch_debounce_enabled": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_l1_indexing_interval": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_long_term_flush_interval": { + "version": "v26.1.1-rc2" + }, "cloud_topics_long_term_garbage_collection_interval": { - "exclude_from_docs": true, "config_scope": "cluster" }, + "cloud_topics_parallel_fetch_enabled": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_preregistered_object_ttl": { + "version": "v26.1.1-rc2" + }, "cloud_topics_produce_batching_size_threshold": { - "exclude_from_docs": true, "config_scope": "cluster" }, "cloud_topics_produce_cardinality_threshold": { - "exclude_from_docs": true, "config_scope": "cluster" }, "cloud_topics_produce_upload_interval": { - "exclude_from_docs": true, + "description": "Time interval after which data is uploaded to object storage for Cloud Topics. When this time threshold is reached, Redpanda triggers an upload of buffered data to the object storage backend (S3, GCS, or MinIO), regardless of whether the size or cardinality thresholds have been met.\n\nThis property works together with <> and <> to control when uploads occur. An upload is triggered when any of these three thresholds is reached.", + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_batching_size_threshold[`cloud_topics_produce_batching_size_threshold`]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_cardinality_threshold[`cloud_topics_produce_cardinality_threshold`]" + ], "config_scope": "cluster" }, "cloud_topics_reconciliation_interval": { - "exclude_from_docs": true, + "description": "Time interval at which Redpanda reconciles data between short-term local storage and long-term object storage for Cloud Topics. During this reconciliation process, Redpanda optimizes the storage layout of data in short-term storage to improve the cost and performance associated with accessing data. After the reconciliation process has moved data into long-term storage, the data in short-term storage is subject to removal by a garbage collection process.", + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_garbage_collection_interval[`cloud_topics_long_term_garbage_collection_interval`]" + ], "config_scope": "cluster" }, + "cloud_topics_reconciliation_max_interval": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_max_object_size": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_min_interval": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_parallelism": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_slowdown_blend": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_speedup_blend": { + "version": "v26.1.1-rc2" + }, + "cloud_topics_reconciliation_target_fill_ratio": { + "version": "v26.1.1-rc2" + }, "cloud_topics_short_term_gc_backoff_interval": { "description": "The interval, in milliseconds, between invocations of the L0 garbage collection work loop when no progress is being made or errors are occurring.\n\nL0 (level-zero) objects are short-term data objects in Tiered Storage that are periodically garbage collected. When GC encounters errors or cannot make progress (for example, if there are no objects eligible for deletion), this backoff interval prevents excessive retries.\n\nIncrease this value to reduce system load when GC cannot make progress. Decrease it if you need faster retry attempts after transient errors.", "version": "v25.3.3" @@ -516,6 +575,9 @@ "description": "The minimum age, in milliseconds, of an L0 (level-zero) object before it becomes eligible for garbage collection.\n\nThis grace period delays deletion of L0 objects even after they become eligible based on epoch. The delay provides a safety buffer that can support recovery in cases involving accidental deletion or other operational issues.\n\nIncrease this value to extend the retention window for L0 objects, providing more time for recovery from operational errors. Decrease it to free up object storage space more quickly, but with less protection against accidental deletion.", "version": "v25.3.3" }, + "cloud_topics_upload_part_size": { + "version": "v26.1.1-rc2" + }, "cluster_id": { "description": "Cluster identifier.", "config_scope": "cluster" @@ -689,6 +751,9 @@ ], "config_scope": "cluster" }, + "default_redpanda_storage_mode": { + "version": "v26.1.1-rc2" + }, "delete.retention.ms": { "description": "The retention time for tombstone records in a compacted topic. Redpanda removes tombstone records after the retention limit is exceeded.\n\nIf you have enabled Tiered Storage and set <> or <> for the topic, you cannot enable tombstone removal.\n\nIf both `delete.retention.ms` and the cluster property config_ref:tombstone_retention_ms,true,properties/cluster-properties[] are set, `delete.retention.ms` overrides the cluster level tombstone retention for an individual topic.\n\nThis property supports three states:\n\n* Positive value: Sets the milliseconds to retain tombstone records before removal.\n* 0: Tombstone records are immediately eligible for removal.\n* Negative value: Disables tombstone removal entirely for this topic.", "related_topics": [ @@ -697,6 +762,9 @@ ], "config_scope": "topic" }, + "delete_topic_enable": { + "version": "v26.1.1-rc2" + }, "developer_mode": { "description": "CAUTION: Enabling `developer_mode` isn't recommended for production use.\n\nEnable developer mode, which skips most of the checks performed at startup.", "config_scope": "broker", @@ -1008,6 +1076,9 @@ ], "config_scope": "cluster" }, + "internal_rpc_request_timeout_ms": { + "version": "v26.1.1-rc2" + }, "internal_topic_replication_factor": { "description": "Target replication factor for internal topics.\n\n*Unit*: number of replicas per topic.", "config_scope": "cluster" @@ -1215,6 +1286,9 @@ "description": "Prevents log compaction from removing transaction metadata. Only set this to `true` if you experience stability issues related to transaction cleanup during compaction.", "config_scope": "cluster" }, + "log_compaction_max_priority_wait_ms": { + "version": "v26.1.1-rc2" + }, "log_compaction_merge_max_ranges": { "description": "The maximum range of segments that can be processed in a single round of adjacent segment compaction. If `null` (the default value), no maximum is imposed on the number of ranges that can be processed at once. A value below 1 effectively disables adjacent merge compaction.", "config_scope": "cluster" @@ -1378,6 +1452,9 @@ "config_scope": "broker", "category": "schema-registry" }, + "nested_group_behavior": { + "version": "v26.1.1-rc2" + }, "node_id": { "config_scope": "broker", "category": "redpanda", @@ -1422,6 +1499,9 @@ "description": "The URL pointing to the well-known discovery endpoint for the OIDC provider.", "config_scope": "cluster" }, + "oidc_group_claim_path": { + "version": "v26.1.1-rc2" + }, "oidc_principal_mapping": { "description": "Rule for mapping JWT payload claim to a Redpanda user principal.", "related_topics": [ @@ -1494,6 +1574,9 @@ "description": "Controls when and how Redpanda automatically rebalances partition replicas across brokers. For more information, see xref:manage:cluster-maintenance/cluster-balancing.adoc[partition balancing].\n\nValues:\n\n* `continuous`: Partition balancing happens automatically to maintain optimal performance and availability, based on continuous monitoring for node changes (same as `node_add`) and also high disk usage. This option requires an enterprise license, and it is customized by xref:reference:properties/cluster-properties.adoc#partition_autobalancing_node_availability_timeout_sec[`partition_autobalancing_node_availability_timeout_sec`] and xref:reference:properties/cluster-properties.adoc#partition_autobalancing_max_disk_usage_percent[`partition_autobalancing_max_disk_usage_percent`] properties.\n* `node_add`: Partition balancing happens when a node is added.\n* `off`: Partition balancing is disabled. This option is not recommended for production clusters.", "config_scope": "cluster" }, + "partition_autobalancing_node_autodecommission_timeout_sec": { + "version": "v26.1.1-rc2" + }, "partition_autobalancing_node_availability_timeout_sec": { "related_topics": [ "xref:manage:cluster-maintenance/continuous-data-balancing.adoc[Configure Continuous Data Balancing]" @@ -1676,6 +1759,13 @@ ], "config_scope": "topic" }, + "redpanda.storage.mode": { + "description": "The storage mode for a topic. Determines how topic data is stored and whether it is eligible for upload to object storage.\n\nAccepted values:\n\n* `local`: Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of cluster-level Tiered Storage settings.\n* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables xref:manage:tiered-storage.adoc[Tiered Storage] for the topic.\n* `cloud`: Topic data is stored in object storage using the Cloud Topics architecture. Local storage is used only as a write buffer.\n* `unset`: Uses the cluster-level config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting, or falls back to legacy `redpanda.remote.read` and `redpanda.remote.write` topic property behavior for backwards compatibility.\n\nThis property overrides the cluster-wide config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting for individual topics.", + "related_topics": [ + "xref:manage:tiered-storage.adoc[Tiered Storage]" + ], + "config_scope": "topic" + }, "redpanda.value.schema.id.validation": { "description": "Enable validation of the schema ID for values on a record. When enabled, Redpanda validates that the schema ID encoded in the record's value is registered in the Schema Registry according to the configured subject name strategy.", "related_topics": [ @@ -1709,8 +1799,8 @@ "xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`]", "xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`]", - "xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", - "xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor]", + "xref:develop:manage-topics.adoc#choose-the-replication-factor[Choose the replication factor]", + "xref:develop:manage-topics.adoc#change-the-replication-factor[Change the replication factor]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication]" ], "config_scope": "topic" @@ -1864,11 +1954,17 @@ "config_scope": "broker", "category": "schema-registry" }, + "schema_registry_avro_use_named_references": { + "version": "v26.1.1-rc2" + }, "schema_registry_enable_authorization": { "description": "Enables ACL-based authorization for Schema Registry requests. When `true`, Schema Registry\nuses ACL-based authorization instead of the default `public/user/superuser` authorization model.", "related_topics": [], "config_scope": "cluster" }, + "schema_registry_enable_qualified_subjects": { + "version": "v26.1.1-rc2" + }, "schema_registry_replication_factor": { "description": "Replication factor for internal `_schemas` topic. If unset, defaults to the xref:../cluster-properties.adoc#default_topic_replication[`default_topic_replication`] cluster property.", "related_topics": [ @@ -2091,7 +2187,7 @@ "write.caching": { "description": "The write caching mode to apply to a topic.\n\nWhen `write.caching` is set, it overrides the cluster property xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. Fsyncs follow <> and <>, whichever is reached first.", "related_topics": [ - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]", + "xref:develop:manage-topics.adoc#configure-write-caching[Write caching]", "xref:manage:tiered-storage.adoc[Tiered Storage]", "xref:reference:properties/cluster-properties.adoc#write_caching_default[`write_caching_default`]", "xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]" @@ -2101,7 +2197,7 @@ "write_caching_default": { "related_topics": [ "xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`]", - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]" + "xref:develop:manage-topics.adoc#configure-write-caching[Write caching]" ], "config_scope": "cluster", "description": "The default write caching mode to apply to user topics. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. \n\nFsyncs follow <> and <>, whichever is reached first.\n\nThe `write_caching_default` cluster property can be overridden with the xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`] topic property." diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 5eb34695dc..c833e5b2f1 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -36,7 +36,9 @@ ** xref:develop:kafka-clients.adoc[Kafka Compatibility] ** xref:develop:benchmark.adoc[] ** xref:develop:http-proxy.adoc[] -** xref:develop:config-topics.adoc[] +** xref:develop:manage-topics/index.adoc[] +*** xref:develop:manage-topics/config-topics.adoc[] +*** xref:develop:manage-topics/cloud-topics.adoc[] ** xref:console:ui/edit-topic-configuration.adoc[Edit Topic Configuration] ** xref:develop:produce-data/index.adoc[Produce Data] *** xref:develop:produce-data/configure-producers.adoc[] diff --git a/modules/console/pages/ui/edit-topic-configuration.adoc b/modules/console/pages/ui/edit-topic-configuration.adoc index 75dcd24042..ab9207a367 100644 --- a/modules/console/pages/ui/edit-topic-configuration.adoc +++ b/modules/console/pages/ui/edit-topic-configuration.adoc @@ -18,4 +18,4 @@ == Suggested reading - xref:reference:properties/topic-properties.adoc[] -- xref:develop:config-topics.adoc[] \ No newline at end of file +- xref:develop:manage-topics/config-topics.adoc[] \ No newline at end of file diff --git a/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc b/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc index 36e5db81e1..44aaf8cd91 100644 --- a/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc +++ b/modules/deploy/pages/redpanda/manual/production/dev-deployment.adoc @@ -8,7 +8,7 @@ You can deploy Redpanda using well-known configuration properties optimized for [NOTE] ==== -* Development mode enables write caching by default. This is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. Write caching provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:config-topics.adoc#configure-write-caching[write caching]. +* Development mode enables write caching by default. This is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. Write caching provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching]. * Development mode also bypasses `fsync`, acknowledging messages before they're stored to disk. This reduces the durability of messages, could cause potential data loss, and could give unrealistic performance characteristics for a production environment. ==== diff --git a/modules/develop/pages/kafka-clients.adoc b/modules/develop/pages/kafka-clients.adoc index 3f7815ded2..fd115b185e 100644 --- a/modules/develop/pages/kafka-clients.adoc +++ b/modules/develop/pages/kafka-clients.adoc @@ -60,7 +60,7 @@ endif::[] * HTTP Proxy (pandaproxy): Unlike other REST proxy implementations in the Kafka ecosystem, Redpanda HTTP Proxy does not support topic and ACLs CRUD through the HTTP Proxy. HTTP Proxy is designed for clients producing and consuming data that do not perform administrative functions. ifdef::env-cloud[] + -* The `delete.retention.ms` topic configuration in Kafka is not supported. Tombstone markers are not removed for topics with a `compact` xref:get-started:config-topics.adoc#change-the-cleanup-policy[cleanup policy]. Redpanda only deletes tombstone markers when topics with a cleanup policy of `compact,delete` have reached their xref:get-started:create-topic.adoc[retention limits]. +* The `delete.retention.ms` topic configuration in Kafka is not supported. Tombstone markers are not removed for topics with a `compact` cleanup policy. Redpanda only deletes tombstone markers when topics with a cleanup policy of `compact,delete` have reached their retention limits. endif::[] ifndef::env-cloud[] + diff --git a/modules/develop/pages/manage-topics/cloud-topics.adoc b/modules/develop/pages/manage-topics/cloud-topics.adoc new file mode 100644 index 0000000000..4eb9f78fa7 --- /dev/null +++ b/modules/develop/pages/manage-topics/cloud-topics.adoc @@ -0,0 +1,92 @@ += Manage Cloud Topics +:description: Cloud Topics are Redpanda topics that enable users to trade off latency for lower costs. +:learning-objective-1: Describe the latency and cost trade-offs of Cloud Topics compared to standard Redpanda topics +:learning-objective-2: Create a Cloud Topic using rpk after enabling Cloud Topics on your cluster +:learning-objective-3: Identify Cloud Topics limitations and configurations that reduce cross-AZ networking costs +// tag::single-source[] + +ifndef::env-cloud[] +[NOTE] +==== +include::shared:partial$enterprise-license.adoc[] +==== +endif::[] + +Starting in v26.1, Redpanda provides Cloud Topics to support multi-modal streaming workloads in the most cost-effective way possible: as a configurable option on a single cluster. While standard Redpanda xref:config-topics.adoc[topics] that use local storage or Tiered Storage are ideal for latency-tolerant workloads (for example, for audit logs or analytics), Cloud Topics are optimized for latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major consideration that can become the dominant cost driver at high throughput. These workloads can include observability streams, offline analytics, AI/ML model training data feeds, or development environments that have flexible latency requirements. + +Instead of replicating every byte across expensive network links, Cloud Topics leverage durable, inexpensive Cloud storage (S3, ADLS, GCS, MinIO) as the primary mechanism to back up data. This eliminates over 90% of the cost of replicating data over network links in multi-AZ clusters. The difference in end-to-end latency is typically between 500 ms - 1 s, which is often acceptable for many streaming workloads, and can unlock new streaming use cases that previously were not cost effective. + +With Cloud Topics, data from the client is not acknowledged until it is uploaded to cloud storage. This maintains durability in the face of infrastructure failures, but results in an increase in both produce latency and end-to-end latency, driven by both batching of produced data and the inherent latency of the underlying object store. You should generally expect latencies in the range of 250 ms - 1 s. + +== Prerequisites + +- xref:get-started:rpk-install.adoc[] v26.1 or later. +ifndef::env-cloud[] +- xref:manage:tiered-storage.adoc#set-up-tiered-storage[Enable Tiered Storage] on your Redpanda cluster. ++ +[NOTE] +==== +When you enable Tiered Storage at the cluster level, by default, all new topics are Tiered Storage (or standard) topics. However, if you plan to use Cloud Topics for all new topics in a Redpanda cluster, be sure to set the following cluster-level property: ++ +[,bash] +---- +default_redpanda_storage_mode=cloud +---- ++ +This ensures that newly-created Redpanda topics are Cloud Topics by default. ++ +For details, see xref:manage:tiered-storage.adoc#enable-tiered-storage-for-a-cluster[Enable Tiered Storage for a cluster]. +==== + +- xref:manage:tiered-storage.adoc#configure-object-storage[Configure object storage]. +- Ensure that you have an Enterprise license. ++ +To check your license status, run: ++ +[,bash] +---- +rpk cluster license info +---- ++ +- If you intend to use Docker Compose while working with Cloud Topics, see the https://docs.docker.com/compose/install/[Docker Compose installation documentation^]. To verify you have it installed, run: ++ +[source,bash] +---- +docker compose version +---- ++ +You should see the installed Docker Compose version. ++ +For more details about using Docker Compose with Redpanda, see the xref:get-started:quick-start.adoc[Redpanda Self-Managed Quickstart]. + +endif::[] + +== Limitations + +The Remote Read Replicas feature is not supported for Cloud Topics. + +IMPORTANT: Any topic specified to be a Cloud Topic cannot be subsequently converted back to a standard Redpanda topic that uses local or Tiered Storage. + +== Enable Cloud Topics for your cluster + +To enable Cloud Topics for a cluster: + +[,bash] +---- +rpk cluster config set cloud_topics_enabled=true +---- + +NOTE: This configuration update requires a restart. + +[,bash] +---- +rpk topic create -c redpanda.storage.mode=cloud +TOPIC STATUS +audit.analytics.may2025 OK +---- + +You can make a topic a Cloud Topic only at topic creation time. + +In addition to replication, cross-AZ ingress (producer) and egress (consumer) traffic can also contribute substantially to cloud networking cost. When running multi-AZ clusters in general, Redpanda strongly recommends using xref:../consume-data/follower-fetching.adoc[Follower Fetching], which allows consumers to avoid crossing network zones. When possible, you can use xref:../produce-data/leader-pinning.adoc[leader pinning], which positions a topic's partition leader close to the producers, providing a similar benefit for ingress traffic. These features can add additional savings to the replication cost savings of Cloud Topics. + +// end::single-source[] \ No newline at end of file diff --git a/modules/develop/pages/manage-topics/config-topics.adoc b/modules/develop/pages/manage-topics/config-topics.adoc new file mode 100644 index 0000000000..d258d78021 --- /dev/null +++ b/modules/develop/pages/manage-topics/config-topics.adoc @@ -0,0 +1,276 @@ += Manage Topics +:page-categories: Clients, Development +:description: Learn how to create topics, update topic configurations, and delete topics or records. +// tag::single-source[] + +include::develop:partial$topic-defaults.adoc[] + +== Create a topic + +Creating a topic can be as simple as specifying a name for your topic on the command line. For example, to create a topic named `xyz`, run: + +[,bash] +---- +rpk topic create xyz +---- + +ifndef::env-cloud[] +This command creates a topic named `xyz` with one partition and one replica, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. + +endif::[] + +ifdef::env-cloud[] +This command creates a topic named `xyz` with one partition and three replicas, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. + +Redpanda Cloud supports 40,000 topics per cluster. + +endif::[] + +=== Choose the number of partitions + +A partition acts as a log file where topic data is written. Dividing topics into partitions allows producers to write messages in parallel and consumers to read messages in parallel. The higher the number of partitions, the greater the throughput. + +TIP: As a general rule, select a number of partitions that corresponds to the maximum number of consumers in any consumer group that will consume the data. + +For example, suppose you plan to create a consumer group with 10 consumers. To create topic `xyz` with 10 partitions, run: + +[,bash] +---- +rpk topic create xyz -p 10 +---- + +ifndef::env-cloud[] +=== Choose the replication factor + +The default replication factor in the cluster configuration is set to 1. By choosing a replication factor greater than 1, you ensure that each partition has a copy of its data on at least one other broker. One replica acts as the leader, and the other replicas are followers. + +To specify a replication factor of 3 for topic `xyz`, run: + +[,bash] +---- +rpk topic create xyz -r 3 +---- + +NOTE: The replication factor must be an odd number. Redpanda Data recommends a replication factor of 3 for most use cases. Administrators may set a minimum required replication factor for any new topic in the cluster through the cluster-level xref:reference:cluster-properties.adoc#minimum_topic_replications[`minimum_topic_replications`] property. + +TIP: If you enable xref:manage:tiered-storage.adoc[Tiered Storage] on a topic, you can then use xref:manage:topic-recovery.adoc[topic recovery] to restore data for a deleted topic. + +endif::[] + +== Update topic configurations + +After you create a topic, you can update the topic property settings for all new data written to it. For example, you can add partitions or change the cleanup policy. + +=== Add partitions + +You can assign a certain number of partitions when you create a topic, and add partitions later. For example, suppose you add brokers to your cluster, and you want to take advantage of the additional processing power. To increase the number of partitions for existing topics, run: + +[,bash] +---- +rpk topic add-partitions [TOPICS...] --num [#] +---- + +Note that `--num <#>` is the number of partitions to _add_, not the total number of partitions. + +include::develop:partial$balance-existing-topic-redistribution.adoc[] + +ifndef::env-cloud[] +=== Change the replication factor + +Suppose you create a topic with the default replication factor of 1 (which is specified in the cluster properties configuration file). Now you want to change the replication factor to 3, so you can have two backups of topic data in case a broker goes down. To set the replication factor to 3, run: + +[,bash] +---- +rpk topic alter-config [TOPICS...] --set replication.factor=3 +---- + +NOTE: The replication factor can't exceed the number of Redpanda brokers. If you try to set a replication factor greater than the number of brokers, the request is rejected. + +endif::[] + + +=== Change the cleanup policy + +The cleanup policy determines how to clean up the partition log files when they reach a certain size: + +* `delete` deletes data based on age or log size. Topics retain all records until then. +* `compact` compacts the data by only keeping the latest values for each KEY. +* `compact,delete` combines both methods. + +Unlike compacted topics, which keep only the most recent message for a given key, topics configured with a `delete` cleanup policy provide a running history of all changes for those topics. + +include::develop:partial$topic-properties-warning.adoc[] + +For example, to change a topic's policy to `compact`, run: + +[,bash] +---- +rpk topic alter-config [TOPICS…] —-set cleanup.policy=compact +---- + +ifndef::env-cloud[] +For details on compaction in Redpanda, see xref:manage:cluster-maintenance/compaction-settings.adoc[Compaction settings]. + +endif::[] + +[[configure-write-caching]] +=== Configure write caching + +Write caching is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that provides better performance at the expense of durability. It acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. + +Write caching applies to user topics. It does not apply to transactions or consumer offsets: data written in the context of a transaction and consumer offset commits is always written to disk and fsynced before being acknowledged to the client. + +ifndef::env-cloud[] +NOTE: For clusters in xref:reference:rpk/rpk-redpanda/rpk-redpanda-mode.adoc#development-mode[development mode], write caching is enabled by default. For clusters in production mode, it is disabled by default. + +endif::[] + +Only enable write caching on workloads that can tolerate some data loss in the case of multiple, simultaneous broker failures. Leaving write caching disabled safeguards your data against complete data center or availability zone failures. + +ifndef::env-cloud[] + +==== Configure at cluster level + +To enable write caching by default in all user topics, set the cluster-level property xref:reference:cluster-properties.adoc#write_caching_default[`write_caching_default`]: + +`rpk cluster config set write_caching_default=true` + +With `write_caching_default` set to true at the cluster level, Redpanda fsyncs to disk according to xref:reference:cluster-properties.adoc#raft_replica_max_pending_flush_bytes[`raft_replica_max_pending_flush_bytes`] and xref:reference:cluster-properties.adoc#raft_replica_max_flush_delay_ms[`raft_replica_max_flush_delay_ms`], whichever is reached first. + +endif::[] + +==== Configure at topic level + +To override the cluster-level setting at the topic level, set the topic-level property `write.caching`: + +`rpk topic alter-config my_topic --set write.caching=true` + +With `write.caching` enabled at the topic level, Redpanda fsyncs to disk according to `flush.ms` and `flush.bytes`, whichever is reached first. + +=== Remove a configuration setting + +You can remove a configuration that overrides the default setting, and the setting will use the default value again. For example, suppose you altered the cleanup policy to use `compact` instead of the default, `delete`. Now you want to return the policy setting to the default. To remove the configuration setting `cleanup.policy=compact`, run `rpk topic alter-config` with the `--delete` flag: + +[,bash] +---- +rpk topic alter-config [TOPICS...] --delete cleanup.policy +---- + +== List topic configuration settings + +To display all the configuration settings for a topic, run: + +[,bash] +---- +rpk topic describe -c +---- + +The `-c` flag limits the command output to just the topic configurations. This command is useful for checking the default configuration settings before you make any changes and for verifying changes after you make them. + +The following command output displays after running `rpk topic describe test-topic`, where `test-topic` was created with default settings: + +ifndef::env-cloud[] +[,bash] +---- +rpk topic describe test_topic +SUMMARY +======= +NAME test_topic +PARTITIONS 1 +REPLICAS 1 + +CONFIGS +======= +KEY VALUE SOURCE +cleanup.policy delete DYNAMIC_TOPIC_CONFIG +compression.type producer DEFAULT_CONFIG +max.message.bytes 1048576 DEFAULT_CONFIG +message.timestamp.type CreateTime DEFAULT_CONFIG +redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG +redpanda.remote.delete true DEFAULT_CONFIG +redpanda.remote.read false DEFAULT_CONFIG +redpanda.remote.write false DEFAULT_CONFIG +retention.bytes -1 DEFAULT_CONFIG +retention.local.target.bytes -1 DEFAULT_CONFIG +retention.local.target.ms 86400000 DEFAULT_CONFIG +retention.ms 604800000 DEFAULT_CONFIG +segment.bytes 1073741824 DEFAULT_CONFIG +---- + +Suppose you add two partitions, and increase the number of replicas to 3. The new command output confirms the changes in the `SUMMARY` section: + +[.no-copy] +---- +SUMMARY +======= +NAME test_topic +PARTITIONS 3 +REPLICAS 3 +---- + +endif::[] + +ifdef::env-cloud[] +[,bash] +---- +rpk topic describe test_topic +SUMMARY +======= +NAME test_topic +PARTITIONS 1 +REPLICAS 3 + +CONFIGS +======= +KEY VALUE SOURCE +cleanup.policy delete DYNAMIC_TOPIC_CONFIG +compression.type producer DEFAULT_CONFIG +max.message.bytes 20971520 DEFAULT_CONFIG +message.timestamp.type CreateTime DEFAULT_CONFIG +redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG +redpanda.remote.delete true DEFAULT_CONFIG +redpanda.remote.read false DEFAULT_CONFIG +redpanda.remote.write false DEFAULT_CONFIG +retention.bytes -1 DEFAULT_CONFIG +retention.local.target.bytes -1 DEFAULT_CONFIG +retention.local.target.ms 86400000 DEFAULT_CONFIG +retention.ms 604800000 DEFAULT_CONFIG +segment.bytes 1073741824 DEFAULT_CONFIG +---- + +endif::[] + +== Delete a topic + +To delete a topic, run: + +[,bash] +---- +rpk topic delete +---- + +When a topic is deleted, its underlying data is deleted, too. + +To delete multiple topics at a time, provide a space-separated list. For example, to delete two topics named `topic1` and `topic2`, run: + +[,bash] +---- +rpk topic delete topic1 topic2 +---- + +You can also use the `-r` flag to specify one or more regular expressions; then, any topic names that match the pattern you specify are deleted. For example, to delete topics with names that start with "`f`" and end with "`r`", run: + +[,bash] +---- +rpk topic delete -r '^f.*' '.*r$' +---- + +Note that the first regular expression must start with the `^` symbol, and the last expression must end with the `$` symbol. This requirement helps prevent accidental deletions. + +include::develop:partial$delete-topic-records.adoc[] + +== Next steps + +xref:develop:produce-data/configure-producers.adoc[] + +// end::single-source[] diff --git a/modules/develop/pages/manage-topics/index.adoc b/modules/develop/pages/manage-topics/index.adoc new file mode 100644 index 0000000000..d1eb45c4f9 --- /dev/null +++ b/modules/develop/pages/manage-topics/index.adoc @@ -0,0 +1,4 @@ += Topics +:page-categories: Clients, Development +:description: Learn how to manage topics in Redpanda, including creation, configuration, and advanced features. +:page-layout: index diff --git a/modules/develop/pages/produce-data/configure-producers.adoc b/modules/develop/pages/produce-data/configure-producers.adoc index 18de11b961..77a9bc4a3a 100644 --- a/modules/develop/pages/produce-data/configure-producers.adoc +++ b/modules/develop/pages/produce-data/configure-producers.adoc @@ -74,7 +74,7 @@ Kafka, a message is considered acknowledged without the requirement that it has been fsynced. Messages that have not been fsynced to disk may be lost in the event of a broker crash. So when using `acks=all`, the Redpanda default configuration is more resilient than Kafka's. You can also consider -using xref:develop:config-topics.adoc#configure-write-caching[write caching], which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. +using xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching], which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. endif::[] diff --git a/modules/get-started/pages/quick-start.adoc b/modules/get-started/pages/quick-start.adoc index 1c65c751f1..5173dcd955 100644 --- a/modules/get-started/pages/quick-start.adoc +++ b/modules/get-started/pages/quick-start.adoc @@ -205,7 +205,8 @@ You can filter messages in topics using several methods: Suppose you're asked to find all transactions related to the `.edu` domain. You can use a JavaScript filter to display only messages that include email addresses in that domain. . In the menu, click *Topics*. -// (step {"find": {"selector": "[data-testid='nav-link-Topics']", "elementText": "Topics", "click": true, "timeout": 10000}}) +// (step {"goTo": "http://localhost:8080/overview"}) +// (step {"find": {"selector": "nav a[href='/topics']", "click": true, "timeout": 10000}}) . Click the *transactions* topic. When the topic's page opens, the *Messages* tab is selected by default. // (step {"goTo": "http://localhost:8080/topics/transactions"}) . Click *Add filter* > *JavaScript Filter*. @@ -241,7 +242,7 @@ See also: xref:console:ui/programmable-push-filters.adoc[]. On the *Schema Registry* page, you see an overview of your schemas. You can create, manage, and inspect your schemas without leaving Redpanda Console. -// (step {"find": {"selector": "[data-testid='nav-link-Schema Registry']", "elementText": "Schema Registry", "click": true, "timeout": 10000}}) +// (step {"find": {"selector": "nav a[href='/schema-registry']", "click": true, "timeout": 10000}}) // (step {"goTo": "http://localhost:8080/schema-registry/subjects/transactions?version=latest"}) // (step {"screenshot": {"path": "../../modules/console/images/schema-reg.png", "maxVariation": 0.10, "overwrite": "aboveVariation"}}) @@ -268,7 +269,7 @@ On the *Security* page, you can: Suppose you're onboarding a new team member who needs access to specific topics. You can use Redpanda Console to ensure users have the right permissions to perform their tasks. . In the menu, click *Security*. -// (step {"find": {"selector": "[data-testid='nav-link-Security']", "elementText": "Security", "click": true, "timeout": 10000}}) +// (step {"find": {"selector": "nav a[href='/security']", "click": true, "timeout": 10000}}) // (step {"goTo": "http://localhost:8080/security/users"}) . On the *Users* tab, click *Create user*. // (step {"find": {"selector": "[data-testid='create-user-button']", "elementText": "Create user", "click": true, "timeout": 100000}}) @@ -382,7 +383,7 @@ Data transforms let you run common data streaming tasks on the Redpanda broker, This quickstart deployment comes with one transform function called `regex` running in your cluster. Its job is to find records in the `logins` topic that contain email addresses with the `.edu` domain and add those to a new topic called `edu-filtered-domains`. In the menu, click *Transforms*. -// (step {"find": { "selector": "[data-testid='nav-link-Transforms']", "elementText": "Transforms", "click": true, "timeout": 15000}}) +// (step {"find": {"selector": "nav a[href='/transforms']", "click": true, "timeout": 15000}}) On the *Transforms* page, you see your transform. You can use Redpanda Console to manage and monitor your transforms. // (step {"screenshot": {"path": "../../modules/console/images/transforms.png", "maxVariation": 0.10, "overwrite": "aboveVariation"}}) @@ -403,7 +404,8 @@ See also: Audit logs provide a record of all user actions. You can use these logs to track changes, troubleshoot issues, and maintain compliance with your organization's security policies. . In the menu, click *Topics*. -// (step {"find": { "selector": "[data-testid='nav-link-Topics']", "click": true, "timeout": 15000}}) +// (step {"goTo": "http://localhost:8080/overview"}) +// (step {"find": {"selector": "nav a[href='/topics']", "click": true, "timeout": 15000}}) // (step {"goTo": "http://localhost:8080/topics"}) . Click the *Show internal topics* checkbox. // (step {"find": { "selector": "[data-testid='show-internal-topics-checkbox']", "elementText": "Show internal topics", "click": true, "timeout": 15000}}) @@ -443,7 +445,7 @@ docker exec -it redpanda-0 rpk topic create chat-room \ -X pass=secretpassword ``` + -<1> Set a replication factor of 3 to replicate the topic across all 3 brokers. This replication factor provides high availability and data durability. For more details, see xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]. +<1> Set a replication factor of 3 to replicate the topic across all 3 brokers. This replication factor provides high availability and data durability. For more details, see xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor]. <2> Enable remote reads for this topic to read offloaded records from object storage. <3> Enable remote writes for this topic to offload older records to object storage. For more details, see xref:manage:tiered-storage.adoc[]. + diff --git a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc index a4c55a2f79..e0d528b982 100644 --- a/modules/manage/pages/cluster-maintenance/compaction-settings.adoc +++ b/modules/manage/pages/cluster-maintenance/compaction-settings.adoc @@ -12,7 +12,7 @@ image::shared:compaction-example.png[Example of topic compaction] This diagram illustrates a compacted topic. Imagine a remote sensor network that uses image recognition to track appearances of red pandas in a geographic area. The sensor network employs special devices that send records to a topic when they detect one. You might enable compaction to reduce topic storage while still maintaining a record in the topic of the last time each device saw a red panda, perhaps to see if they stop frequenting a given area. The left side of the diagram shows all records sent across the topic. The right side illustrates the results of compaction; older records for certain keys are deleted from the log. -NOTE: If your application requires consuming every record for a given key, consider using the `delete` xref:develop:config-topics#change-the-cleanup-policy.adoc[cleanup policy] instead. +NOTE: If your application requires consuming every record for a given key, consider using the `delete` xref:develop:manage-topics/config-topics.adoc#change-the-cleanup-policy[cleanup policy] instead. IMPORTANT: When using xref:manage:tiered-storage.adoc[Tiered Storage], compaction functions at the local storage level. As long as a segment remains in local storage, its records are eligible for compaction. Once a segment is uploaded to object storage and removed from local storage it is not retrieved for further compaction operations. A key may therefore appear in multiple segments between Tiered Storage and local storage. diff --git a/modules/manage/pages/cluster-maintenance/disk-utilization.adoc b/modules/manage/pages/cluster-maintenance/disk-utilization.adoc index 8d66849891..b1484a49a7 100644 --- a/modules/manage/pages/cluster-maintenance/disk-utilization.adoc +++ b/modules/manage/pages/cluster-maintenance/disk-utilization.adoc @@ -85,7 +85,7 @@ Redpanda runs a log cleanup process in the background to apply these policy sett See also: * xref:manage:tiered-storage.adoc#manage-local-capacity-for-tiered-storage-topics[Manage local capacity for Tiered Storage topics] -* xref:develop:config-topics.adoc#delete-records-from-a-topic[Delete records from a topic] +* xref:develop:manage-topics/config-topics.adoc#delete-records-from-a-topic[Delete records from a topic] [[set-time-based-retention]] === Set time-based retention diff --git a/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc b/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc index 8a182e138a..7f1d6410f1 100644 --- a/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc +++ b/modules/manage/pages/cluster-maintenance/topic-property-configuration.adoc @@ -220,6 +220,6 @@ For complete details about all available topic properties, see xref:reference:pr * xref:reference:properties/topic-properties.adoc[Topic Configuration Properties] - Complete reference of all available topic properties * xref:manage:cluster-maintenance/cluster-property-configuration.adoc[Configure Cluster Properties] - Configure cluster-wide defaults -* xref:develop:config-topics.adoc[Manage Topics] - Create and manage topics +* xref:develop:manage-topics/config-topics.adoc[Manage Topics] - Create and manage topics * xref:manage:kubernetes/k-manage-topics.adoc[Manage Topics in Kubernetes] - Topic management in Kubernetes deployments * xref:console:ui/edit-topic-configuration.adoc[Edit Topic Configuration in Redpanda Console] - Graphical topic configuration \ No newline at end of file diff --git a/modules/manage/partials/audit-logging.adoc b/modules/manage/partials/audit-logging.adoc index ad1fe36a02..faa1714f8e 100644 --- a/modules/manage/partials/audit-logging.adoc +++ b/modules/manage/partials/audit-logging.adoc @@ -433,6 +433,6 @@ xref:manage:audit-logging/audit-log-samples.adoc[See samples of audit log messag include::shared:partial$suggested-reading.adoc[] - xref:reference:topic-properties.adoc[] -- xref:develop:config-topics.adoc[] +- xref:develop:manage-topics/config-topics.adoc[] endif::[] diff --git a/modules/reference/attachments/redpanda-properties-v25.3.7.json b/modules/reference/attachments/redpanda-properties-v25.3.7.json index 0e9b183d44..8789db316c 100644 --- a/modules/reference/attachments/redpanda-properties-v25.3.7.json +++ b/modules/reference/attachments/redpanda-properties-v25.3.7.json @@ -3207,7 +3207,6 @@ "default": false, "defined_in": "src/v/config/configuration.cc", "description": "Disables the cloud topics reconciliation loop. Disabling the loop can negatively impact performance and stability of the cluster.", - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": false, "name": "cloud_topics_disable_reconciliation_loop", @@ -3225,7 +3224,7 @@ "config_scope": "cluster", "default": false, "defined_in": "src/v/config/configuration.cc", - "description": "Enable cloud topics.", + "description": "Enable Cloud Topics for the cluster. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics.", "enterprise_constructor": "restricted_only", "enterprise_restricted_value": [ "true" @@ -3233,12 +3232,15 @@ "enterprise_value": [ "true" ], - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": true, "name": "cloud_topics_enabled", "needs_restart": false, "nullable": false, + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/topic-properties.adoc#redpandacloudtopicenabled[`redpanda.cloud_topic.enabled`]" + ], "type": "boolean", "visibility": "user" }, @@ -3297,7 +3299,6 @@ "default_human_readable": "5 minutes", "defined_in": "src/v/config/configuration.cc", "description": "Time interval after which data is garbage collected from long term storage.", - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, @@ -3318,7 +3319,6 @@ "default": 4194304, "defined_in": "src/v/config/configuration.cc", "description": "The size limit for the object size in cloud topics. When the amount of data on a shard reaches this limit, an upload is triggered.", - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": false, "name": "cloud_topics_produce_batching_size_threshold", @@ -3337,7 +3337,6 @@ "default": 1000, "defined_in": "src/v/config/configuration.cc", "description": "Threshold for the object cardinality in cloud topics. When the number of partitions in waiting for the upload reach this limit, an upload is triggered.", - "exclude_from_docs": true, "is_deprecated": false, "is_enterprise": false, "name": "cloud_topics_produce_cardinality_threshold", @@ -3356,8 +3355,7 @@ "default": 250, "default_human_readable": "250 milliseconds", "defined_in": "src/v/config/configuration.cc", - "description": "Time interval after which the upload is triggered.", - "exclude_from_docs": true, + "description": "Time interval after which data is uploaded to object storage for Cloud Topics. When this time threshold is reached, Redpanda triggers an upload of buffered data to the object storage backend (S3, GCS, or MinIO), regardless of whether the size or cardinality thresholds have been met.\n\nThis property works together with <> and <> to control when uploads occur. An upload is triggered when any of these three thresholds is reached.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, @@ -3365,6 +3363,11 @@ "name": "cloud_topics_produce_upload_interval", "needs_restart": false, "nullable": false, + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_batching_size_threshold[`cloud_topics_produce_batching_size_threshold`]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_cardinality_threshold[`cloud_topics_produce_cardinality_threshold`]" + ], "type": "integer", "visibility": "user" }, @@ -3378,8 +3381,7 @@ "default": 10000, "default_human_readable": "10 seconds", "defined_in": "src/v/config/configuration.cc", - "description": "Time interval after which data is moved from short term storage to long term storage.", - "exclude_from_docs": true, + "description": "Time interval at which Redpanda reconciles data between short-term local storage and long-term object storage for Cloud Topics. During reconciliation, Redpanda moves data that has been successfully persisted to object storage from local storage, freeing up local disk space while maintaining data durability in the cloud.\n\nThis reconciliation process is essential for Cloud Topics to achieve their cost-efficiency goals by minimizing the amount of expensive local storage required, while ensuring data remains accessible from durable object storage.", "is_deprecated": false, "is_enterprise": false, "maximum": 17592186044415, @@ -3387,6 +3389,10 @@ "name": "cloud_topics_reconciliation_interval", "needs_restart": false, "nullable": false, + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_garbage_collection_interval[`cloud_topics_long_term_garbage_collection_interval`]" + ], "type": "integer", "visibility": "tunable" }, @@ -11021,13 +11027,16 @@ "corresponding_cluster_property": null, "default": null, "defined_in": "src/v/kafka/protocol/topic_properties.h", - "description": "", - "exclude_from_docs": true, + "description": "Enable this topic as a Cloud Topic. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics.\n\nIMPORTANT: You can only set this property when creating a topic. After a topic is created as a Cloud Topic, it cannot be converted back to a standard Kafka topic. Before creating Cloud Topics, you must enable the cluster-wide <> property.", "is_deprecated": false, "is_enterprise": false, "is_topic_property": true, "name": "redpanda.cloud_topic.enabled", "needs_restart": false, + "related_topics": [ + "xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "xref:reference:properties/cluster-properties.adoc#cloud_topics_enabled[`cloud_topics_enabled`]" + ], "type": "string" }, "redpanda.iceberg.delete": { @@ -11512,8 +11521,8 @@ "xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`]", "xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`]", - "xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", - "xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor]", + "xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor]", "xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication]" ], "type": "integer" @@ -13886,7 +13895,7 @@ "name": "write.caching", "needs_restart": false, "related_topics": [ - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]", + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]", "xref:manage:tiered-storage.adoc[Tiered Storage]", "xref:reference:properties/cluster-properties.adoc#write_caching_default[`write_caching_default`]", "xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]" @@ -13916,7 +13925,7 @@ "nullable": false, "related_topics": [ "xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`]", - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]" + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]" ], "type": "string", "visibility": "user" diff --git a/modules/reference/pages/properties/cluster-properties.adoc b/modules/reference/pages/properties/cluster-properties.adoc index f3460c1adc..053173d11d 100644 --- a/modules/reference/pages/properties/cluster-properties.adoc +++ b/modules/reference/pages/properties/cluster-properties.adoc @@ -10,4 +10,4 @@ NOTE: Some cluster properties require that you restart the cluster for any updat == Cluster configuration -include::reference:partial$properties/cluster-properties.adoc[tags=!deprecated;!exclude-from-docs] \ No newline at end of file +include::reference:partial$properties/cluster-properties.adoc[tags=!deprecated;!exclude-from-docs] diff --git a/modules/reference/pages/properties/topic-properties.adoc b/modules/reference/pages/properties/topic-properties.adoc index ce669ba8e9..e99bff7283 100644 --- a/modules/reference/pages/properties/topic-properties.adoc +++ b/modules/reference/pages/properties/topic-properties.adoc @@ -33,6 +33,112 @@ These properties control disk flushing, replication, and how topics interact wit include::reference:partial$properties/topic-properties.adoc[tags=category-performance-cluster;!deprecated;!exclude-from-docs] +The maximum bytes not fsynced per partition. If this configured threshold is reached, the log is automatically fsynced, even though it wasn't explicitly requested. + +*Type:* integer + +*Unit:* bytes + +*Accepted values:* [`1`, `9223372036854775807`] + +*Default:* `262144` + +*Related cluster property:* xref:./cluster-properties.adoc#flush_bytes[`flush_bytes`] + +**Related topics**: + +- xref:develop:produce-data/configure-producers.adoc[] + +--- +[[flushms]] +=== flush.ms + +The maximum delay (in ms) between two subsequent fsyncs. After this delay, the log is automatically fsynced. + +*Type:* integer + +*Unit:* milliseconds + +*Accepted values:* [`1`, `9223372036854775`] + +*Default:* `100` + +*Related cluster property:* xref:./cluster-properties.adoc#flush_ms[`flush_ms`] + +**Related topics**: + +- xref:develop:produce-data/configure-producers.adoc[] + +--- +[[redpandaleaderspreference]] +=== redpanda.leaders.preference + +The preferred location (rack) for partition leaders of a topic. + +This property inherits the value from the config_ref:default_leaders_preference,true,properties/cluster-properties[] cluster configuration property. You may override the cluster-wide setting by specifying the value for individual topics. + +If the cluster configuration property config_ref:enable_rack_awareness,true,properties/cluster-properties[] is set to `false`, Leader Pinning is disabled across the cluster. + +*Type:* string + +*Default:* `none` + +**Values**: + +- `none`: Opt out the topic from Leader Pinning. +- `racks:[,,...]`: Specify the preferred location (rack) of all topic partition leaders. The list can contain one or more rack IDs. If you specify multiple IDs, Redpanda tries to distribute the partition leader locations equally across brokers in these racks. + +**Related topics**: + +- xref:develop:produce-data/leader-pinning.adoc[Leader pinning] + +--- +[[replicationfactor]] +=== replication.factor + +The number of replicas of a topic to save in different nodes (brokers) of a cluster. + +If `replication.factor` is set to a positive value, it overrides the cluster property xref:./cluster-properties.adoc#default_topic_replication[default_topic_replication] for the topic. + +NOTE: Although `replication.factor` isn't returned or displayed by xref:reference:rpk/rpk-topic/rpk-topic-describe.adoc[`rpk topic describe`] as a valid Kafka property, you can set it using xref:reference:rpk/rpk-topic/rpk-topic-alter-config.adoc[`rpk topic alter-config`]. When the `replication.factor` of a topic is altered, it isn't simply a property value that's updated, but rather the actual replica sets of topic partitions that are changed. + +*Type:* integer + +*Accepted values:* [`1`, `512`] + +*Default:* null + +*Related cluster property:* xref:./cluster-properties.adoc#default_topic_replication[`default_topic_replication`] + +**Related topics**: + +- xref:develop:manage-topics/config-topics.adoc#choose-the-replication-factor[Choose the replication factor] +- xref:develop:manage-topics/config-topics.adoc#change-the-replication-factor[Change the replication factor] + +--- +[[writecaching]] +=== write.caching + +The write caching mode to apply to a topic. + +When `write.caching` is set, it overrides the cluster property xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. Fsyncs follow <> and <>, whichever is reached first. + +*Type:* boolean + +*Default:* `false` + +**Values**: + +- `true` - Enables write caching for a topic, according to <> and <>. +- `false` - Disables write caching for a topic, according to <> and <>. + +*Related cluster property:* xref:./cluster-properties.adoc#write_caching_default[`write_caching_default`] + +**Related topics**: + +- xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching] + +--- == Tiered Storage properties Configure properties to manage topics for xref:manage:tiered-storage.adoc[Tiered Storage]. diff --git a/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc b/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc index 9cabe78320..efd6b2a553 100644 --- a/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc +++ b/modules/reference/pages/rpk/rpk-redpanda/rpk-redpanda-mode.adoc @@ -10,7 +10,7 @@ Development mode (`development` or `dev`) includes the following development-onl * Sets `developer_mode` to `true`. This starts Redpanda with dev-mode only settings, including: ** No minimal memory limits are enforced. ** No core assignment rules for Redpanda nodes are enforced. - ** Enables write caching, which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:config-topics.adoc#configure-write-caching[write caching]. + ** Enables write caching, which is a relaxed mode of `acks=all` that acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to fsync to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. For more information, or to disable this, see xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching]. ** Bypasses `fsync` (from https://docs.seastar.io/master/structseastar_1_1reactor%5F%5Foptions.html#ad66cb23f59ed5dfa8be8189313988692[Seastar option `unsafe_bypass_fsync`^]), which results in unrealistically fast clusters and may result in data loss. * Sets `overprovisioned` to `true`. Redpanda expects a dev system to be an overprovisioned environment. Based on a https://docs.seastar.io/master/structseastar_1_1reactor%5F%5Foptions.html#a0caf6c2ad579b8c22e1352d796ec3c1d[Seastar option^], setting `overprovisioned` disables thread affinity, zeros idle polling time, and disables busy-poll for disk I/O. * Sets all autotuner xref:./rpk-redpanda-tune-list.adoc#tuners[tuners] to `false`. The tuners are intended to run only for production mode. diff --git a/modules/reference/partials/properties/cluster-properties.adoc b/modules/reference/partials/properties/cluster-properties.adoc index 5443c0734f..a2ab5b07b6 100644 --- a/modules/reference/partials/properties/cluster-properties.adoc +++ b/modules/reference/partials/properties/cluster-properties.adoc @@ -860,7 +860,6 @@ endif::[] // end::redpanda-cloud[] -// tag::exclude-from-docs[] === cloud_topics_disable_reconciliation_loop Disables the cloud topics reconciliation loop. Disabling the loop can negatively impact performance and stability of the cluster. @@ -901,12 +900,10 @@ endif::[] |=== -// end::exclude-from-docs[] -// tag::exclude-from-docs[] === cloud_topics_enabled -Enable cloud topics. +Enable Cloud Topics for the cluster. Cloud Topics are optimized for high-throughput, cost-sensitive workloads that can tolerate higher latencies compared to standard Kafka topics. ifndef::env-cloud[] .Enterprise license required @@ -952,9 +949,14 @@ ifndef::env-cloud[] | User endif::[] +| Related topics +| +* xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] + +* xref:reference:properties/topic-properties.adoc#redpandacloudtopicenabled[`redpanda.cloud_topic.enabled`] + |=== -// end::exclude-from-docs[] === cloud_topics_epoch_service_epoch_increment_interval @@ -1104,9 +1106,7 @@ endif::[] |=== -// end::exclude-from-docs[] -// tag::exclude-from-docs[] === cloud_topics_produce_batching_size_threshold The size limit for the object size in cloud topics. When the amount of data on a shard reaches this limit, an upload is triggered. @@ -1147,9 +1147,7 @@ endif::[] |=== -// end::exclude-from-docs[] -// tag::exclude-from-docs[] === cloud_topics_produce_cardinality_threshold Threshold for the object cardinality in cloud topics. When the number of partitions in waiting for the upload reach this limit, an upload is triggered. @@ -1190,12 +1188,12 @@ endif::[] |=== -// end::exclude-from-docs[] -// tag::exclude-from-docs[] === cloud_topics_produce_upload_interval -Time interval after which the upload is triggered. +Time interval after which data is uploaded to object storage for Cloud Topics. When this time threshold is reached, Redpanda triggers an upload of buffered data to the object storage backend (S3, GCS, or MinIO), regardless of whether the size or cardinality thresholds have been met. + +This property works together with <> and <> to control when uploads occur. An upload is triggered when any of these three thresholds is reached. [cols="1s,2a"] |=== @@ -1234,14 +1232,22 @@ ifndef::env-cloud[] | User endif::[] +| Related topics +| +* xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] + +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_batching_size_threshold[`cloud_topics_produce_batching_size_threshold`] + +* xref:reference:properties/cluster-properties.adoc#cloud_topics_produce_cardinality_threshold[`cloud_topics_produce_cardinality_threshold`] + |=== -// end::exclude-from-docs[] -// tag::exclude-from-docs[] === cloud_topics_reconciliation_interval -Time interval after which data is moved from short term storage to long term storage. +Time interval at which Redpanda reconciles data between short-term local storage and long-term object storage for Cloud Topics. During reconciliation, Redpanda moves data that has been successfully persisted to object storage from local storage, freeing up local disk space while maintaining data durability in the cloud. + +This reconciliation process is essential for Cloud Topics to achieve their cost-efficiency goals by minimizing the amount of expensive local storage required, while ensuring data remains accessible from durable cloud storage. [cols="1s,2a"] |=== @@ -1280,9 +1286,14 @@ ifndef::env-cloud[] | Tunable endif::[] +| Related topics +| +* xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] + +* xref:reference:properties/cluster-properties.adoc#cloud_topics_long_term_garbage_collection_interval[`cloud_topics_long_term_garbage_collection_interval`] + |=== -// end::exclude-from-docs[] === cloud_topics_short_term_gc_backoff_interval @@ -19931,7 +19942,7 @@ endif::[] | * xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`] -* xref:develop:config-topics.adoc#configure-write-caching[Write caching] +* xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching] |=== diff --git a/modules/reference/partials/properties/topic-properties.adoc b/modules/reference/partials/properties/topic-properties.adoc index 1463d30bfd..e89847f356 100644 --- a/modules/reference/partials/properties/topic-properties.adoc +++ b/modules/reference/partials/properties/topic-properties.adoc @@ -948,8 +948,7 @@ endif::[] // tag::exclude-from-docs[] === redpanda.cloud_topic.enabled -No description available. - +Configuration property: redpanda.cloud_topic.enabled [cols="1s,2a"] |=== @@ -1550,6 +1549,64 @@ endif::[] // end::category-tiered-storage[] +// tag::category-other[] +=== redpanda.storage.mode + +The storage mode for a topic. Determines how topic data is stored and whether it is eligible for upload to object storage. + +Accepted values: + +* `local`: Topic data is stored only on the broker's local disk. Object storage upload is disabled for the topic, regardless of cluster-level Tiered Storage settings. +* `tiered`: Topic data is stored on local disk and also uploaded to object storage. Enables xref:manage:tiered-storage.adoc[Tiered Storage] for the topic. +* `cloud`: Topic data is stored in object storage using the Cloud Topics architecture. Local storage is used only as a write buffer. +* `unset`: Uses the cluster-level config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting, or falls back to legacy `redpanda.remote.read` and `redpanda.remote.write` topic property behavior for backwards compatibility. + +This property overrides the cluster-wide config_ref:default_redpanda_storage_mode,true,properties/cluster-properties[] setting for individual topics. + +[cols="1s,2a"] +|=== +| Property | Value + +| Type +| `string` (enum) + +| Accepted Values +| +ifndef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] +ifdef::env-cloud[] +`local`, `tiered`, `cloud`, `unset` +endif::[] + + +| Corresponding cluster property +| xref:reference:cluster-properties.adoc#default_redpanda_storage_mode[default_redpanda_storage_mode] + +| Default +| +ifdef::env-cloud[] +Available in the Redpanda Cloud Console +endif::[] +ifndef::env-cloud[] +`unset` +endif::[] + +| Nullable +| No + +ifndef::env-cloud[] +| Restored on xref:manage:whole-cluster-restore.adoc[Whole Cluster Restore] +| Yes +endif::[] + +| Related topics +|xref:manage:tiered-storage.adoc[Tiered Storage] + +|=== + +// end::category-other[] + // tag::category-schema-registry[] === redpanda.value.schema.id.validation @@ -1710,9 +1767,9 @@ endif::[] * xref:reference:properties/cluster-properties.adoc#default_topic_replication[`default_topic_replication`] -* xref:develop:config-topics.adoc#choose-the-replication-factor[Choose the replication factor] +* xref:develop:manage-topics.adoc#choose-the-replication-factor[Choose the replication factor] -* xref:develop:config-topics.adoc#change-the-replication-factor[Change the replication factor] +* xref:develop:manage-topics.adoc#change-the-replication-factor[Change the replication factor] * xref:reference:properties/cluster-properties.adoc#default_topic_replication[default_topic_replication] @@ -2106,7 +2163,7 @@ endif::[] | Related topics | -* xref:develop:config-topics.adoc#configure-write-caching[Write caching] +* xref:develop:manage-topics.adoc#configure-write-caching[Write caching] * xref:manage:tiered-storage.adoc[Tiered Storage] diff --git a/modules/reference/partials/properties/topic-property-mappings.adoc b/modules/reference/partials/properties/topic-property-mappings.adoc index 238d37a826..6843535cc9 100644 --- a/modules/reference/partials/properties/topic-property-mappings.adoc +++ b/modules/reference/partials/properties/topic-property-mappings.adoc @@ -63,6 +63,9 @@ | <> | xref:./object-storage-properties.adoc#cloud_storage_enable_remote_write[`cloud_storage_enable_remote_write`] +| <> +| xref:./cluster-properties.adoc#default_redpanda_storage_mode[`default_redpanda_storage_mode`] + | <> | xref:./cluster-properties.adoc#retention_bytes[`retention_bytes`] diff --git a/modules/shared/partials/tristate-behavior-change-25-3.adoc b/modules/shared/partials/tristate-behavior-change-25-3.adoc index e79d95b59a..cce6a25c22 100644 --- a/modules/shared/partials/tristate-behavior-change-25-3.adoc +++ b/modules/shared/partials/tristate-behavior-change-25-3.adoc @@ -1,5 +1,5 @@ Starting in Redpanda v25.3, several topic properties support enhanced tristate behavior. Properties like `retention.ms`, `retention.bytes`, `segment.ms`, and others now distinguish between zero values (immediate eligibility for cleanup/compaction) and negative values (disable the feature entirely). Previously, zero and negative values were treated the same way. ifndef::env-cloud[] -For the complete list of affected properties and detailed information, see xref:25.3@get-started:release-notes/redpanda.adoc#behavior-changes[Redpanda v25.3 behavior changes]. +For the complete list of affected properties and detailed information, see xref:get-started:release-notes/redpanda.adoc#behavior-changes[Redpanda v26.1 behavior changes]. endif::[] Review your topic configurations if you currently use zero values for these properties. \ No newline at end of file