diff --git a/antora.yml b/antora.yml index 2b76f5eff8..4133bf92a1 100644 --- a/antora.yml +++ b/antora.yml @@ -7,6 +7,7 @@ start_page: overview:introduction-to-eclipse-che.adoc nav: - modules/overview/nav.adoc - modules/hosted-che/nav.adoc + - modules/plan/nav.adoc - modules/end-user-guide/nav.adoc - modules/administration-guide/nav.adoc - modules/extensions/nav.adoc diff --git a/modules/administration-guide/examples/snip_che-upgrading-the-chectl-management-tool.adoc b/modules/administration-guide/examples/snip_che-upgrading-the-chectl-management-tool.adoc index e171617f41..7f3ad72386 100644 --- a/modules/administration-guide/examples/snip_che-upgrading-the-chectl-management-tool.adoc +++ b/modules/administration-guide/examples/snip_che-upgrading-the-chectl-management-tool.adoc @@ -4,7 +4,7 @@ This section describes how to upgrade the `{prod-cli}` management tool. .Prerequisites -* `{prod-cli}`. See: xref:installing-the-chectl-management-tool.adoc[]. +* `{prod-cli}`. See: xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/nav.adoc b/modules/administration-guide/nav.adoc index e26825866c..8b243d63c6 100644 --- a/modules/administration-guide/nav.adoc +++ b/modules/administration-guide/nav.adoc @@ -3,7 +3,6 @@ * xref:security-best-practices.adoc[] * xref:preparing-the-installation.adoc[] ** xref:supported-platforms.adoc[] -** xref:installing-the-chectl-management-tool.adoc[] ** xref:architecture-overview.adoc[] *** xref:server-components.adoc[] **** xref:che-operator.adoc[] @@ -14,8 +13,6 @@ **** xref:che-devfile-registry.adoc[] **** xref:plugin-registry.adoc[] *** xref:user-workspaces.adoc[] -** xref:calculating-che-resource-requirements.adoc[] -* xref:running-at-scale.adoc[] // Install * xref:installing-che.adoc[] ** Deploy on OpenShift diff --git a/modules/administration-guide/pages/installing-che-on-amazon-elastic-kubernetes-service.adoc b/modules/administration-guide/pages/installing-che-on-amazon-elastic-kubernetes-service.adoc index e179cdc752..40c9232bbb 100644 --- a/modules/administration-guide/pages/installing-che-on-amazon-elastic-kubernetes-service.adoc +++ b/modules/administration-guide/pages/installing-che-on-amazon-elastic-kubernetes-service.adoc @@ -13,7 +13,7 @@ To provide cloud development environments on AWS, deploy {prod-short} on an Amaz * You have `helm` installed. See link:https://helm.sh/docs/intro/install/[Installing Helm]. -* You have `{prod-cli}` installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have `{prod-cli}` installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. * You have the `aws` CLI installed. See link:https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html[AWS CLI install and update instructions]. diff --git a/modules/administration-guide/pages/installing-the-chectl-management-tool.adoc b/modules/administration-guide/pages/installing-the-chectl-management-tool.adoc deleted file mode 100644 index 99576a06fa..0000000000 --- a/modules/administration-guide/pages/installing-the-chectl-management-tool.adoc +++ /dev/null @@ -1,22 +0,0 @@ -:_content-type: ASSEMBLY -:description: Installing the {prod-cli} management tool -:keywords: overview, using-the-chectl-management-tool -:navtitle: Installing the {prod-cli} management tool -:page-aliases: installation-guide:using-the-chectl-management-tool.adoc, overview:using-the-chectl-management-tool.adoc, using-the-chectl-management-tool.adoc - - -[id="installing-the-{prod-cli}-management-tool"] -= Installing the {prod-cli} management tool - - -You can install `{prod-cli}`, the {prod} command-line management tool, on Microsoft Windows, Apple MacOS, and Linux. With `{prod-cli}`, you can perform operations the {prod-short} server such as starting, stopping, updating, and deleting the server. - -include::example$proc_{project-context}-installing-the-chectl-management-tool-on-windows.adoc[leveloffset=+1] - -include::example$proc_{project-context}-installing-the-chectl-management-tool-on-linux-or-macos.adoc[leveloffset=+1] - - -.Additional resources - -* {link-prod-cli-documentation} - diff --git a/modules/administration-guide/pages/preparing-the-installation.adoc b/modules/administration-guide/pages/preparing-the-installation.adoc index 0e5ff54637..cba4d7f17e 100644 --- a/modules/administration-guide/pages/preparing-the-installation.adoc +++ b/modules/administration-guide/pages/preparing-the-installation.adoc @@ -10,7 +10,7 @@ To prepare a {prod-short} installation, learn about the {prod-short} ecosystem and deployment constraints: * xref:supported-platforms.adoc[] -* xref:installing-the-chectl-management-tool.adoc[] +* xref:plan:installing-the-chectl-management-tool.adoc[] * xref:architecture-overview.adoc[] -* xref:calculating-che-resource-requirements.adoc[] +* xref:plan:calculating-che-resource-requirements.adoc[] * xref:understanding-the-checluster-custom-resource.adoc[] diff --git a/modules/administration-guide/pages/proc_installing-che-on-minikube-keycloak-oidc.adoc b/modules/administration-guide/pages/proc_installing-che-on-minikube-keycloak-oidc.adoc index 4ac905f2e6..43d62bf4d8 100644 --- a/modules/administration-guide/pages/proc_installing-che-on-minikube-keycloak-oidc.adoc +++ b/modules/administration-guide/pages/proc_installing-che-on-minikube-keycloak-oidc.adoc @@ -23,7 +23,7 @@ Single-node {kubernetes} clusters are suited only for testing or development. Do * You have `{orch-cli}` installed. See link:https://kubernetes.io/docs/tasks/tools/#kubectl[Installing `{orch-cli}`]. -* You have `{prod-cli}` installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have `{prod-cli}` installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/pages/proc_installing-che-on-minikube.adoc b/modules/administration-guide/pages/proc_installing-che-on-minikube.adoc index e106fdf2f5..bb36e695e2 100644 --- a/modules/administration-guide/pages/proc_installing-che-on-minikube.adoc +++ b/modules/administration-guide/pages/proc_installing-che-on-minikube.adoc @@ -21,7 +21,7 @@ Single-node {kubernetes} clusters are suited only for testing or development. Do * You have `{orch-cli}` installed. See link:https://kubernetes.io/docs/tasks/tools/#kubectl[Installing `{orch-cli}`]. -* You have `{prod-cli}` installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have `{prod-cli}` installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/pages/proc_installing-che-on-openshift-using-cli.adoc b/modules/administration-guide/pages/proc_installing-che-on-openshift-using-cli.adoc index b4f9c2f28d..8edf7a81d6 100644 --- a/modules/administration-guide/pages/proc_installing-che-on-openshift-using-cli.adoc +++ b/modules/administration-guide/pages/proc_installing-che-on-openshift-using-cli.adoc @@ -16,7 +16,7 @@ To set up cloud development environments for your team on an {orch-name} cluster * You have an active `{orch-cli}` session with administrative permissions to the {orch-name} cluster. See link:https://docs.openshift.com/container-platform/{ocp4-ver}/cli_reference/openshift_cli/getting-started-cli.html[Getting started with the OpenShift CLI]. -* You have the `{prod-cli}` management tool installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have the `{prod-cli}` management tool installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/pages/proc_installing-che-on-red-hat-openshift-local.adoc b/modules/administration-guide/pages/proc_installing-che-on-red-hat-openshift-local.adoc index 669900ab42..9963f2667d 100644 --- a/modules/administration-guide/pages/proc_installing-che-on-red-hat-openshift-local.adoc +++ b/modules/administration-guide/pages/proc_installing-che-on-red-hat-openshift-local.adoc @@ -12,7 +12,7 @@ To evaluate {prod-short} on a local {orch-name} cluster, deploy {prod-short} by .Prerequisites -* You have `{prod-cli}` installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have `{prod-cli}` installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. * You have a running instance of {rh-os-local}. See link:https://developers.redhat.com/products/openshift-local/overview[{rh-os-local} overview]. diff --git a/modules/administration-guide/pages/proc_installing-che-on-the-virtual-kubernetes-cluster.adoc b/modules/administration-guide/pages/proc_installing-che-on-the-virtual-kubernetes-cluster.adoc index 0e35b96b55..9e6f4e4118 100644 --- a/modules/administration-guide/pages/proc_installing-che-on-the-virtual-kubernetes-cluster.adoc +++ b/modules/administration-guide/pages/proc_installing-che-on-the-virtual-kubernetes-cluster.adoc @@ -20,7 +20,7 @@ To deploy {prod-short} when the host cluster version is incompatible or lacks ex * You have `kubelogin` installed. See link:https://github.com/int128/kubelogin[Installing kubelogin]. -* You have `{prod-cli}` installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have `{prod-cli}` installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. * You have an active `{orch-cli}` session with administrative permissions to the destination {orch-name} cluster. diff --git a/modules/administration-guide/pages/proc_uninstalling-che.adoc b/modules/administration-guide/pages/proc_uninstalling-che.adoc index ec1436bbfa..6cbbc53531 100644 --- a/modules/administration-guide/pages/proc_uninstalling-che.adoc +++ b/modules/administration-guide/pages/proc_uninstalling-che.adoc @@ -17,7 +17,7 @@ Uninstalling {prod-short} removes all {prod-short}-related user data. .Prerequisites -* You have the `{prod-cli}` management tool installed. See xref:installing-the-chectl-management-tool.adoc[]. +* You have the `{prod-cli}` management tool installed. See xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure @@ -43,4 +43,4 @@ Standard operating procedure (SOP) for removing {devworkspace} Operator manually [role="_additional-resources"] .Additional resources -* xref:installing-the-chectl-management-tool.adoc[] +* xref:plan:installing-the-chectl-management-tool.adoc[] diff --git a/modules/administration-guide/pages/running-at-scale.adoc b/modules/administration-guide/pages/running-at-scale.adoc deleted file mode 100644 index fc62192cb0..0000000000 --- a/modules/administration-guide/pages/running-at-scale.adoc +++ /dev/null @@ -1,231 +0,0 @@ -:_content-type: CONCEPT -:description: Running {prod} at scale -:keywords: scale, infrastructure, workload, scalability, CDE, cloud -:navtitle: Running {prod} at scale -//:page-aliases: - -[id="running-at-scale"] -= Running {prod} at scale - -Even though link:https://kubernetes.io/[Kubernetes] has emerged as a powerful foundation for deploying and managing containerized workloads at scale, achieving scale with Cloud Development Environments (CDEs), particularly in the range of thousands of concurrent workspaces, presents significant challenges. - -Such a scale imposes high infrastructure demands and introduces potential bottlenecks that can impact the performance and stability of the entire system. Addressing these challenges requires meticulous planning, strategic architectural choices, monitoring, and continuous optimization to ensure a seamless and efficient development experience for a large number of users. - -[NOTE] -==== -CDE workloads are complex to scale mainly because the underlying IDE solutions, -such as link:https://github.com/microsoft/vscode[Visual Studio Code - -Open Source ("Code - OSS")] or link:https://www.jetbrains.com/remote-development/gateway/[JetBrains Gateway], -are designed as single-user applications, not as multitenant services. -==== - -.Resource quantity and object maximums - -While there is no strict limit on the number of resources in a {kubernetes} cluster, -there are certain link:https://kubernetes.io/docs/setup/best-practices/cluster-large/[considerations for large clusters] to remember - -link:https://www.redhat.com/en/technologies/cloud-computing/openshift[OpenShift Container Platform], which is a certified distribution of {kubernetes}, also provides a set of tested maximums for various resources, which can serve as an initial guideline for planning your environment: - -[%header,format=csv] -|=== -Resource type, Tested maximum -Number of nodes,2000 -Number of pods,150000 -Number of pods per node,2500 -Number of namespace,10000 -Number of services,10000 -Number of secrets,80000 -Number of config maps,90000 -|=== - -Table 1: OpenShift Container Platform tested cluster maximums for various resources. - -[NOTE] -==== -You can find more details on the OpenShift Container Platform tested object maximums in the link:https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/scalability_and_performance/planning-your-environment-according-to-object-maximums#planning-your-environment-according-to-object-maximums[official documentation]. -==== - -For example, it is generally not recommended to have more than 10,000 namespaces due to potential performance and management overhead. In {prod}, each user is allocated a namespace. If you expect the user base to be large, consider spreading workloads across multiple "fit-for-purpose" clusters and potentially leveraging solutions for multi-cluster orchestration. - -.Resource requirements - -When deploying {prod} on {kubernetes}, -it is crucial to accurately calculate the resource requirements -and determine memory and CPU / GPU needs for each CDE to come up with the right sizing of the cluster. -In general, the CDE size is limited by and cannot be bigger than the worker node size. -The resource requirements for CDEs can vary significantly based on the specific workloads and configurations. -For example, a simple CDE might require only a few hundred megabytes of memory, -while a more complex one will need several gigabytes of memory and multiple CPU cores. - -include::example$snip_{project-context}-running-at-scale-calculate-resources.adoc[] - -.Using etcd - -The primary datastore of {kubernetes} cluster configuration and state is link:https://etcd.io/[etcd]. It holds the cluster state and configuration, including information about nodes, pods, services, and custom resources. As a distributed key-value store, etcd does not scale well past a certain threshold, and as the size of etcd grows, so does the load on the cluster, risking its stability. - -[IMPORTANT] -==== -The default etcd size is 2 GB, and the recommended maximum is 8 GB. Exceeding the maximum limit can make the {kubernetes} cluster unstable and unresponsive. -==== - -.Object size as a factor - -Not only the overall number, but also the size of the objects stored in etcd is a critical factor that can significantly impact its performance and stability. Each object stored in etcd consumes space, and as the number of objects increases, the overall size of etcd grows too. The larger the object is, the more space it takes in etcd. For example, etcd can be overloaded with just a few thousand of relatively big {kubernetes} objects. - -[IMPORTANT] -==== -Even though the data stored in a `ConfigMap` cannot exceed 1 MiB by design, a few thousand of relatively big `ConfigMap` objects can overload etcd storage. -==== - -In the context of {prod}, by default the Operator creates and manages the 'ca-certs-merged' ConfigMap, which contains the Certificate Authorities (CAs) bundle, in every user namespace. With a large number of TLS certificates in the cluster, this results in additional etcd usage. - -To disable mounting the CA bundle by using the `ConfigMap` under the `/etc/pki/ca-trust/extracted/pem` path, -configure the `CheCluster` Custom Resource by setting the `disableWorkspaceCaBundleMount` property to `true`. -With this configuration, only custom certificates will be mounted under the path `/public-certs`: - -[source,yaml] ----- -spec: - devEnvironments: - trustedCerts: - disableWorkspaceCaBundleMount: true ----- - -.{devworkspace} objects - -For large {kubernetes} deployments, particularly those involving a high number of custom resources such as `DevWorkspace` objects, which represent CDEs, etcd can become a significant performance bottleneck. - -[IMPORTANT] -==== -Based on the load testing for 6,000 `DevWorkspace` objects, storage consumption for etcd was approximately 2.5GB. -==== - -Starting from {devworkspace} Operator version 0.34.0, -you can configure a pruner -that automatically cleans up `DevWorkspace` objects that were not in use for a certain period of time. -To set the pruner up, configure the `DevWorkspaceOperatorConfig` object as follows: - -[source,yaml] ----- -apiVersion: controller.devfile.io/v1alpha1 -kind: DevWorkspaceOperatorConfig -metadata: - name: devworkspace-operator-config - namespace: crw -config: - workspace: - cleanupCronJob: - enabled: true - dryRun: false - retainTime: 2592000 <1> - schedule: “0 0 1 * *” <2> ----- -<1> By default, if a workspace was not started for more than 30 days it will be marked for deletion. -<2> By default, the pruner will run once per month. - -.OLMConfig - -When an Operator is installed by the link:https://olm.operatorframework.io/[Operator Lifecycle Manager (OLM)], -a stripped-down copy of its CSV is created in every {namespace} the Operator is configured to watch. -These stripped-down CSVs are known as “Copied CSVs” -and communicate to users which controllers are actively reconciling resource events in a given namespace. -On especially large clusters, with namespaces and installed operators tending in the hundreds or thousands, -Copied CSVs consume an unsustainable amount of resources; for example, OLM memory usage, cluster etcd limits, -networking, etc. To eliminate the CSVs copied to every namespace, configure the `OLMConfig` object accordingly: - -[source,yaml] ----- -apiVersion: operators.coreos.com/v1 -kind: OLMConfig -metadata: - name: cluster -spec: - features: - disableCopiedCSVs: true ----- - -[NOTE] -==== -Additional information about the `disableCopiedCSVs` feature is available in its original link:https://github.com/operator-framework/enhancements/blob/master/enhancements/olm-toggle-copied-csvs.md[enhancement proposal]. -==== - -The primary impact of the `disableCopiedCSVs` property on etcd is related to resource consumption. In clusters with a large number of namespaces and many cluster-wide Operators, the creation and maintenance of numerous Copied CSVs can lead to increased etcd storage usage and memory consumption. By disabling Copied CSVs, the amount of data stored in etcd is significantly reduced, which can help improve overall cluster performance and stability. - -This is particularly important for large clusters where the number of namespaces and operators can quickly add up to a significant amount of data. Disabling Copied CSVs can help reduce the load on etcd, leading to improved performance and responsiveness of the cluster. -Additionally, it can help reduce the memory footprint of OLM, as it no longer needs to maintain and manage these additional resources. - -[NOTE] -==== -You can find more details about "Disabling Copied CSVs" in the link:https://olm.operatorframework.io/docs/advanced-tasks/configuring-olm/#disabling-copied-csvs[official documentation]. -==== - -.Cluster Autoscaling - -Although cluster autoscaling is a powerful {kubernetes} feature, you cannot always fall back on it. You should always consider predictive scaling by analyzing load data on your environment to detect daily or weekly usage patterns. If your workloads follow a pattern and there are dramatic peaks throughout the day, you should consider provisioning worker nodes accordingly. For example, if you have a predictable load pattern where the number of workspaces increases during business hours and decreases during off-hours, you can use predictive scaling to adjust the number of worker nodes based on the expected load. -This can help ensure that you have enough resources available to handle the peak load while minimizing costs during off-peak hours. - -[NOTE] -==== -Consider leveraging open-source solutions such as link:https://karpenter.sh/[Karpenter] for configuration and lifecycle management of the worker nodes. Karpenter can dynamically provision and optimize worker nodes based on the specific requirements of the workloads, helping to improve resource utilization and reduce costs. -==== - -.Multi-cluster - -By design, {prod} is not multi-cluster aware, and you can only have one instance per cluster. -However, -you can run {prod} in a multi-cluster environment by deploying {prod} in each different cluster -and using a load balancer or DNS-based routing to direct traffic to the appropriate instance based on the user’s location or other criteria. -This approach can help -improve performance and reliability by distributing the workload across multiple clusters -and providing redundancy in case of cluster failures. - -.Developer Sandbox example - -You can test running {prod-short} in a multi-cluster environment by using the link:https://developers.redhat.com/developer-sandbox[Developer Sandbox], -a free trial environment by Red Hat with pre-configured tools and services. -From an infrastructure perspective, -the Developer Sandbox consists of multiple link:https://www.redhat.com/en/technologies/cloud-computing/openshift/aws[ROSA] clusters. -On each cluster, -the productized version of {prod} is installed -and configured using link:https://argo-cd.readthedocs.io/en/stable/[Argo CD]. -Since the user base is spread across multiple clusters, -link:https://workspaces.openshift.com/[workspaces.openshift.com] is used as a single entry point to the productized {prod} instances. - -image::running-at-scale/snip_{project-context}-multi-cluster.png[Scheme of a multi-cluster environment] - -You can find implementation details about the multicluster redirector in the following link:https://github.com/codeready-toolchain/crw-multicluster-redirector[GitHub repository]. - -[IMPORTANT] -==== -The multi-cluster architecture of link:https://workspaces.openshift.com/[workspaces.openshift.com] is part of the link:https://developers.redhat.com/developer-sandbox[Developer Sandbox]. -It is a Developer Sandbox-specific solution that cannot be reused as-is in other environments. However, you can use it as a reference for implementing a similar solution well-tailored to your specific multicluster needs. -==== - -.The multicluster redirector solution for Red Hat OpenShift - -Red Hat offers an open-source, Quarkus-based service that acts as a single gateway for developers. This service automatically redirects users to the correct {prod} instance on the appropriate cluster based on their Red Hat OpenShift group membership. The community-supported version is available in the link:https://github.com/redhat-developer/devspaces-multicluster-redirector[GitHub repository]. - -.Architecture and requirements - -A critical requirement for the multicluster redirector is that all users must be provisioned to the host cluster where the redirector is deployed. Users authenticate through the OAuth flow of this cluster, even if they never run workloads there. The host cluster's Red Hat OpenShift groups determine the routing logic. See the link:https://github.com/redhat-developer/devspaces-multicluster-redirector?tab=readme-ov-file#-deploying-to-openshift[Deploying to Red Hat OpenShift] section for deployment instructions. - -.Configuration - -The routing configuration uses a `ConfigMap` that contains JSON to map Red Hat OpenShift groups to {prod} URLs. The redirector uses this file to update routing tables in real-time without requiring restarts. - -.Operational flow - -The routing process follows these steps: - -. Authenticate via OAuth by using a proxy sidecar. -. Pass identity and group information through HTTP headers. -. Verify group memberships by using Red Hat OpenShift API queries. -. Determine the appropriate {prod} URL by using a mapping lookup. -. Redirect the user to the designated cluster instance. - -If users belong to multiple Red Hat OpenShift groups, they can choose their desired {prod} instance from a selection dashboard. - -.Additional resources - -* link:https://che.eclipseprojects.io/2025/04/29/@ilya.buziuk-running-at-scale.html[Running at scale] -* link:https://developers.redhat.com/articles/2026/01/23/enterprise-multi-cluster-scalability-openshift-dev-spaces[Enterprise multi-cluster scalability] diff --git a/modules/administration-guide/pages/upgrading-che-using-the-cli-management-tool.adoc b/modules/administration-guide/pages/upgrading-che-using-the-cli-management-tool.adoc index 2a76ac8e76..0ea3bd4b43 100644 --- a/modules/administration-guide/pages/upgrading-che-using-the-cli-management-tool.adoc +++ b/modules/administration-guide/pages/upgrading-che-using-the-cli-management-tool.adoc @@ -15,7 +15,7 @@ This section describes how to upgrade from the previous minor version using the * A running instance of a previous minor version of {prod-prev-short}, installed using the CLI management tool on the same instance of {platforms-name}, in the `{prod-namespace}` {platforms-namespace}. -* `{prod-cli}` for {prod-short} version {prod-ver}. See: xref:installing-the-chectl-management-tool.adoc[]. +* `{prod-cli}` for {prod-short} version {prod-ver}. See: xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/pages/using-chectl-to-configure-the-checluster-custom-resource-during-installation.adoc b/modules/administration-guide/pages/using-chectl-to-configure-the-checluster-custom-resource-during-installation.adoc index 995609a9c0..f2f5c8a4b2 100644 --- a/modules/administration-guide/pages/using-chectl-to-configure-the-checluster-custom-resource-during-installation.adoc +++ b/modules/administration-guide/pages/using-chectl-to-configure-the-checluster-custom-resource-during-installation.adoc @@ -14,7 +14,7 @@ To deploy {prod-short} with a suitable configuration, edit the `CheCluster` Cust * An active `{orch-cli}` session with administrative permissions to the {orch-name} cluster. See {orch-cli-link}. -* `{prod-cli}`. See: xref:installing-the-chectl-management-tool.adoc[]. +* `{prod-cli}`. See: xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure * Create a `che-operator-cr-patch.yaml` YAML file that contains the subset of the `CheCluster` Custom Resource to configure: diff --git a/modules/administration-guide/partials/proc_configuring-routes.adoc b/modules/administration-guide/partials/proc_configuring-routes.adoc index 11fc42e164..982b400d02 100644 --- a/modules/administration-guide/partials/proc_configuring-routes.adoc +++ b/modules/administration-guide/partials/proc_configuring-routes.adoc @@ -8,7 +8,7 @@ You can configure labels, annotations, and domains for OpenShift Route to work w * An active `oc` session with administrative permissions to the OpenShift cluster. See link:https://docs.openshift.com/container-platform/{ocp4-ver}/cli_reference/openshift_cli/getting-started-cli.html[Getting started with the OpenShift CLI]. -* `{prod-cli}`. See: xref:installing-the-chectl-management-tool.adoc[]. +* `{prod-cli}`. See: xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/administration-guide/partials/snip_preparing-images-for-a-restricted-environment.adoc b/modules/administration-guide/partials/snip_preparing-images-for-a-restricted-environment.adoc index c9fe34af04..432353a9e8 100644 --- a/modules/administration-guide/partials/snip_preparing-images-for-a-restricted-environment.adoc +++ b/modules/administration-guide/partials/snip_preparing-images-for-a-restricted-environment.adoc @@ -28,7 +28,7 @@ * An active `skopeo` session with administrative access to the private Docker registry. link:https://github.com/containers/skopeo#authenticating-to-a-registry[Authenticating to a registry], and link:https://docs.redhat.com/en/documentation/openshift_container_platform/{ocp4-ver}/html/disconnected_environments/installing-mirroring-disconnected-about[Mirroring images for a disconnected installation]. -* `{prod-cli}` for {prod-short} version {prod-ver}. See xref:installing-the-chectl-management-tool.adoc[]. +* `{prod-cli}` for {prod-short} version {prod-ver}. See xref:plan:installing-the-chectl-management-tool.adoc[]. .Procedure diff --git a/modules/overview/pages/introduction-to-eclipse-che.adoc b/modules/overview/pages/introduction-to-eclipse-che.adoc index 1319cda5a0..50530118b4 100644 --- a/modules/overview/pages/introduction-to-eclipse-che.adoc +++ b/modules/overview/pages/introduction-to-eclipse-che.adoc @@ -25,7 +25,7 @@ * xref:hosted-che:hosted-che.adoc[] * xref:administration-guide:installing-che.adoc[] * xref:administration-guide:supported-platforms.adoc[] -* xref:administration-guide:calculating-che-resource-requirements.adoc[] +* xref:plan:calculating-che-resource-requirements.adoc[] == Joining the community diff --git a/modules/administration-guide/examples/proc_che-installing-the-chectl-management-tool-on-linux-or-macos.adoc b/modules/plan/examples/proc_che-installing-the-chectl-management-tool-on-linux-or-macos.adoc similarity index 100% rename from modules/administration-guide/examples/proc_che-installing-the-chectl-management-tool-on-linux-or-macos.adoc rename to modules/plan/examples/proc_che-installing-the-chectl-management-tool-on-linux-or-macos.adoc diff --git a/modules/administration-guide/examples/proc_che-installing-the-chectl-management-tool-on-windows.adoc b/modules/plan/examples/proc_che-installing-the-chectl-management-tool-on-windows.adoc similarity index 100% rename from modules/administration-guide/examples/proc_che-installing-the-chectl-management-tool-on-windows.adoc rename to modules/plan/examples/proc_che-installing-the-chectl-management-tool-on-windows.adoc diff --git a/modules/administration-guide/examples/snip_che-example-devfile-disclaimer.adoc b/modules/plan/examples/snip_che-example-devfile-disclaimer.adoc similarity index 100% rename from modules/administration-guide/examples/snip_che-example-devfile-disclaimer.adoc rename to modules/plan/examples/snip_che-example-devfile-disclaimer.adoc diff --git a/modules/administration-guide/examples/snip_che-running-at-scale-calculate-resources.adoc b/modules/plan/examples/snip_che-running-at-scale-calculate-resources.adoc similarity index 100% rename from modules/administration-guide/examples/snip_che-running-at-scale-calculate-resources.adoc rename to modules/plan/examples/snip_che-running-at-scale-calculate-resources.adoc diff --git a/modules/administration-guide/images/running-at-scale/snip_che-multi-cluster.png b/modules/plan/images/running-at-scale/snip_che-multi-cluster.png similarity index 100% rename from modules/administration-guide/images/running-at-scale/snip_che-multi-cluster.png rename to modules/plan/images/running-at-scale/snip_che-multi-cluster.png diff --git a/modules/plan/nav.adoc b/modules/plan/nav.adoc new file mode 100644 index 0000000000..dc9b4bd654 --- /dev/null +++ b/modules/plan/nav.adoc @@ -0,0 +1,5 @@ +.Plan + +* xref:calculating-che-resource-requirements.adoc[Calculate {prod-short} resource requirements] +* xref:running-at-scale.adoc[{prod-short} scalability] +* xref:installing-the-chectl-management-tool.adoc[Install the {prod-cli} management tool] diff --git a/modules/administration-guide/pages/calculating-che-resource-requirements.adoc b/modules/plan/pages/calculating-che-resource-requirements.adoc similarity index 66% rename from modules/administration-guide/pages/calculating-che-resource-requirements.adoc rename to modules/plan/pages/calculating-che-resource-requirements.adoc index 9c1b4e3b7d..efe00b0675 100644 --- a/modules/administration-guide/pages/calculating-che-resource-requirements.adoc +++ b/modules/plan/pages/calculating-che-resource-requirements.adoc @@ -1,12 +1,15 @@ :_content-type: PROCEDURE -:description: Calculating {prod-short} resource requirements -:keywords: administration-guide, calculating-che-resource-requirements -:navtitle: Calculating {prod-short} resource requirements -:page-aliases: .:calculating-che-resource-requirements.adoc +:description: Calculate the CPU and memory requirements for the {prod-short} Operator, {devworkspace} Controller, and user workspaces. Accurate resource estimates ensure your cluster can handle the expected number of concurrent users without performance issues. +:keywords: plan, calculating-che-resource-requirements +:navtitle: Calculate {prod-short} resource requirements +:page-aliases: administration-guide:calculating-che-resource-requirements.adoc [id="calculating-{prod-id-short}-resource-requirements"] -= Calculating {prod-short} resource requirements += Calculate {prod-short} resource requirements + +[role="_abstract"] +Calculate the CPU and memory requirements for the {prod-short} Operator, {devworkspace} Controller, and user workspaces. Accurate resource estimates ensure your cluster can handle the expected number of concurrent users without performance issues. The {prod-short} Operator, {devworkspace} Controller, and user workspaces consist of a set of pods. The pods contribute to the resource consumption in CPU and memory limits and requests. @@ -15,26 +18,21 @@ include::example$snip_{project-context}-example-devfile-disclaimer.adoc[] .Procedure -. Identify the workspace resource requirements which depend on the devfile that is used for defining the development environment. This includes identifying the workspace components explicitly specified in the `components` section of the devfile. +. Identify the workspace resource requirements from the devfile `components` section. The following example uses the link:https://github.com/che-incubator/quarkus-api-example/blob/main/devfile.yaml[Quarkus API example devfile]. -** Here is an link:https://github.com/che-incubator/quarkus-api-example/blob/main/devfile.yaml[example devfile] with the following components: +** The `tools` component of the devfile defines the following requests and limits: + -.`tools` -==== -The `tools` component of the devfile defines the following requests and limits: - -[source,yaml,subs="+attributes,+quotes,macros"] +[source,yaml] ---- memoryLimit: 6G memoryRequest: 512M cpuRequest: 1000m cpuLimit: 4000m ---- -==== -** During the workspace startup, an internal `che-gateway` container is implicitly provisioned with the following requests and limits: +** During workspace startup, an internal `che-gateway` container is implicitly provisioned with the following requests and limits: + -[source,yaml,subs="+attributes,+quotes,macros"] +[source,yaml] ---- memoryLimit: 256M memoryRequest: 64M @@ -42,9 +40,9 @@ The `tools` component of the devfile defines the following requests and limits: cpuLimit: 500m ---- -** IMPORTANT: Additional memory and CPU will be added implicitly for the Visual Studio Code - Open Source ("Code - OSS"): +** Additional memory and CPU are added implicitly for the Visual Studio Code - Open Source ("Code - OSS") editor: + -[source,yaml,subs="+attributes,+quotes,macros"] +[source,yaml] ---- memoryLimit: 1024M memoryRequest: 256M @@ -52,9 +50,9 @@ The `tools` component of the devfile defines the following requests and limits: cpuLimit: 500m ---- -** IMPORTANT: Additional memory and CPU will be added implicitly for a JetBrains IDE (for example IntelliJ IDEA Ultimate): +** Additional memory and CPU are added implicitly for a JetBrains IDE, for example IntelliJ IDEA Ultimate: + -[source,yaml,subs="+attributes,+quotes,macros"] +[source,yaml] ---- memoryLimit: 6144M memoryRequest: 2048M @@ -65,7 +63,6 @@ The `tools` component of the devfile defines the following requests and limits: . Calculate the sums of the resources required for each workspace. If you intend to use multiple devfiles, repeat this calculation for every expected devfile. + .Workspace requirements for the link:https://github.com/che-incubator/quarkus-api-example/blob/main/devfile.yaml[example devfile] in the previous step -==== [cols="1a,1a,1a,1a,1a,1a,1a",options="header"] |=== |Purpose @@ -106,7 +103,6 @@ The `tools` component of the devfile defines the following requests and limits: >s|5000 m >s|1080 m |=== -==== . Multiply the resources calculated per workspace by the number of workspaces that you expect all of your users to run simultaneously. @@ -203,14 +199,6 @@ The `tools` component of the devfile defines the following requests and limits: |N/A |N/A -|{devworkspace} webhook server -|`{devworkspace-id}-webhook-server` -|`webhook-server` -|300 MiB -|20 MiB -|200 m -|100 m - |{devworkspace} Operator Catalog |`{devworkspace-id}-operator-catalog` |`registry-server` @@ -236,13 +224,19 @@ The `tools` component of the devfile defines the following requests and limits: |N/A 3+>s|Total ->s|13 GiB ->s|1.2 GiB ->s|8.4 ->s|1.2 +>s|12.3 GiB +>s|1.1 GiB +>s|8.2 +>s|1.1 |=== +. Add the workspace resources from step 3 and the operator resources from step 4 to determine total cluster resource requirements. + +.Verification + +* Verify that the total resource requirements account for all {prod-short} Operator components, {devworkspace} Controller components, and the expected number of concurrent workspaces. + .Additional resources * link:https://devfile.io/docs/2.2.0/what-is-a-devfile[What is a devfile] diff --git a/modules/plan/pages/installing-the-chectl-management-tool.adoc b/modules/plan/pages/installing-the-chectl-management-tool.adoc new file mode 100644 index 0000000000..e368f85515 --- /dev/null +++ b/modules/plan/pages/installing-the-chectl-management-tool.adoc @@ -0,0 +1,22 @@ +:_content-type: ASSEMBLY +:description: Install {prod-cli}, the {prod} CLI management tool, on Linux, macOS, or Windows. The {prod-cli} tool lets you deploy, update, and manage {prod-short} from the command line. +:keywords: plan, installing-the-chectl-management-tool +:navtitle: Install the {prod-cli} management tool +:page-aliases: administration-guide:installing-the-chectl-management-tool.adoc, installation-guide:using-the-chectl-management-tool.adoc, overview:using-the-chectl-management-tool.adoc, using-the-chectl-management-tool.adoc + + +[id="installing-the-{prod-cli}-management-tool"] += Install the {prod-cli} management tool + +[role="_abstract"] +Install `{prod-cli}`, the {prod} CLI management tool, on Linux, macOS, or Windows. The `{prod-cli}` tool lets you deploy, update, and manage {prod-short} from the command line. + +include::example$proc_{project-context}-installing-the-chectl-management-tool-on-windows.adoc[leveloffset=+1] + +include::example$proc_{project-context}-installing-the-chectl-management-tool-on-linux-or-macos.adoc[leveloffset=+1] + + +.Additional resources + +* {link-prod-cli-documentation} + diff --git a/modules/plan/pages/running-at-scale.adoc b/modules/plan/pages/running-at-scale.adoc new file mode 100644 index 0000000000..f80bd137f2 --- /dev/null +++ b/modules/plan/pages/running-at-scale.adoc @@ -0,0 +1,205 @@ +:_content-type: CONCEPT +:description: Scaling cloud development environments (CDEs) to thousands of concurrent workspaces imposes high infrastructure demands on etcd storage, Operator memory, and worker node capacity. This topic covers the bottlenecks, tested maximums, and architectural patterns — including multi-cluster deployments — that address these challenges. +:keywords: plan, scale, infrastructure, workload, scalability, CDE, cloud +:navtitle: {prod-short} scalability +:page-aliases: administration-guide:running-at-scale.adoc + +[id="running-at-scale"] += {prod-short} scalability + +[role="_abstract"] +Scaling cloud development environments (CDEs) to thousands of concurrent workspaces imposes high infrastructure demands on etcd storage, Operator memory, and worker node capacity. This topic covers the bottlenecks, tested maximums, and architectural patterns — including multi-cluster deployments — that address these challenges. + +Such a scale imposes high infrastructure demands and introduces potential bottlenecks that can impact performance and stability. Addressing these challenges requires meticulous planning, strategic architectural choices, monitoring, and continuous optimization. + +CDE workloads are particularly complex to scale. The underlying IDE solutions, such as link:https://github.com/microsoft/vscode[Visual Studio Code - Open Source ("Code - OSS")] or link:https://www.jetbrains.com/remote-development/gateway/[JetBrains Gateway], are designed as single-user applications, not as multitenant services. + +== Resource quantity and object maximums + +While there is no strict limit on the number of resources in a {kubernetes} cluster, there are certain link:https://kubernetes.io/docs/setup/best-practices/cluster-large/[considerations for large clusters] to remember. + +link:https://www.redhat.com/en/technologies/cloud-computing/openshift[OpenShift Container Platform], a certified distribution of {kubernetes}, provides a set of tested maximums for various resources. These maximums can serve as an initial guideline for planning your environment: + +.OpenShift Container Platform tested cluster maximums +[%header,format=csv] +|=== +Resource type, Tested maximum +Number of nodes,2000 +Number of pods,150000 +Number of pods per node,2500 +Number of namespace,10000 +Number of services,10000 +Number of secrets,80000 +Number of config maps,90000 +|=== + +For more details on OpenShift Container Platform tested object maximums, see the link:https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/scalability_and_performance/planning-your-environment-according-to-object-maximums#planning-your-environment-according-to-object-maximums[OpenShift Container Platform scalability and performance documentation]. + +For example, it is generally not recommended to have more than 10,000 namespaces due to potential performance and management overhead. In {prod}, each user is allocated a namespace. If you expect the user base to be large, consider spreading workloads across multiple "fit-for-purpose" clusters and potentially using solutions for multi-cluster orchestration. + +== Resource requirements + +When deploying {prod} on {kubernetes}, accurately calculate the resource requirements for each CDE, including memory and CPU or GPU needs. This determines the right sizing of the cluster. In general, the CDE size is limited by and cannot be bigger than the worker node size. + +The resource requirements for CDEs can vary significantly based on the specific workloads and configurations. A simple CDE might require only a few hundred megabytes of memory. A more complex one might need several gigabytes of memory and multiple CPU cores. + +include::example$snip_{project-context}-running-at-scale-calculate-resources.adoc[] + +== Using etcd + +The primary datastore of {kubernetes} cluster configuration and state is link:https://etcd.io/[etcd]. It holds information about nodes, pods, services, and custom resources. + +As a distributed key-value store, etcd does not scale well past a certain threshold. As the size of etcd grows, so does the load on the cluster, risking its stability. + +[IMPORTANT] +==== +The default etcd size is 2 GB, and the recommended maximum is 8 GB. Exceeding the maximum limit can make the {kubernetes} cluster unstable and unresponsive. Even though the data stored in a `ConfigMap` cannot exceed 1 MiB by design, a few thousand relatively large `ConfigMap` objects can overload etcd storage. +==== + +== Object size as a factor + +The size of the objects stored in etcd is also a critical factor. Each object consumes space, and as the number of objects increases, the overall size of etcd grows. The larger the object, the more space it takes. For example, etcd can be overloaded with only a few thousand large {kubernetes} objects. + +In the context of {prod}, by default the Operator creates and manages the 'ca-certs-merged' ConfigMap, which contains the Certificate Authorities (CAs) bundle, in every user namespace. With a large number of Transport Layer Security (TLS) certificates in the cluster, this results in additional etcd usage. + +To disable mounting the CA bundle by using the `ConfigMap` under the `/etc/pki/ca-trust/extracted/pem` path, configure the `CheCluster` Custom Resource by setting the `disableWorkspaceCaBundleMount` property to `true`. With this configuration, only custom certificates are mounted under the path `/public-certs`: + +[source,yaml] +---- +spec: + devEnvironments: + trustedCerts: + disableWorkspaceCaBundleMount: true +---- + +== {devworkspace} objects + +For large {kubernetes} deployments, particularly those involving a high number of custom resources such as `DevWorkspace` objects, which represent CDEs, etcd can become a significant performance bottleneck. + +[IMPORTANT] +==== +Based on the load testing for 6,000 `DevWorkspace` objects, storage consumption for etcd was approximately 2.5GB. +==== + +Starting from {devworkspace} Operator version 0.34.0, you can configure a pruner that automatically cleans up `DevWorkspace` objects that were not in use for a certain period of time. To set the pruner up, configure the `DevWorkspaceOperatorConfig` object as follows: + +[source,yaml] +---- +apiVersion: controller.devfile.io/v1alpha1 +kind: DevWorkspaceOperatorConfig +metadata: + name: devworkspace-operator-config + namespace: crw +config: + workspace: + cleanupCronJob: + enabled: true + dryRun: false + retainTime: 2592000 + schedule: “0 0 1 * *” +---- + +retainTime:: By default, if a workspace was not started for more than 30 days, it is marked for deletion. + +schedule:: By default, the pruner runs once per month. + +== OLMConfig + +When an Operator is installed by the link:https://olm.operatorframework.io/[Operator Lifecycle Manager (OLM)], a stripped-down copy of its CSV is created in every {namespace} the Operator watches. These “Copied CSVs” communicate which controllers are reconciling resource events in a given namespace. + +On large clusters with hundreds or thousands of namespaces, Copied CSVs consume an unsustainable amount of resources, including OLM memory, etcd storage, and network bandwidth. To eliminate the CSVs copied to every namespace, configure the `OLMConfig` object: + +[source,yaml] +---- +apiVersion: operators.coreos.com/v1 +kind: OLMConfig +metadata: + name: cluster +spec: + features: + disableCopiedCSVs: true +---- + +Additional information about the `disableCopiedCSVs` feature is available in its original link:https://github.com/operator-framework/enhancements/blob/master/enhancements/olm-toggle-copied-csvs.md[enhancement proposal]. + +In clusters with many namespaces and cluster-wide Operators, Copied CSVs increase etcd storage usage and memory consumption. Disabling Copied CSVs significantly reduces the data stored in etcd and improves cluster performance and stability. + +Disabling Copied CSVs also reduces the memory footprint of OLM, as it no longer maintains these additional resources. + +For more details about disabling Copied CSVs, see the link:https://olm.operatorframework.io/docs/advanced-tasks/configuring-olm/#disabling-copied-csvs[OLM documentation]. + +== Cluster Autoscaling + +Although cluster autoscaling is a powerful {kubernetes} feature, you cannot always rely on it. Consider predictive scaling by analyzing load data to detect daily or weekly usage patterns. + +If your workloads follow a pattern with dramatic peaks throughout the day, provision worker nodes accordingly. For example, if workspaces increase during business hours and decrease during off-hours, predictive scaling adjusts the number of worker nodes. This ensures enough resources are available during peak load while minimizing costs during off-peak hours. + +You can also use open-source solutions such as link:https://karpenter.sh/[Karpenter] for configuration and lifecycle management of the worker nodes. Karpenter can dynamically provision and optimize worker nodes based on the specific requirements of the workloads. This helps improve resource utilization and reduce costs. + +== Multi-cluster + +By design, {prod} is not multi-cluster aware. You can only have one instance per cluster. + +However, you can run {prod} in a multi-cluster environment by deploying {prod} in each cluster. Use a load balancer or Domain Name System (DNS)-based routing to direct traffic to the appropriate instance. This approach distributes the workload across clusters and provides redundancy in case of cluster failures. + +== Developer Sandbox example + +You can test running {prod-short} in a multi-cluster environment by using the link:https://developers.redhat.com/developer-sandbox[Developer Sandbox], a free trial environment by Red Hat. + +From an infrastructure perspective, the Developer Sandbox consists of multiple link:https://www.redhat.com/en/technologies/cloud-computing/openshift/aws[Red Hat OpenShift Service on AWS (ROSA)] clusters. On each cluster, the productized version of {prod} is installed and configured using link:https://argo-cd.readthedocs.io/en/stable/[Argo CD]. The link:https://workspaces.openshift.com/[workspaces.openshift.com] URL is used as a single entry point to the {prod} instances across clusters. + +image::running-at-scale/snip_{project-context}-multi-cluster.png[Scheme of a multi-cluster environment] + +You can find implementation details about the multicluster redirector in the link:https://github.com/codeready-toolchain/crw-multicluster-redirector[crw-multicluster-redirector GitHub repository]. + +[IMPORTANT] +==== +The multi-cluster architecture of link:https://workspaces.openshift.com/[workspaces.openshift.com] is part of the link:https://developers.redhat.com/developer-sandbox[Developer Sandbox]. It is a Developer Sandbox-specific solution that cannot be reused as-is in other environments. However, you can use it as a reference for implementing a similar solution well-tailored to your specific multicluster needs. +==== + +== The multicluster redirector solution for OpenShift Container Platform + +Red Hat offers an open-source, Quarkus-based service that acts as a single gateway for developers. This service automatically redirects users to the correct {prod} instance on the appropriate cluster based on their OpenShift Container Platform group membership. The community-supported version is available in the link:https://github.com/redhat-developer/devspaces-multicluster-redirector[devspaces-multicluster-redirector GitHub repository]. + +== Architecture and requirements + +A critical requirement for the multicluster redirector is that all users are provisioned to the host cluster where the redirector is deployed. Users authenticate through the OAuth flow of this cluster, even if they never run workloads there. The host cluster’s OpenShift Container Platform groups determine the routing logic. See the link:https://github.com/redhat-developer/devspaces-multicluster-redirector?tab=readme-ov-file#-deploying-to-openshift[devspaces-multicluster-redirector documentation] for deployment instructions. + +== Configuration + +The routing configuration uses a `ConfigMap` that contains JSON to map OpenShift Container Platform groups to {prod} URLs. The redirector uses this file to update routing tables in real-time without requiring restarts. + +== Operational flow + +The routing process follows these steps: + +. Authenticate by using OAuth through a proxy sidecar. +. Pass identity and group information through HTTP headers. +. Verify group memberships by using OpenShift Container Platform API queries. +. Determine the appropriate {prod} URL by using a mapping lookup. +. Redirect the user to the designated cluster instance. + +If users belong to multiple OpenShift Container Platform groups, they can choose their desired {prod} instance from a selection dashboard. + +.Additional resources + +* link:https://che.eclipseprojects.io/2025/04/29/@ilya.buziuk-running-at-scale.html[Running at scale] +* link:https://developers.redhat.com/articles/2026/01/23/enterprise-multi-cluster-scalability-openshift-dev-spaces[Enterprise multi-cluster scalability] +* link:https://kubernetes.io/[Kubernetes] +* link:https://github.com/microsoft/vscode[Visual Studio Code - Open Source ("Code - OSS")] +* link:https://www.jetbrains.com/remote-development/gateway/[JetBrains Gateway] +* link:https://kubernetes.io/docs/setup/best-practices/cluster-large/[Considerations for large clusters] +* link:https://www.redhat.com/en/technologies/cloud-computing/openshift[OpenShift Container Platform] +* link:https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/scalability_and_performance/planning-your-environment-according-to-object-maximums#planning-your-environment-according-to-object-maximums[OpenShift Container Platform tested object maximums] +* xref:calculating-che-resource-requirements.adoc[] +* link:https://etcd.io/[etcd] +* link:https://olm.operatorframework.io/[Operator Lifecycle Manager (OLM)] +* link:https://github.com/operator-framework/enhancements/blob/master/enhancements/olm-toggle-copied-csvs.md[OLM toggle Copied CSVs enhancement proposal] +* link:https://olm.operatorframework.io/docs/advanced-tasks/configuring-olm/#disabling-copied-csvs[Disabling Copied CSVs in OLM] +* link:https://karpenter.sh/[Karpenter] +* link:https://developers.redhat.com/developer-sandbox[Developer Sandbox] +* link:https://www.redhat.com/en/technologies/cloud-computing/openshift/aws[Red Hat OpenShift Service on AWS (ROSA)] +* link:https://argo-cd.readthedocs.io/en/stable/[Argo CD] +* link:https://workspaces.openshift.com/[workspaces.openshift.com] +* link:https://github.com/codeready-toolchain/crw-multicluster-redirector[crw-multicluster-redirector GitHub repository] +* link:https://github.com/redhat-developer/devspaces-multicluster-redirector[devspaces-multicluster-redirector GitHub repository]