From 35e794ea11428ceb5ca3a3df993dd6754d26e513 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 15:41:43 +0200 Subject: [PATCH 01/20] tilt implementation --- deployment-configuration/tilt-deploy.ext | 67 ++++++ .../templates/tilt/tilt-template.tpl | 56 +++++ .../deployment-cli-tools/ch_cli_tools/tilt.py | 216 ++++++++++++++++++ 3 files changed, 339 insertions(+) create mode 100644 deployment-configuration/tilt-deploy.ext create mode 100644 tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl create mode 100644 tools/deployment-cli-tools/ch_cli_tools/tilt.py diff --git a/deployment-configuration/tilt-deploy.ext b/deployment-configuration/tilt-deploy.ext new file mode 100644 index 000000000..1d1ed04a7 --- /dev/null +++ b/deployment-configuration/tilt-deploy.ext @@ -0,0 +1,67 @@ +load('ext://namespace', 'namespace_create', "namespace_inject") + + +def deploy(name, namespace, extra_env, watch): + + # create namespaces + namespace_create(namespace) + + # load helm chart + yaml = decode_yaml_stream(helm( + "deployment/helm", + # The release name, equivalent to helm --name + name=name, + # The namespace to install in, equivalent to helm --namespace + namespace=namespace, + # The values file to substitute into the chart. + values=["deployment/helm/values.yaml"], + # Values to set from the command-line + set=["service.port=1234", "ingress.enabled=true"] + ) + ) + + source_root = os.path.abspath(os.getcwd()) + # modify deployments + for r in yaml: + if r.get("kind") == "Deployment": + deployment_name = r["metadata"]["name"] + print("+ patching deployment:", deployment_name) + r["spec"]["template"]["spec"].setdefault("volumes", []).append({ + "name": name + "-root", + "hostPath": { + "path": source_root + } + }) + for container in r["spec"]["template"]["spec"]["containers"]: + print(" + modifying container:", container["name"]) + print(" - add " + name + " root folder") + container.setdefault("volumeMounts", []).append({ + "mountPath": "/usr/src/" + name, + "name": name + "-root" + }) + if "resources" in container: + print(" - modifying resource requests and limits") + if "limits" not in container["resources"]: + container["resources"]["limits"] = {} + if "requests" not in container["resources"]: + container["resources"]["requests"] = {} + container["resources"]["requests"]["cpu"] = "100m" + container["resources"]["requests"]["memory"] = "256Mi" + container["resources"]["limits"]["cpu"] = "8000m" + container["resources"]["limits"]["memory"] = "4096Mi" + + if deployment_name in extra_env and len(extra_env[deployment_name]) > 0: + print("Adding tasks images dependencies to env ", deployment_name) + for env in extra_env[deployment_name]: + container["env"].append({ + "name": env, "value": env + }) + + if not watch: + # don't watch mnp folder + watch_settings(ignore=source_root) + else: + print("Watching for file changes") + + # install applications + k8s_yaml(namespace_inject(encode_yaml_stream(yaml), namespace)) diff --git a/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl new file mode 100644 index 000000000..7c0ec06c2 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl @@ -0,0 +1,56 @@ +load('{{ch_root}}/deployment-configuration/tilt-deploy.ext', 'deploy') +load('ext://uibutton', 'cmd_button') + +config.define_bool('setup-infrastructure') +config.define_bool('watch') +cfg = config.parse() +setup_infrastructure = cfg.get('setup-infrastructure', False) +watch = cfg.get('watch', False) +if setup_infrastructure: + # setup ingress + print("Installing ingress controller") + # local("cd infrastructure/cluster-configuration && source cluster-init.sh") + local("kubectl get namespace ingress-nginx 2>/dev/null 1>/dev/null || bash -c 'helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version v4.2.5 && sleep 10'") + # print("Let's wait a few seconds...") + # local("sleep 30") +else: + print("To setup the infrastructure (f.e. ingress controller)") + print("run: tilt up -- --setup-infrastructure") + +if not watch: + print("To watch file changes, run: tilt up -- --watch") + + +# build images +{% for image in images -%} +docker_build(ref='{{image.image}}', context='{{image.context}}', dockerfile='{{image.docker.dockerfile}}', build_args={{image.docker.buildArgs}}{% if image.is_task %}, match_in_env_vars=True{% endif %}) +{% endfor %} + +extra_env = {} +{% for image in images -%} +{% if image.is_app -%} +extra_env.setdefault("{{ image.name }}", []) +{% for task in images -%} +{% if task.is_task and task.parent_app_name == image.name -%} +extra_env["{{ image.name }}"].append("{{ task.image }}") +{% endif -%} +{% endfor -%} +{% endif -%} +{% endfor %} + +# deploy +deploy(name='{{name}}', namespace='{{namespace}}', extra_env=extra_env, watch=watch) + +{% for app in apps -%} +# Add Tilt ui elements for: {{app.name}} +k8s_resource( + '{{app.app_key}}', + links=[link('http://{{app.name}}.{{domain}}', 'Open {{app.name}} page')] +) +cmd_button('{{app.app_key}}:set debug mode', + argv=["sh", "-c", "kubectl -n {{namespace}} patch deployment {{app.app_key}} --patch '{\"spec\": {\"template\": {\"spec\": {\"containers\": [{\"name\": \"{{app.app_key}}\", \"command\": [\"/bin/bash\"], \"args\": [\"-c\", \"sleep infinity\"], \"livenessProbe\": null, \"readinessProbe\": null}]}}}}'"], + resource='{{app.app_key}}', + icon_name='bug_report', + text='set debug mode', +) +{% endfor %} diff --git a/tools/deployment-cli-tools/ch_cli_tools/tilt.py b/tools/deployment-cli-tools/ch_cli_tools/tilt.py new file mode 100644 index 000000000..29738564e --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/tilt.py @@ -0,0 +1,216 @@ +import os +import logging +import json +import time + +from os.path import join, relpath, basename, exists, abspath +from jinja2 import Environment, PackageLoader, select_autoescape +from cloudharness_model import ApplicationTestConfig, HarnessMainConfig, GitDependencyConfig + +from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ + BASE_IMAGES_PATH, STATIC_IMAGES_PATH, HELM_ENGINE, COMPOSE_ENGINE +from .helm import KEY_APPS, KEY_HARNESS, KEY_DEPLOYMENT, KEY_TASK_IMAGES +from .utils import get_template, dict_merge, find_dockerfiles_paths, app_name_from_path, yaml, \ + find_file_paths, guess_build_dependencies_from_dockerfile, get_json_template, get_image_name + +from . import HERE, CH_ROOT + + +env = Environment( + loader=PackageLoader(package_name="ch_cli_tools", package_path="templates/tilt"), + autoescape=select_autoescape() +) + + +def relpath_if(p1, p2): + if os.path.isabs(p1): + return p1 + return relpath(p1, p2) + + +def get_all_images(helm_values: HarnessMainConfig) -> dict[str, str]: + all_images = {**helm_values[KEY_TASK_IMAGES]} + for app_name, app in helm_values.apps.items(): + if app.harness.deployment and app.harness.deployment.image: + all_images[app_name] = app.harness.deployment.image + return all_images + + +def create_tilt_configuration(root_paths, helm_values: HarnessMainConfig, manage_task_images=True, output_path='.', name='', namespace='', domain=''): + template_name = 'tilt-template.tpl' + tilt_template = env.get_template(template_name) + apps = helm_values.apps + artifacts = {} + overrides = {} + + all_images = get_all_images(helm_values) + + def remove_tag(image_name): + return image_name.split(":")[0] + + def get_image_tag(name): + return remove_tag(all_images[name]) + + builds = {} + + def build_artifact( + app_name: str, + context_path: str, + requirements: list[str] = None, + dockerfile_path: str = '', + additional_build_args: dict[str, str] = None, + is_app: bool = False, + is_task: bool = False, + parent_app_name: str = None, + app_key: str = None, + ) -> dict: + app_key = app_key or app_name + build_args = { + 'DEBUG': 'true' if helm_values.local or helm_values.debug else '' + } + + if additional_build_args: + build_args.update(additional_build_args) + + if requirements: + for req in requirements: + build_args.update({req.replace('-', '_').upper(): get_image_tag(req)}) + + image_name = get_image_tag(app_key) + + artifact_spec = { + 'name': app_name, + 'app_key': app_key, + 'image': image_name, + 'context': context_path, + 'is_app': is_app, + 'is_task': is_task, + 'parent_app_name': parent_app_name, + 'docker': { + 'dockerfile': join(dockerfile_path, 'Dockerfile'), + 'buildArgs': build_args + } + } + return artifact_spec + + base_images = set() + + def process_build_dockerfile( + dockerfile_path: str, + root_path: str, + global_context: bool = False, + requirements: list[str] = None, + app_name: str = None, + app_key: str = None, + is_app: bool = True + ) -> None: + if app_name is None: + app_name = app_name_from_path(basename(dockerfile_path)) + is_app = False + app_key = app_key or app_name + if app_key in helm_values.apps and not helm_values.apps[app_key]['build']: + return + if app_name in helm_values[KEY_TASK_IMAGES] or app_key in helm_values.apps: + context_path = relpath_if(root_path, output_path) if global_context else relpath_if(dockerfile_path, output_path) + + builds[app_name] = context_path + base_images.add(get_image_name(app_key)) + + artifacts[app_name] = build_artifact( + app_name, + context_path, + dockerfile_path=relpath(dockerfile_path, output_path), + requirements=requirements or guess_build_dependencies_from_dockerfile(dockerfile_path), + additional_build_args=get_additional_build_args(helm_values, app_key), + is_app=is_app, + app_key=app_key + ) + + if app_key in helm_values.apps and helm_values.apps[app_key].harness.dependencies and helm_values.apps[app_key].harness.dependencies.git: + artifacts[app_name]['hooks'] = { + 'before': [git_clone_hook(conf, context_path) for conf in helm_values.apps[app_key].harness.dependencies.git] + } + + images = set() + for root_path in root_paths: + base_dockerfiles = find_dockerfiles_paths( + join(root_path, BASE_IMAGES_PATH)) + + for dockerfile_path in base_dockerfiles: + process_build_dockerfile(dockerfile_path, root_path, global_context=True) + + static_images = set() + for root_path in root_paths: + static_dockerfiles = find_dockerfiles_paths( + join(root_path, STATIC_IMAGES_PATH)) + + for dockerfile_path in static_dockerfiles: + process_build_dockerfile(dockerfile_path, root_path) + + for root_path in root_paths: + apps_path = join(root_path, APPS_PATH) + + # Get all dockerfiles in the applications directory, including those in subdirectories + app_dockerfiles = find_dockerfiles_paths(apps_path) + + for dockerfile_path in app_dockerfiles: + app_relative_to_skaffold = os.path.relpath( + dockerfile_path, output_path) + app_relative_to_base = os.path.relpath(dockerfile_path, apps_path) + app_name = app_name_from_path(app_relative_to_base) + app_key = app_name + + if app_key not in apps: + if 'tasks' in app_relative_to_base and manage_task_images: + parent_app_name = app_name_from_path( + app_relative_to_base.split('/tasks')[0]) + parent_app_key = parent_app_name + + if parent_app_key in apps: + artifacts[app_key] = build_artifact( + app_name, + app_relative_to_skaffold, + guess_build_dependencies_from_dockerfile(dockerfile_path), + dockerfile_path=dockerfile_path, + is_task=True, + parent_app_name=parent_app_name) + elif app_name in helm_values[KEY_TASK_IMAGES]: + process_build_dockerfile(dockerfile_path, root_path, + requirements=guess_build_dependencies_from_dockerfile(dockerfile_path), dockerfile_path=dockerfile_path, app_name=app_name) + continue + + if app_name in helm_values["apps"]: + is_app = helm_values["apps"][app_name]["harness"]["service"]["auto"] + app_name = helm_values["apps"][app_name]["harness"]["subdomain"] + process_build_dockerfile(dockerfile_path, root_path, requirements=guess_build_dependencies_from_dockerfile(dockerfile_path), app_name=app_name, app_key=app_key, is_app=is_app) + app = apps[app_key] + if not app['build']: + continue + + images = [artifact for artifact in artifacts.values() if artifact['image']] + apps = [artifact for artifact in artifacts.values() if artifact['is_app']] + with open(os.path.join(output_path, 'Tiltfile'), "w") as f: + f.write(tilt_template.render(ch_root=CH_ROOT, name=name, namespace=namespace, images=images, apps=apps, domain=domain)) + return None + + +def git_clone_hook(conf: GitDependencyConfig, context_path: str): + return { + 'command': [ + 'sh', + join(os.path.dirname(os.path.dirname(HERE)), 'clone.sh'), + conf.branch_tag, + conf.url, + join(context_path, "dependencies", conf.path or os.path.basename(conf.url).split('.')[0]) + ] + } + + +def get_additional_build_args(helm_values: HarnessMainConfig, app_key: str) -> dict[str, str]: + if app_key not in helm_values.apps: + return None + + if not (helm_values.apps[app_key].harness.dockerfile and helm_values.apps[app_key].harness.dockerfile.buildArgs): + return None + + return helm_values.apps[app_key].harness.dockerfile.buildArgs From b371161f2964d9c889be9c8822c569408f8f1043 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 15:43:45 +0200 Subject: [PATCH 02/20] generate tilt configuration --- tools/deployment-cli-tools/harness-deployment | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment-cli-tools/harness-deployment b/tools/deployment-cli-tools/harness-deployment index 8befc3f25..a5aaf1b0a 100644 --- a/tools/deployment-cli-tools/harness-deployment +++ b/tools/deployment-cli-tools/harness-deployment @@ -8,6 +8,7 @@ from ch_cli_tools.dockercompose import create_docker_compose_configuration from ch_cli_tools.helm import create_helm_chart, deploy from ch_cli_tools.configurationgenerator import hosts_info from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_debug_configuration +from ch_cli_tools.tilt import create_tilt_configuration from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file from ch_cli_tools.preprocessing import preprocess_build_overrides, generate_hash_based_image_tags from ch_cli_tools.utils import merge_app_directories, merge_to_yaml_file @@ -154,6 +155,7 @@ if __name__ == "__main__": write_env_file(helm_values, os.path.join(root_paths[-1], DEPLOYMENT_PATH, ".env"), args.image_cache_url) create_skaffold_configuration(merged_root_paths, helm_values, backend_deploy=COMPOSE_ENGINE if args.docker_compose else HELM_ENGINE, env=envs) + create_tilt_configuration(merged_root_paths, helm_values, name=args.namespace, namespace=args.namespace, domain=args.domain) if not args.docker_compose: create_vscode_debug_configuration(root_paths, helm_values) From fb26db89df379ba888695d5e3168f10da25fa215 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 15:45:14 +0200 Subject: [PATCH 03/20] mongo and kc upgrade --- .../accounts/deploy/templates/_components.tpl | 4 +-- .../helm/templates/auto-database-mongo.yaml | 6 ++--- .../cloudharness_django/apps.py | 27 +++++++++++++++++++ .../cloudharness_django/services/events.py | 17 +++++++----- .../cloudharness/auth/keycloak.py | 4 +++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/applications/accounts/deploy/templates/_components.tpl b/applications/accounts/deploy/templates/_components.tpl index 4f8180d52..89bc16e3a 100644 --- a/applications/accounts/deploy/templates/_components.tpl +++ b/applications/accounts/deploy/templates/_components.tpl @@ -6,7 +6,7 @@ "subComponents": {}, "config": { "kc.user.profile.config": [ - "{\"attributes\":[{\"name\":\"username\",\"displayName\":\"${username}\",\"validations\":{\"length\":{\"min\":3,\"max\":255},\"username-prohibited-characters\":{},\"up-username-not-idn-homograph\":{}},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"email\",\"displayName\":\"${email}\",\"validations\":{\"email\":{},\"length\":{\"max\":255}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"firstName\",\"displayName\":\"${firstName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"lastName\",\"displayName\":\"${lastName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false}],\"groups\":[{\"name\":\"user-metadata\",\"displayHeader\":\"User metadata\",\"displayDescription\":\"Attributes, which refer to user metadata\"}],\"unmanagedAttributePolicy\":\"ENABLED\"}" + "{\"attributes\":[{\"name\":\"username\",\"displayName\":\"${username}\",\"validations\":{\"length\":{\"min\":3,\"max\":255},\"username-prohibited-characters\":{},\"up-username-not-idn-homograph\":{}},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"email\",\"displayName\":\"${email}\",\"validations\":{\"email\":{},\"length\":{\"max\":255}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"firstName\",\"displayName\":\"${firstName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"lastName\",\"displayName\":\"${lastName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"annotations\":{},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"stripe_uid\",\"displayName\":\"Stripe UID\",\"validations\":{},\"annotations\":{},\"permissions\":{\"view\":[\"admin\"],\"edit\":[\"admin\"]},\"multivalued\":false}],\"groups\":[{\"name\":\"user-metadata\",\"displayHeader\":\"User metadata\",\"displayDescription\":\"Attributes, which refer to user metadata\"}],\"unmanagedAttributePolicy\":\"ENABLED\"}" ] } } @@ -72,4 +72,4 @@ {{template "deploy_accounts_utils.user_profile_provider_component" }}, {{template "deploy_accounts_utils.key_provider_component" }} }, -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deployment-configuration/helm/templates/auto-database-mongo.yaml b/deployment-configuration/helm/templates/auto-database-mongo.yaml index 420e14f7d..f6bfd9ad8 100644 --- a/deployment-configuration/helm/templates/auto-database-mongo.yaml +++ b/deployment-configuration/helm/templates/auto-database-mongo.yaml @@ -12,7 +12,7 @@ livenessProbe: exec: command: - - mongo + - mongosh - --eval - "db.adminCommand('ping')" initialDelaySeconds: 30 @@ -21,10 +21,10 @@ readinessProbe: exec: command: - - mongo + - mongosh - --eval - "db.adminCommand('ping')" initialDelaySeconds: 5 timeoutSeconds: 5 failureThreshold: 6 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/apps.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/apps.py index fae83e067..2aa7edd2d 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/apps.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/apps.py @@ -4,3 +4,30 @@ class cloudharness_djangoConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'cloudharness_django' + + def ready(self): + # imports + import sys + for skip_cmd in [ + "--help", + "collectstatic", + "compilemessages", + "compress", + "dbshell", + "dumpdata", + "loaddata", + "makemessages", + "makemigrations", + "migrate", + "reset_db", + "showmigrations", + "sqlmigrate", + "squashmigrations", + "test", + ]: + # for these commands we skip initializing the event listener + if skip_cmd in sys.argv: + return + + from cloudharness_django.services.events import init_listener_in_background + init_listener_in_background() diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index e1b0c5214..1b591cdda 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -1,3 +1,5 @@ +import time + from cloudharness.applications import ConfigurationCallException from django.conf import settings @@ -26,6 +28,7 @@ def event_handler(app, event_client, message): log.info(f"{event_client} {message}") if resource in ["CLIENT_ROLE_MAPPING", "GROUP", "USER", "GROUP_MEMBERSHIP", "ORGANIZATION_MEMBERSHIP"]: try: + time.sleep(1) # wait a bit to make sure the transaction is committed in Keycloak before trying to fetch the updated data init_services() user_service = get_user_service() auth_client = get_auth_service().get_auth_client() @@ -33,19 +36,23 @@ def event_handler(app, event_client, message): if resource == "GROUP": kc_group = auth_client.get_group(resource_path[1]) user_service.sync_kc_group(kc_group) + return if resource == "USER": kc_user = auth_client.get_user(resource_path[1]) user_service.sync_kc_user(kc_user, delete=operation == "DELETE") + return if resource == "CLIENT_ROLE_MAPPING": # adding/deleting user client roles # set/user user is_superuser kc_user = auth_client.get_user(resource_path[1]) user_service.sync_kc_user(kc_user) + return if resource == "GROUP_MEMBERSHIP" or resource == "ORGANIZATION_MEMBERSHIP": # adding / deleting users from groups, update the user # updating the user will also update the user groups kc_user = auth_client.get_user(resource_path[1]) user_service.sync_kc_user(kc_user) + return except Exception as e: log.error(e) raise e @@ -91,26 +98,24 @@ def init_listener(): if not hasattr(settings, "PROJECT_NAME"): raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") + kafka_group_id = settings.PROJECT_NAME.lower() global _message_service_singleton if _message_service_singleton is None: - _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME) - + _message_service_singleton = KeycloakMessageService(kafka_group_id) _message_service_singleton.setup_event_service() def init_listener_in_background(): import threading - import time from cloudharness import log def background_operation(): - listener_initialized = False - while not listener_initialized: + while True: try: init_listener() log.info('User sync events listener started') - listener_initialized = True + break except: log.exception('Error initializing event queue. Retrying in 5 seconds...') time.sleep(5) diff --git a/libraries/cloudharness-common/cloudharness/auth/keycloak.py b/libraries/cloudharness-common/cloudharness/auth/keycloak.py index 8f6756983..977327a8d 100644 --- a/libraries/cloudharness-common/cloudharness/auth/keycloak.py +++ b/libraries/cloudharness-common/cloudharness/auth/keycloak.py @@ -636,6 +636,8 @@ def user_add_update_attribute(self, user_id, attribute_name, attribute_value): { 'attributes': attributes, 'username': user.username, + 'firstName': user.first_name, + 'lastName': user.last_name, 'email': user.email, } ) @@ -659,6 +661,8 @@ def user_delete_attribute(self, user_id, attribute_name): { 'attributes': attributes, 'username': user.username, + 'firstName': user.first_name, + 'lastName': user.last_name, 'email': user.email, }) return True From 50d4db0f8033e538206593933a5c8c6559557cca Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 15:45:40 +0200 Subject: [PATCH 04/20] dns resolution issue --- libraries/cloudharness-common/cloudharness/utils/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/cloudharness-common/cloudharness/utils/env.py b/libraries/cloudharness-common/cloudharness/utils/env.py index cf3d47b8d..bc06fd4ca 100644 --- a/libraries/cloudharness-common/cloudharness/utils/env.py +++ b/libraries/cloudharness-common/cloudharness/utils/env.py @@ -84,7 +84,7 @@ def get_cloudharness_events_client_id(): def get_cloudharness_events_service(): - return get_service_cluster_address('BOOTSTRAP') + return get_service_cluster_address('bootstrap') def get_service_cluster_address(cloudharness_app_name): From 85ecd94ef04ff702e2d9c24a908a5e16168a9910 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 15:46:30 +0200 Subject: [PATCH 05/20] minikube local development cluster ip logic updated --- .../ch_cli_tools/configurationgenerator.py | 3 +- .../deployment-cli-tools/ch_cli_tools/helm.py | 2 +- .../ch_cli_tools/utils.py | 51 +++++++++++++++++-- tools/deployment-cli-tools/requirements.txt | 1 + 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py b/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py index ba3dbbe2b..22effe958 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py +++ b/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py @@ -580,7 +580,8 @@ def hosts_info(values): f"127.0.0.1\t{' '.join('%s.%s' % (values[KEY_APPS][s][KEY_HARNESS][KEY_SERVICE]['name'], values['namespace']) for s in deployments)}") try: - ip = get_cluster_ip() + local = values.get('local', False) + ip = get_cluster_ip(local=local) except: logging.warning('Cannot get cluster ip') ip = "127.0.0.1" diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index 060f42142..fecf29e9c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -213,7 +213,7 @@ def __finish_helm_values(self, values, defer_task_images=False): values['local'] = self.local if self.local: try: - values['localIp'] = get_cluster_ip() + values['localIp'] = get_cluster_ip(local=True) except subprocess.TimeoutExpired: logging.warning("Minikube not available") except: diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 306ea0285..690c7ea88 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -103,11 +103,52 @@ def env_variable(name, value): return {'name': f"{name}".upper(), 'value': str(value)} -def get_cluster_ip(): - out = subprocess.check_output( - ['kubectl', 'cluster-info'], timeout=10).decode("utf-8") - ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", out) - return ips[0] if ips else get_host_address() +def get_cluster_ip(local=False): + if local: + # Try to get LoadBalancer IP from ingress-nginx first (preferred for local dev with minikube tunnel) + try: + out = subprocess.check_output([ + 'kubectl', '-n', 'ingress-nginx', 'get', 'svc', 'ingress-nginx-controller', + '-o', 'jsonpath={.status.loadBalancer.ingress[0].ip}' + ], timeout=5).decode("utf-8").strip() + if out and out != '': + return out + except: + pass + # Try minikube with profile detection for local development + try: + # Get current kubectl context to extract minikube profile + context = subprocess.check_output(['kubectl', 'config', 'current-context'], timeout=5).decode("utf-8").strip() + + # Try with profile if context looks like minikube + if 'minikube' in context.lower(): + profile = context # Context name is often the profile name + try: + out = subprocess.check_output(['minikube', '-p', profile, 'ip'], timeout=5).decode("utf-8").strip() + if out: + return out + except: + pass + + # Try without profile (default minikube) + out = subprocess.check_output(['minikube', 'ip'], timeout=5).decode("utf-8").strip() + if out: + return out + except: + pass + + # Try kubectl cluster-info + try: + out = subprocess.check_output( + ['kubectl', 'cluster-info'], timeout=10).decode("utf-8") + ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", out) + if ips: + return ips[0] + except: + pass + + # Fallback to host address (used for non-local deployments) + return get_host_address() def get_host_address(): diff --git a/tools/deployment-cli-tools/requirements.txt b/tools/deployment-cli-tools/requirements.txt index 266ae27f3..966ce8612 100644 --- a/tools/deployment-cli-tools/requirements.txt +++ b/tools/deployment-cli-tools/requirements.txt @@ -5,4 +5,5 @@ oyaml cloudharness_model cloudharness_utils dirhash +jinja2 StrEnum ; python_version < '3.11' \ No newline at end of file From c003f506d0834cb774d8dd5f420621b682c8267b Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Thu, 16 Apr 2026 22:36:43 +0200 Subject: [PATCH 06/20] finalizing last changes after CH rebase from develop --- applications/events/deploy/templates/deployments.yml | 12 ++++++++---- .../cloudharness/sentry/__init__.py | 7 +++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/applications/events/deploy/templates/deployments.yml b/applications/events/deploy/templates/deployments.yml index b2f14f9b9..dca328df9 100644 --- a/applications/events/deploy/templates/deployments.yml +++ b/applications/events/deploy/templates/deployments.yml @@ -62,11 +62,15 @@ spec: periodSeconds: 15 timeoutSeconds: 5 livenessProbe: - tcpSocket: - port: client - initialDelaySeconds: 30 + exec: + command: + - /bin/sh + - -c + - timeout 10 /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 + initialDelaySeconds: 60 periodSeconds: 30 - timeoutSeconds: 5 + timeoutSeconds: 15 + failureThreshold: 3 lifecycle: preStop: exec: diff --git a/libraries/cloudharness-common/cloudharness/sentry/__init__.py b/libraries/cloudharness-common/cloudharness/sentry/__init__.py index ccc093165..61045cf4f 100644 --- a/libraries/cloudharness-common/cloudharness/sentry/__init__.py +++ b/libraries/cloudharness-common/cloudharness/sentry/__init__.py @@ -24,8 +24,11 @@ def get_dsn(appname): dsn = get_dsn('notifications') """ url = get_common_service_cluster_address() + f'/api/sentry/getdsn/{appname}' - response = requests.get(url, verify=False).json() - dsn = response['dsn'] + try: + response = requests.get(url, verify=False, timeout=5).json() + dsn = response.get('dsn') + except Exception: + return None if dsn and len(dsn) > 0: return dsn else: From ac07aa2c63f688bdf75454793094a408ce5a81fd Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 01:07:03 +0200 Subject: [PATCH 07/20] requirement missing --- tools/deployment-cli-tools/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index aa7a85f99..8e5342b99 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -28,6 +28,7 @@ 'cloudharness_model', 'cloudharness_utils', 'dirhash', + 'jinja2', "StrEnum ; python_version < '3.11'", ] From 462e9996bf867cd2faa93e0fda7e2115f3e2e32f Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 01:24:16 +0200 Subject: [PATCH 08/20] fixing operator none in database config --- deployment-configuration/helm/templates/auto-database.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-database.yaml b/deployment-configuration/helm/templates/auto-database.yaml index a23e97a6c..8cb624a7c 100644 --- a/deployment-configuration/helm/templates/auto-database.yaml +++ b/deployment-configuration/helm/templates/auto-database.yaml @@ -14,7 +14,7 @@ spec: --- {{- end }} {{- define "deploy_utils.database" }} -{{- if and (eq .app.harness.database.type "postgres") .app.harness.database.postgres.operator }} +{{- if and (eq .app.harness.database.type "postgres") (dig "postgres" "operator" false .app.harness.database) }} {{- include "deploy_utils.database.postgres.operator" . }} {{- else }} --- From 0ee76816868a2453763489d5f2f4b26e1acae974 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 01:30:00 +0200 Subject: [PATCH 09/20] another operator instance --- .../helm/templates/auto-network-policies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-network-policies.yaml b/deployment-configuration/helm/templates/auto-network-policies.yaml index 63f5dc4c3..52c80c405 100644 --- a/deployment-configuration/helm/templates/auto-network-policies.yaml +++ b/deployment-configuration/helm/templates/auto-network-policies.yaml @@ -117,7 +117,7 @@ spec: protocol: UDP - port: 53 protocol: TCP - {{- if and (eq .app.harness.database.type "postgres") .app.harness.database.postgres.operator }} + {{- if and (eq .app.harness.database.type "postgres") (dig "postgres" "operator" false .app.harness.database) }} # Allow CNPG pods to reach the Kubernetes API server {{- $apiCidrs := list }} {{- $kubeSvc := (lookup "v1" "Service" "default" "kubernetes") }} From dbb05a9c5f25bcd2900e6d141e1cf3a4686a0cf8 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 01:51:32 +0200 Subject: [PATCH 10/20] removing immutable matchLabel, only app should be in matchLabel where instead chart, release and heritage belongs to metadata.labels --- deployment-configuration/helm/templates/auto-deployments.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment-configuration/helm/templates/auto-deployments.yaml b/deployment-configuration/helm/templates/auto-deployments.yaml index 7e92629e8..5c3023ebd 100644 --- a/deployment-configuration/helm/templates/auto-deployments.yaml +++ b/deployment-configuration/helm/templates/auto-deployments.yaml @@ -15,7 +15,6 @@ spec: selector: matchLabels: app: {{ .app.harness.deployment.name| quote }} -{{- include "deploy_utils.labels" .root | indent 6 }} template: metadata: {{- if .app.harvest }} From 78f250ea0f177a22b8307864d02fd7ad40a891ef Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 09:32:35 +0200 Subject: [PATCH 11/20] resource policy set to keep --- deployment-configuration/helm/templates/auto-database.yaml | 4 ++++ deployment-configuration/helm/templates/auto-volumes.yaml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/deployment-configuration/helm/templates/auto-database.yaml b/deployment-configuration/helm/templates/auto-database.yaml index 8cb624a7c..f3c13b443 100644 --- a/deployment-configuration/helm/templates/auto-database.yaml +++ b/deployment-configuration/helm/templates/auto-database.yaml @@ -5,6 +5,8 @@ apiVersion: v1 metadata: name: "db-backups" namespace: {{ .Values.namespace }} + annotations: + helm.sh/resource-policy: keep spec: accessModes: - ReadWriteOnce @@ -23,6 +25,8 @@ apiVersion: v1 metadata: name: {{ .app.harness.database.name | quote }} namespace: {{ .root.Values.namespace }} + annotations: + helm.sh/resource-policy: keep spec: accessModes: - ReadWriteOnce diff --git a/deployment-configuration/helm/templates/auto-volumes.yaml b/deployment-configuration/helm/templates/auto-volumes.yaml index 8e217aebd..f2f55f329 100644 --- a/deployment-configuration/helm/templates/auto-volumes.yaml +++ b/deployment-configuration/helm/templates/auto-volumes.yaml @@ -4,6 +4,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ .app.harness.deployment.volume.name }} + annotations: + helm.sh/resource-policy: keep labels: app: {{ .app.harness.deployment.name| quote }} spec: From 5bb5adc28e3427ebd0c84e1224a9a75fffa22e92 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 12:46:54 +0200 Subject: [PATCH 12/20] removing resource policy --- deployment-configuration/helm/templates/auto-database.yaml | 4 ---- deployment-configuration/helm/templates/auto-volumes.yaml | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/deployment-configuration/helm/templates/auto-database.yaml b/deployment-configuration/helm/templates/auto-database.yaml index f3c13b443..8cb624a7c 100644 --- a/deployment-configuration/helm/templates/auto-database.yaml +++ b/deployment-configuration/helm/templates/auto-database.yaml @@ -5,8 +5,6 @@ apiVersion: v1 metadata: name: "db-backups" namespace: {{ .Values.namespace }} - annotations: - helm.sh/resource-policy: keep spec: accessModes: - ReadWriteOnce @@ -25,8 +23,6 @@ apiVersion: v1 metadata: name: {{ .app.harness.database.name | quote }} namespace: {{ .root.Values.namespace }} - annotations: - helm.sh/resource-policy: keep spec: accessModes: - ReadWriteOnce diff --git a/deployment-configuration/helm/templates/auto-volumes.yaml b/deployment-configuration/helm/templates/auto-volumes.yaml index f2f55f329..3656630d4 100644 --- a/deployment-configuration/helm/templates/auto-volumes.yaml +++ b/deployment-configuration/helm/templates/auto-volumes.yaml @@ -4,8 +4,6 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: {{ .app.harness.deployment.volume.name }} - annotations: - helm.sh/resource-policy: keep labels: app: {{ .app.harness.deployment.name| quote }} spec: @@ -29,4 +27,4 @@ spec: --- {{- include "deploy_utils.pvolume" (dict "root" $ "app" $app) }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} From 03c447b992efb2579a33cd727e9f9bb350159a3a Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Fri, 17 Apr 2026 16:26:47 +0200 Subject: [PATCH 13/20] fix mount volume rights and pvc existing data that kafka does not like --- applications/events/deploy/templates/deployments.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/applications/events/deploy/templates/deployments.yml b/applications/events/deploy/templates/deployments.yml index dca328df9..e5a210449 100644 --- a/applications/events/deploy/templates/deployments.yml +++ b/applications/events/deploy/templates/deployments.yml @@ -37,7 +37,7 @@ spec: - name: KAFKA_CONTROLLER_QUORUM_VOTERS value: 1@kafka-0.broker.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.apps.events.kafka.controllerPort }} - name: KAFKA_LOG_DIRS - value: /var/lib/kafka/data + value: /var/lib/kafka/data/logs - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS value: "0" {{- range $key, $value := .Values.apps.events.kafka.config }} @@ -83,6 +83,8 @@ spec: volumeMounts: - mountPath: /var/lib/kafka/data name: data + securityContext: + fsGroup: 1000 terminationGracePeriodSeconds: 30 updateStrategy: type: RollingUpdate From cffa4a37eec2af6c603f5ee5c6910392ea3c6425 Mon Sep 17 00:00:00 2001 From: ddelpiano Date: Tue, 28 Apr 2026 03:06:13 +0200 Subject: [PATCH 14/20] linter fix --- tools/deployment-cli-tools/ch_cli_tools/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 690c7ea88..a3deb37dd 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -119,7 +119,7 @@ def get_cluster_ip(local=False): try: # Get current kubectl context to extract minikube profile context = subprocess.check_output(['kubectl', 'config', 'current-context'], timeout=5).decode("utf-8").strip() - + # Try with profile if context looks like minikube if 'minikube' in context.lower(): profile = context # Context name is often the profile name @@ -129,14 +129,14 @@ def get_cluster_ip(local=False): return out except: pass - + # Try without profile (default minikube) out = subprocess.check_output(['minikube', 'ip'], timeout=5).decode("utf-8").strip() if out: return out except: pass - + # Try kubectl cluster-info try: out = subprocess.check_output( @@ -146,7 +146,7 @@ def get_cluster_ip(local=False): return ips[0] except: pass - + # Fallback to host address (used for non-local deployments) return get_host_address() From fe2a4811636515a65d6e13ae62f8ea7133e7b38d Mon Sep 17 00:00:00 2001 From: Dario <37704177+ddelpiano@users.noreply.github.com> Date: Tue, 28 Apr 2026 03:33:02 +0200 Subject: [PATCH 15/20] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tools/deployment-cli-tools/ch_cli_tools/tilt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/tilt.py b/tools/deployment-cli-tools/ch_cli_tools/tilt.py index 29738564e..d86203fb6 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/tilt.py +++ b/tools/deployment-cli-tools/ch_cli_tools/tilt.py @@ -10,7 +10,7 @@ from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ BASE_IMAGES_PATH, STATIC_IMAGES_PATH, HELM_ENGINE, COMPOSE_ENGINE from .helm import KEY_APPS, KEY_HARNESS, KEY_DEPLOYMENT, KEY_TASK_IMAGES -from .utils import get_template, dict_merge, find_dockerfiles_paths, app_name_from_path, yaml, \ +from .utils import get_template, dict_merge, find_dockerfiles_paths, app_name_from_path, \ find_file_paths, guess_build_dependencies_from_dockerfile, get_json_template, get_image_name from . import HERE, CH_ROOT From 4037a96f7520d3f63c95838db7cb193ab6eab348 Mon Sep 17 00:00:00 2001 From: Dario <37704177+ddelpiano@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:34:40 +0200 Subject: [PATCH 16/20] Potential fix for pull request finding 'Unused local variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tools/deployment-cli-tools/ch_cli_tools/tilt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/tilt.py b/tools/deployment-cli-tools/ch_cli_tools/tilt.py index d86203fb6..ba69869f7 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/tilt.py +++ b/tools/deployment-cli-tools/ch_cli_tools/tilt.py @@ -139,7 +139,6 @@ def process_build_dockerfile( for dockerfile_path in base_dockerfiles: process_build_dockerfile(dockerfile_path, root_path, global_context=True) - static_images = set() for root_path in root_paths: static_dockerfiles = find_dockerfiles_paths( join(root_path, STATIC_IMAGES_PATH)) From 6882671c37286e95ef4e62e35cacfeaee0c1fe5c Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Mon, 4 May 2026 09:08:14 +0200 Subject: [PATCH 17/20] chore: make get_cluster_ip a bit more robust and independent on minikube and ingress nginx --- .../ch_cli_tools/utils.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index a3deb37dd..2192fb451 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -104,15 +104,19 @@ def env_variable(name, value): def get_cluster_ip(local=False): + print("Getting cluster IP...") if local: - # Try to get LoadBalancer IP from ingress-nginx first (preferred for local dev with minikube tunnel) try: - out = subprocess.check_output([ - 'kubectl', '-n', 'ingress-nginx', 'get', 'svc', 'ingress-nginx-controller', - '-o', 'jsonpath={.status.loadBalancer.ingress[0].ip}' - ], timeout=5).decode("utf-8").strip() - if out and out != '': - return out + # Check if ingress-nginx is installed by trying to get its pods + errcode, out = subprocess.getstatusoutput("kubectl -n ingress-nginx get pods") + if errcode==0 and out and out != 'No resources found in ingress-nginx namespace.': + # Try to get LoadBalancer IP from ingress-nginx first (preferred for local dev with minikube tunnel) + out = subprocess.check_output([ + 'kubectl', '-n', 'ingress-nginx', 'get', 'svc', 'ingress-nginx-controller', + '-o', 'jsonpath={.status.loadBalancer.ingress[0].ip}', + ], timeout=5).decode("utf-8").strip() + if out and out != '': + return out except: pass # Try minikube with profile detection for local development @@ -131,24 +135,19 @@ def get_cluster_ip(local=False): pass # Try without profile (default minikube) - out = subprocess.check_output(['minikube', 'ip'], timeout=5).decode("utf-8").strip() - if out: - return out + err_code, out = subprocess.getstatusoutput(['minikube ip']) + if err_code == 0: + # If minikube is running, this should return the IP + out = subprocess.check_output(['minikube', 'ip'], timeout=5).decode("utf-8").strip() + if out: + return out except: pass - # Try kubectl cluster-info - try: - out = subprocess.check_output( - ['kubectl', 'cluster-info'], timeout=10).decode("utf-8") - ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", out) - if ips: - return ips[0] - except: - pass - - # Fallback to host address (used for non-local deployments) - return get_host_address() + out = subprocess.check_output( + ['kubectl', 'cluster-info'], timeout=10).decode("utf-8") + ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", out) + return ips[0] if ips else get_host_address() def get_host_address(): From 9ad54b062201e7783ce990a533023a65d10990f2 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Mon, 4 May 2026 09:14:05 +0200 Subject: [PATCH 18/20] chore: tiltfile setup infra waits for helm and jobs to be finished before continuing --- .../ch_cli_tools/templates/tilt/tilt-template.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl index 7c0ec06c2..029e34d86 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl +++ b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl @@ -10,7 +10,7 @@ if setup_infrastructure: # setup ingress print("Installing ingress controller") # local("cd infrastructure/cluster-configuration && source cluster-init.sh") - local("kubectl get namespace ingress-nginx 2>/dev/null 1>/dev/null || bash -c 'helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version v4.2.5 && sleep 10'") + local("kubectl get namespace ingress-nginx 2>/dev/null 1>/dev/null || bash -c 'helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version v4.2.5 --wait --wait-for-jobs") # print("Let's wait a few seconds...") # local("sleep 30") else: From 0ba9fbe5d34fa51a14a4a89145fdb14c17f00e1f Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Mon, 4 May 2026 10:36:45 +0200 Subject: [PATCH 19/20] chore: removed wait for webhook, tiltfile now contains helm ... --wait --wait-for-jobs --- .../ch_cli_tools/templates/tilt/tilt-template.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl index 029e34d86..45cda8616 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl +++ b/tools/deployment-cli-tools/ch_cli_tools/templates/tilt/tilt-template.tpl @@ -10,7 +10,7 @@ if setup_infrastructure: # setup ingress print("Installing ingress controller") # local("cd infrastructure/cluster-configuration && source cluster-init.sh") - local("kubectl get namespace ingress-nginx 2>/dev/null 1>/dev/null || bash -c 'helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version v4.2.5 --wait --wait-for-jobs") + local("kubectl get namespace ingress-nginx 2>/dev/null 1>/dev/null || bash -c 'helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version v4.2.5 --wait --wait-for-jobs'") # print("Let's wait a few seconds...") # local("sleep 30") else: From 383bbc1e4554a550b7cd639d23afef7e8ac08db8 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Tue, 5 May 2026 10:51:42 +0200 Subject: [PATCH 20/20] chore: remove hardcoded storageClassName:standard from kafka (events) and auto volumes template --- applications/events/deploy/values.yaml | 1 - deployment-configuration/helm/templates/auto-volumes.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/applications/events/deploy/values.yaml b/applications/events/deploy/values.yaml index 740a71fa8..9b82a79a8 100644 --- a/applications/events/deploy/values.yaml +++ b/applications/events/deploy/values.yaml @@ -17,7 +17,6 @@ kafka: imagePullPolicy: IfNotPresent clusterId: 5L6g3nShT-eMCtK--X86sw storage: 10Gi - storageClassName: standard config: auto.create.topics.enable: "true" num.partitions: "3" diff --git a/deployment-configuration/helm/templates/auto-volumes.yaml b/deployment-configuration/helm/templates/auto-volumes.yaml index 3656630d4..b9170a35d 100644 --- a/deployment-configuration/helm/templates/auto-volumes.yaml +++ b/deployment-configuration/helm/templates/auto-volumes.yaml @@ -14,7 +14,6 @@ spec: accessModes: {{- if or (not (hasKey .app.harness.deployment.volume "usenfs")) (not .app.harness.deployment.volume.usenfs) }} - ReadWriteOnce - storageClassName: standard {{- else }} - ReadWriteMany storageClassName: {{ printf "%s-%s" .root.Values.namespace .root.Values.apps.nfsserver.storageClass.name }}