CH-262 / CH-263 / CH-264 - Tilt and some other minor changes #847
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,67 @@ | ||||||
| load('ext://namespace', 'namespace_create', "namespace_inject") | ||||||
|
|
||||||
|
|
||||||
| def deploy(name, namespace, extra_env, watch): | ||||||
|
|
||||||
| # create namespaces | ||||||
| namespace_create(namespace) | ||||||
|
|
||||||
| # load helm chart | ||||||
| yaml = decode_yaml_stream(helm( | ||||||
| "deployment/helm", | ||||||
| # The release name, equivalent to helm --name | ||||||
| name=name, | ||||||
| # The namespace to install in, equivalent to helm --namespace | ||||||
| namespace=namespace, | ||||||
| # The values file to substitute into the chart. | ||||||
| values=["deployment/helm/values.yaml"], | ||||||
| # Values to set from the command-line | ||||||
| set=["service.port=1234", "ingress.enabled=true"] | ||||||
| ) | ||||||
| ) | ||||||
|
|
||||||
| source_root = os.path.abspath(os.getcwd()) | ||||||
| # modify deployments | ||||||
| for r in yaml: | ||||||
| if r.get("kind") == "Deployment": | ||||||
| deployment_name = r["metadata"]["name"] | ||||||
| print("+ patching deployment:", deployment_name) | ||||||
| r["spec"]["template"]["spec"].setdefault("volumes", []).append({ | ||||||
| "name": name + "-root", | ||||||
| "hostPath": { | ||||||
| "path": source_root | ||||||
| } | ||||||
| }) | ||||||
| for container in r["spec"]["template"]["spec"]["containers"]: | ||||||
| print(" + modifying container:", container["name"]) | ||||||
| print(" - add " + name + " root folder") | ||||||
| container.setdefault("volumeMounts", []).append({ | ||||||
| "mountPath": "/usr/src/" + name, | ||||||
| "name": name + "-root" | ||||||
| }) | ||||||
| if "resources" in container: | ||||||
| print(" - modifying resource requests and limits") | ||||||
| if "limits" not in container["resources"]: | ||||||
| container["resources"]["limits"] = {} | ||||||
| if "requests" not in container["resources"]: | ||||||
| container["resources"]["requests"] = {} | ||||||
| container["resources"]["requests"]["cpu"] = "100m" | ||||||
| container["resources"]["requests"]["memory"] = "256Mi" | ||||||
| container["resources"]["limits"]["cpu"] = "8000m" | ||||||
| container["resources"]["limits"]["memory"] = "4096Mi" | ||||||
|
|
||||||
| if deployment_name in extra_env and len(extra_env[deployment_name]) > 0: | ||||||
| print("Adding tasks images dependencies to env ", deployment_name) | ||||||
| for env in extra_env[deployment_name]: | ||||||
| container["env"].append({ | ||||||
| "name": env, "value": env | ||||||
| }) | ||||||
|
|
||||||
| if not watch: | ||||||
| # don't watch mnp folder | ||||||
|
There was a problem hiding this comment. Typo in comment: "mnp" should be "npm".
Suggested change
|
||||||
| watch_settings(ignore=source_root) | ||||||
| else: | ||||||
| print("Watching for file changes") | ||||||
|
|
||||||
| # install applications | ||||||
| k8s_yaml(namespace_inject(encode_yaml_stream(yaml), namespace)) | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,3 +4,30 @@ | |
| class cloudharness_djangoConfig(AppConfig): | ||
| default_auto_field = 'django.db.models.BigAutoField' | ||
| name = 'cloudharness_django' | ||
|
|
||
| def ready(self): | ||
| # imports | ||
| import sys | ||
| for skip_cmd in [ | ||
| "--help", | ||
| "collectstatic", | ||
| "compilemessages", | ||
| "compress", | ||
| "dbshell", | ||
| "dumpdata", | ||
| "loaddata", | ||
| "makemessages", | ||
| "makemigrations", | ||
| "migrate", | ||
| "reset_db", | ||
| "showmigrations", | ||
| "sqlmigrate", | ||
| "squashmigrations", | ||
| "test", | ||
| ]: | ||
| # for these commands we skip initializing the event listener | ||
| if skip_cmd in sys.argv: | ||
| return | ||
|
|
||
| from cloudharness_django.services.events import init_listener_in_background | ||
| init_listener_in_background() | ||
|
Comment on lines
+8
to
+33
There was a problem hiding this comment. Starting the Kafka listener in |
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,3 +1,5 @@ | ||||||
| import time | ||||||
|
|
||||||
| from cloudharness.applications import ConfigurationCallException | ||||||
|
|
||||||
| from django.conf import settings | ||||||
|
|
@@ -26,26 +28,31 @@ def event_handler(app, event_client, message): | |||||
| log.info(f"{event_client} {message}") | ||||||
| if resource in ["CLIENT_ROLE_MAPPING", "GROUP", "USER", "GROUP_MEMBERSHIP", "ORGANIZATION_MEMBERSHIP"]: | ||||||
| try: | ||||||
| time.sleep(1) # wait a bit to make sure the transaction is committed in Keycloak before trying to fetch the updated data | ||||||
| init_services() | ||||||
| user_service = get_user_service() | ||||||
|
Comment on lines
29
to
33
There was a problem hiding this comment.
|
||||||
| auth_client = get_auth_service().get_auth_client() | ||||||
|
|
||||||
| if resource == "GROUP": | ||||||
| kc_group = auth_client.get_group(resource_path[1]) | ||||||
| user_service.sync_kc_group(kc_group) | ||||||
| return | ||||||
| if resource == "USER": | ||||||
| kc_user = auth_client.get_user(resource_path[1]) | ||||||
| user_service.sync_kc_user(kc_user, delete=operation == "DELETE") | ||||||
| return | ||||||
| if resource == "CLIENT_ROLE_MAPPING": | ||||||
| # adding/deleting user client roles | ||||||
| # set/user user is_superuser | ||||||
| kc_user = auth_client.get_user(resource_path[1]) | ||||||
| user_service.sync_kc_user(kc_user) | ||||||
| return | ||||||
| if resource == "GROUP_MEMBERSHIP" or resource == "ORGANIZATION_MEMBERSHIP": | ||||||
| # adding / deleting users from groups, update the user | ||||||
| # updating the user will also update the user groups | ||||||
| kc_user = auth_client.get_user(resource_path[1]) | ||||||
| user_service.sync_kc_user(kc_user) | ||||||
| return | ||||||
| except Exception as e: | ||||||
| log.error(e) | ||||||
| raise e | ||||||
|
There was a problem hiding this comment. In Python,
Suggested change
|
||||||
|
|
@@ -91,26 +98,24 @@ def init_listener(): | |||||
| if not hasattr(settings, "PROJECT_NAME"): | ||||||
| raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") | ||||||
|
|
||||||
| kafka_group_id = settings.PROJECT_NAME.lower() | ||||||
| global _message_service_singleton | ||||||
| if _message_service_singleton is None: | ||||||
| _message_service_singleton = KeycloakMessageService(kafka_group_id) | ||||||
| _message_service_singleton.setup_event_service() | ||||||
|
|
||||||
|
|
||||||
| def init_listener_in_background(): | ||||||
| import threading | ||||||
| from cloudharness import log | ||||||
|
|
||||||
| def background_operation(): | ||||||
|
|
||||||
| while True: | ||||||
| try: | ||||||
| init_listener() | ||||||
| log.info('User sync events listener started') | ||||||
| break | ||||||
| except: | ||||||
| log.exception('Error initializing event queue. Retrying in 5 seconds...') | ||||||
| time.sleep(5) | ||||||
|
Comment on lines
+114
to
121
There was a problem hiding this comment. This retry loop runs indefinitely on any exception. Since it is executed in a background thread (started below), make sure failures don’t prevent clean shutdown and aren’t retried forever unintentionally. Consider (a) catching |
||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -213,7 +213,7 @@ def __finish_helm_values(self, values, defer_task_images=False): | |
| values['local'] = self.local | ||
| if self.local: | ||
| try: | ||
| values['localIp'] = get_cluster_ip(local=True) | ||
| except subprocess.TimeoutExpired: | ||
| logging.warning("Minikube not available") | ||
| except: | ||
|
Comment on lines
215
to
219
There was a problem hiding this comment.
|
||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
container["env"]may not exist on all rendered Deployments. This will raise a KeyError when adding task-image env vars. Usecontainer.setdefault("env", []).append(...)(or initializeenvif missing) before appending.