From 43e3f4455cb1a62da592855d31707485ec9f71ec Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Tue, 21 May 2024 10:25:47 -0500 Subject: [PATCH 01/24] Working Jupyterhub Integration --- docker-compose.dev.yml | 1 + docker-compose.jupyter.yml | 30 +++ docker-dev.sh | 9 + frontend/src/components/Layout.tsx | 14 ++ jupyterhub/.env-example | 12 + jupyterhub/Dockerfile.jupyterhub | 14 ++ .../customauthenticator/__init__.py | 0 .../customauthenticator/custom.py | 216 ++++++++++++++++++ jupyterhub/authenticator/setup.py | 17 ++ jupyterhub/authenticator/test_jwt.py | 40 ++++ jupyterhub/jupyterhub_config.py | 74 ++++++ 11 files changed, 427 insertions(+) create mode 100644 docker-compose.jupyter.yml create mode 100644 jupyterhub/.env-example create mode 100644 jupyterhub/Dockerfile.jupyterhub create mode 100644 jupyterhub/authenticator/customauthenticator/__init__.py create mode 100644 jupyterhub/authenticator/customauthenticator/custom.py create mode 100644 jupyterhub/authenticator/setup.py create mode 100644 jupyterhub/authenticator/test_jwt.py create mode 100644 jupyterhub/jupyterhub_config.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 92df388f8..a6ff21566 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -172,6 +172,7 @@ services: networks: clowder2: + name: clowder2 ## By default this config uses default local driver, ## For custom volumes replace with volume driver configuration. diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml new file mode 100644 index 000000000..333ca5555 --- /dev/null +++ b/docker-compose.jupyter.yml @@ -0,0 +1,30 @@ +version: '3' +services: + jupyterhub: + build: + context: jupyterhub + dockerfile: Dockerfile.jupyterhub + args: + JUPYTERHUB_VERSION: latest + restart: always + networks: + - clowder2 + volumes: + # The JupyterHub configuration file + - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro + # Bind Docker socket on the hostso we can connect to the daemon from + # within the container + - /var/run/docker.sock:/var/run/docker.sock:rw + # Bind Docker volume on host for JupyterHub database and cookie secrets + - jupyterhub-data:/data + ports: + - "8765:8000" + env_file: + - jupyterhub/.env + command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py + + depends_on: + - keycloak + +volumes: + jupyterhub-data: diff --git a/docker-dev.sh b/docker-dev.sh index 44f50afab..2c3fd1a00 100755 --- a/docker-dev.sh +++ b/docker-dev.sh @@ -7,3 +7,12 @@ if [ "$1" = "down" ] then docker-compose -f docker-compose.dev.yml -p clowder2-dev down fi +if [ "$1" = "jupyter" ] & [ "$2" = "up" ] +then + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev up -d --build +fi + +if [ "$1" = "jupyter" ] & [ "$2" = "down" ] +then + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev down +fi \ No newline at end of file diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 49ce98ad1..a7a4a0869 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -23,6 +23,7 @@ import { RootState } from "../types/data"; import { AddBox, Explore } from "@material-ui/icons"; import HistoryIcon from "@mui/icons-material/History"; import GroupIcon from "@mui/icons-material/Group"; +import MenuBookIcon from '@mui/icons-material/MenuBook'; import Gravatar from "react-gravatar"; import PersonIcon from "@mui/icons-material/Person"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; @@ -419,6 +420,19 @@ export default function PersistentDrawerLeft(props) { + + {/*TODO: Need to make link dynamic */} + + + + + + + + + +
diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example new file mode 100644 index 000000000..8092bde32 --- /dev/null +++ b/jupyterhub/.env-example @@ -0,0 +1,12 @@ +# Example configuration file for Clowder JupyterHub +KEYCLOAK_HOSTNAME="keycloak:8080/keycloak" +# Development mode use the following line instead +#KEYCLOAK_HOSTNAME="host.docker.internal:8080/keycloak" +KEYCLOAK_AUDIENCE="clowder" +KEYCLOAK_REALM="clowder" +JUPYTERHUB_ADMIN="admin" +#Change network name to the one created by docker-compose +DOCKER_NETWORK_NAME="clowder2" +DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" +DOCKER_NOTEBOOK_DIR="/home/jovyan/work" +JUPYTERHUB_CRYPT_KEY="" \ No newline at end of file diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub new file mode 100644 index 000000000..9956c7ab8 --- /dev/null +++ b/jupyterhub/Dockerfile.jupyterhub @@ -0,0 +1,14 @@ +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION + +# Install dockerspawner, +# hadolint ignore=DL3013 +RUN python3 -m pip install --no-cache-dir \ + dockerspawner + +# Install custom authenticator +WORKDIR /tmp/authenticator/ +COPY authenticator /tmp/authenticator/ +RUN pip3 install /tmp/authenticator + +CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] \ No newline at end of file diff --git a/jupyterhub/authenticator/customauthenticator/__init__.py b/jupyterhub/authenticator/customauthenticator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py new file mode 100644 index 000000000..1d9b8b101 --- /dev/null +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -0,0 +1,216 @@ +import os +import urllib.parse +import json + +from jose import jwt +from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +from jupyterhub.auth import Authenticator +from jupyterhub.handlers import LoginHandler, LogoutHandler +from traitlets import Unicode +from tornado import web +import requests + + +class CustomTokenAuthenticator(Authenticator): + """ + Accept the authenticated Access Token from cookie. + """ + auth_cookie_header = Unicode( + os.environ.get('AUTH_COOKIE_HEADER', ''), + config=True, + help="the cookie header we put in browser to retrieve token", + ) + + auth_username_key = Unicode( + os.environ.get('AUTH_USERNAME_KEY', ''), + config=True, + help="the key to retreive username from the json", + ) + + + + landing_page_login_url = Unicode( + os.environ.get('LANDING_PAGE_LOGIN_URL', ''), + config=True, + help="the landing page login entry", + ) + + keycloak_url = Unicode( + os.environ.get('KEYCLOAK_URL', ''), + config=True, + help="the URL where keycloak is installed", + ) + + keycloak_audience = Unicode( + os.environ.get('KEYCLOAK_AUDIENCE', ''), + config=True, + help="the audience for keycloak to check", + ) + + keycloak_pem_key = Unicode( + os.environ.get('KEYCLOAK_PEM_KEY', ''), + config=True, + help="the RSA pem key with proper header and footer (deprecated)", + ) + + space_service_url = Unicode( + os.environ.get('SPACE_SERVICE_URL', ''), + config=True, + help="the internal space service url" + ) + + quotas = None + + def get_handlers(self, app): + return [ + (r'/', LoginHandler), + (r'/user', LoginHandler), + (r'/lab', LoginHandler), + (r'/login', LoginHandler), + (r'/logout', CustomTokenLogoutHandler), + ] + + def get_keycloak_pem(self): + if not self.keycloak_url: + raise web.HTTPError(500, log_message="JupyterHub is not correctly configured.") + + # fetch the key + response = urllib.request.urlopen(self.keycloak_url) + if response.code >= 200 or response <= 299: + encoding = response.info().get_content_charset('utf-8') + result = json.loads(response.read().decode(encoding)) + self.keycloak_pem_key = f"-----BEGIN PUBLIC KEY-----\n" \ + f"{result['public_key']}\n" \ + f"-----END PUBLIC KEY-----" + else: + raise web.HTTPError(500, log_message="Could not get key from keycloak.") + + def check_jwt_token(self, access_token): + # make sure we have the pem cert + if not self.keycloak_pem_key: + self.get_keycloak_pem() + + # make sure audience is set + if not self.keycloak_audience: + raise web.HTTPError(403, log_message="JupyterHub is not correctly configured.") + + # no token in the cookie + if not access_token: + raise web.HTTPError(401, log_message="Please login to access Clowder.") + + # make sure it is a valid token + if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': + raise web.HTTPError(403, log_message="Token format not valid, it has to be bearer xxxx!") + + # decode jwt token instead of sending it to userinfo endpoint: + access_token = access_token.split(" ")[1] + public_key = self.keycloak_pem_key + audience = self.keycloak_audience + try: + resp_json = jwt.decode(access_token, public_key, audience=audience) + except ExpiredSignatureError: + raise web.HTTPError(403, log_message='JWT Expired Signature Error: token signature has expired') + except JWTClaimsError: + raise web.HTTPError(403, log_message='JWT Claims Error: token signature is invalid') + except JWTError: + raise web.HTTPError(403, log_message='JWT Error: token signature is invalid') + except Exception as e: + raise web.HTTPError(403, log_message="Not a valid jwt token!") + + # make sure we know username + if self.auth_username_key not in resp_json.keys(): + raise web.HTTPError(500, log_message=f"Required field {self.auth_username_key} does not exist in jwt token") + username = resp_json[self.auth_username_key] + + # # make sure there is a user id + # if self.auth_uid_number_key not in resp_json.keys(): + # raise web.HTTPError(500, log_message=f"Required field {self.auth_uid_number_key} does not exist in jwt token") + # uid = resp_json[self.auth_uid_number_key] + # + # get the groups/roles for the user + if "roles" in resp_json: + user_roles = resp_json.get("roles", []) + elif "realm_access" in resp_json: + user_roles = resp_json["realm_access"].get("roles", []) + else: + user_roles = [] + + + self.log.info(f"username={username}") + return { + 'name': username, + 'auth_state': { + 'roles': user_roles, + }, + } + + async def authenticate(self, handler, data): + self.log.info("Authenticate") + try: + access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) + if not access_token: + raise web.HTTPError(401, log_message="Please login to access Clowder.") + + # check token and authorization + user = self.check_jwt_token(access_token) + return user + except web.HTTPError as e: + if e.log_message: + error_msg = urllib.parse.quote(e.log_message.encode('utf-8')) + else: + error_msg = urllib.parse.quote(f"Error {e}".encode('utf-8')) + ". Please login to access Clowder." + handler.redirect(f"{self.landing_page_login_url}?error={error_msg}") + + + # async def pre_spawn_start(self, user, spawner): + # auth_state = await user.get_auth_state() + # if not auth_state: + # self.log.error("No auth state") + # return + # + # spawner.environment['NB_USER'] = user.name + # spawner.environment['NB_UID'] = str(auth_state['uid']) + # + # quota = self.find_quota(user, auth_state) + # if "cpu" in quota: + # spawner.cpu_guarantee = quota["cpu"][0] + # spawner.cpu_limit = quota["cpu"][1] + # else: + # spawner.cpu_guarantee = 1 + # spawner.cpu_limit = 2 + # if "mem" in quota: + # spawner.mem_guarantee = f"{quota['mem'][0]}G" + # spawner.mem_limit = f"{quota['mem'][1]}G" + # else: + # spawner.mem_guarantee = "2G" + # spawner.mem_limit = "4G" + +# +# # This is called from the jupyterlab so there is no cookies that this depends on +# async def refresh_user(self, user, handler): +# self.log.info("Refresh User") +# try: +# access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) +# # if no token present +# if not access_token: +# return False +# +# # if token present, check token and authorization +# if self.check_jwt_token(access_token): +# True +# return False +# except: +# self.log.exception("Error in refresh user") +# return False + + +class CustomTokenLogoutHandler(LogoutHandler): + + async def handle_logout(self): + # remove clowder token on logout + self.log.info("Remove clowder token on logout") + error_msg = "You have logged out of Clowder system from Clowder . Please login again if you want to use " \ + "Clowder components." + self.set_cookie(self.authenticator.auth_cookie_header, "") + self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") + diff --git a/jupyterhub/authenticator/setup.py b/jupyterhub/authenticator/setup.py new file mode 100644 index 000000000..7e37c3d7a --- /dev/null +++ b/jupyterhub/authenticator/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +setup( + name='customauthenticator', + version='0.8.0', + description='Custom Authenticator for JupyterHub', + author='cwang138', + author_email='cwang138@illinois.edu', + license='MPL 2.0', + packages=['customauthenticator'], + install_requires=[ + 'jupyterhub', + 'pyjwt', + 'requests', + 'python-jose' + ] +) diff --git a/jupyterhub/authenticator/test_jwt.py b/jupyterhub/authenticator/test_jwt.py new file mode 100644 index 000000000..12df103db --- /dev/null +++ b/jupyterhub/authenticator/test_jwt.py @@ -0,0 +1,40 @@ +from jose import jwt +from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +import os +import urllib.request +import json + +response = urllib.request.urlopen("") + +if response.code >= 200 or response <= 299: + encoding = response.info().get_content_charset('utf-8') + result = json.loads(response.read().decode(encoding)) + public_key = f"-----BEGIN PUBLIC KEY-----\n" \ + f"{result['public_key']}\n" \ + f"-----END PUBLIC KEY-----" +else: + print("Could not get key from keycloak.") + + +access_token ="" + +# make sure it is a valid token +if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': + print("Token format not valid, it has to be bearer xxxx!") + +# decode jwt token instead of sending it to userinfo endpoint: +access_token = access_token.split(" ")[1] + +try: + decoded = jwt.decode(access_token, public_key, audience="clowder") + print(decoded) + +except ExpiredSignatureError: + print('JWT Expired Signature Error: token signature has expired') +except JWTClaimsError: + print('JWT Claims Error: token signature is invalid') +except JWTError: + print('JWT Error: token signature is invalid') +except Exception as e: + print("Not a valid jwt token!") + diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py new file mode 100644 index 000000000..1e704e316 --- /dev/null +++ b/jupyterhub/jupyterhub_config.py @@ -0,0 +1,74 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Configuration file for JupyterHub +import os + +c = get_config() # noqa: F821 + +# We rely on environment variables to configure JupyterHub so that we +# avoid having to rebuild the JupyterHub container every time we change a +# configuration parameter. + +# Spawn single-user servers as Docker containers +c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" + +# Spawn containers from this image +c.DockerSpawner.image = os.environ["DOCKER_NOTEBOOK_IMAGE"] + +# Connect containers to this Docker network +network_name = os.environ["DOCKER_NETWORK_NAME"] +c.DockerSpawner.use_internal_ip = True +c.DockerSpawner.network_name = network_name + +# Explicitly set notebook directory because we'll be mounting a volume to it. +# Most `jupyter/docker-stacks` *-notebook images run the Notebook server as +# user `jovyan`, and set the notebook directory to `/home/jovyan/work`. +# We follow the same convention. +notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work") +c.DockerSpawner.notebook_dir = notebook_dir + +# Mount the real user's Docker volume on the host to the notebook user's +# notebook directory in the container +c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} + +# Remove containers once they are stopped +c.DockerSpawner.remove = True + +# For debugging arguments passed to spawned containers +c.DockerSpawner.debug = True + +# User containers will access hub by container name on the Docker network +c.JupyterHub.hub_ip = "jupyterhub" +c.JupyterHub.hub_port = 8080 + +# Persist hub data on volume mounted inside container +# c.JupyterHub.cookie_secret_file = "/data/jupyterhub_cookie_secret" +c.JupyterHub.db_url = "sqlite:////data/jupyterhub.sqlite" + +# # Authenticate users with Native Authenticator +# c.JupyterHub.authenticator_class = "nativeauthenticator.NativeAuthenticator" +# +# # Allow anyone to sign-up without approval +# c.NativeAuthenticator.open_signup = True + +# Authenticate with Custom Token Authenticator +from customauthenticator.custom import CustomTokenAuthenticator +c.Spawner.cmd = ['start.sh', 'jupyterhub-singleuser', '--allow-root'] +c.KubeSpawner.args = ['--allow-root'] +c.JupyterHub.authenticator_class = CustomTokenAuthenticator +# TODO:Change this keycloak_url as required +c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % (os.getenv('KEYCLOAK_HOSTNAME'), os.getenv('KEYCLOAK_REALM')) +c.CustomTokenAuthenticator.auth_cookie_header= "Authorization" +c.CustomTokenAuthenticator.auth_username_key= "preferred_username" +c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" +c.CustomTokenAuthenticator.enable_auth_state = True +c.CustomTokenAuthenticator.auto_login = True +c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv('KEYCLOAK_HOSTNAME') + +c.JupyterHub.cookie_secret = os.getenv('JUPYTERHUB_CRYPT_KEY') + +# Allowed admins +admin = os.environ.get("JUPYTERHUB_ADMIN") +if admin: + c.Authenticator.admin_users = [admin] From 2e3992d8ead10071baf8e8ef71c582d7045464ce Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Wed, 22 May 2024 11:45:55 -0500 Subject: [PATCH 02/24] Linting --- docker-dev.sh | 6 +- .../docs/devs/standalone-docker-deployment.md | 2 +- frontend/src/components/Layout.tsx | 10 +- jupyterhub/.env-example | 2 +- jupyterhub/Dockerfile.jupyterhub | 2 +- .../customauthenticator/custom.py | 113 +++++++++++------- jupyterhub/authenticator/setup.py | 21 ++-- jupyterhub/authenticator/test_jwt.py | 31 ++--- jupyterhub/jupyterhub_config.py | 22 ++-- 9 files changed, 118 insertions(+), 91 deletions(-) diff --git a/docker-dev.sh b/docker-dev.sh index 2c3fd1a00..6e08082b4 100755 --- a/docker-dev.sh +++ b/docker-dev.sh @@ -7,12 +7,12 @@ if [ "$1" = "down" ] then docker-compose -f docker-compose.dev.yml -p clowder2-dev down fi -if [ "$1" = "jupyter" ] & [ "$2" = "up" ] +if [ "$1" = "jupyter" ] && [ "$2" = "up" ] then docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev up -d --build fi -if [ "$1" = "jupyter" ] & [ "$2" = "down" ] +if [ "$1" = "jupyter" ] && [ "$2" = "down" ] then docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev down -fi \ No newline at end of file +fi diff --git a/docs/docs/devs/standalone-docker-deployment.md b/docs/docs/devs/standalone-docker-deployment.md index 64680772d..bc9b3ef0c 100644 --- a/docs/docs/devs/standalone-docker-deployment.md +++ b/docs/docs/devs/standalone-docker-deployment.md @@ -60,4 +60,4 @@ docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d 4. Access the keycloak admin console at `http://{{IP_ADDRESS}}/keycloak/` and login with the credentials from the `docker-compose.yml` file and finish the steps in [Keycloak](keycloak.md), using the same IP address as the `{hostname}` placeholder. -You shall be able to log in now at `http://{{IP_ADDRESS}}/`. \ No newline at end of file +You shall be able to log in now at `http://{{IP_ADDRESS}}/`. diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index a7a4a0869..cc485aff9 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -23,7 +23,7 @@ import { RootState } from "../types/data"; import { AddBox, Explore } from "@material-ui/icons"; import HistoryIcon from "@mui/icons-material/History"; import GroupIcon from "@mui/icons-material/Group"; -import MenuBookIcon from '@mui/icons-material/MenuBook'; +import MenuBookIcon from "@mui/icons-material/MenuBook"; import Gravatar from "react-gravatar"; import PersonIcon from "@mui/icons-material/Person"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; @@ -424,8 +424,12 @@ export default function PersistentDrawerLeft(props) { {/*TODO: Need to make link dynamic */} - + diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example index 8092bde32..0df50c06b 100644 --- a/jupyterhub/.env-example +++ b/jupyterhub/.env-example @@ -9,4 +9,4 @@ JUPYTERHUB_ADMIN="admin" DOCKER_NETWORK_NAME="clowder2" DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" DOCKER_NOTEBOOK_DIR="/home/jovyan/work" -JUPYTERHUB_CRYPT_KEY="" \ No newline at end of file +JUPYTERHUB_CRYPT_KEY="" diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index 9956c7ab8..349782ad7 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -11,4 +11,4 @@ WORKDIR /tmp/authenticator/ COPY authenticator /tmp/authenticator/ RUN pip3 install /tmp/authenticator -CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] \ No newline at end of file +CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 1d9b8b101..6f555a7bb 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -1,87 +1,90 @@ +import json import os import urllib.parse -import json from jose import jwt -from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError +from tornado import web +from traitlets import Unicode + from jupyterhub.auth import Authenticator from jupyterhub.handlers import LoginHandler, LogoutHandler -from traitlets import Unicode -from tornado import web -import requests class CustomTokenAuthenticator(Authenticator): """ - Accept the authenticated Access Token from cookie. + Accept the authenticated Access Token from cookie. """ + auth_cookie_header = Unicode( - os.environ.get('AUTH_COOKIE_HEADER', ''), + os.environ.get("AUTH_COOKIE_HEADER", ""), config=True, help="the cookie header we put in browser to retrieve token", ) auth_username_key = Unicode( - os.environ.get('AUTH_USERNAME_KEY', ''), + os.environ.get("AUTH_USERNAME_KEY", ""), config=True, help="the key to retreive username from the json", ) - - landing_page_login_url = Unicode( - os.environ.get('LANDING_PAGE_LOGIN_URL', ''), + os.environ.get("LANDING_PAGE_LOGIN_URL", ""), config=True, help="the landing page login entry", ) keycloak_url = Unicode( - os.environ.get('KEYCLOAK_URL', ''), + os.environ.get("KEYCLOAK_URL", ""), config=True, help="the URL where keycloak is installed", ) keycloak_audience = Unicode( - os.environ.get('KEYCLOAK_AUDIENCE', ''), + os.environ.get("KEYCLOAK_AUDIENCE", ""), config=True, help="the audience for keycloak to check", ) keycloak_pem_key = Unicode( - os.environ.get('KEYCLOAK_PEM_KEY', ''), + os.environ.get("KEYCLOAK_PEM_KEY", ""), config=True, help="the RSA pem key with proper header and footer (deprecated)", ) space_service_url = Unicode( - os.environ.get('SPACE_SERVICE_URL', ''), + os.environ.get("SPACE_SERVICE_URL", ""), config=True, - help="the internal space service url" + help="the internal space service url", ) quotas = None def get_handlers(self, app): return [ - (r'/', LoginHandler), - (r'/user', LoginHandler), - (r'/lab', LoginHandler), - (r'/login', LoginHandler), - (r'/logout', CustomTokenLogoutHandler), + (r"/", LoginHandler), + (r"/user", LoginHandler), + (r"/lab", LoginHandler), + (r"/login", LoginHandler), + (r"/logout", CustomTokenLogoutHandler), ] def get_keycloak_pem(self): if not self.keycloak_url: - raise web.HTTPError(500, log_message="JupyterHub is not correctly configured.") + raise web.HTTPError( + 500, log_message="JupyterHub is not correctly configured." + ) # fetch the key response = urllib.request.urlopen(self.keycloak_url) if response.code >= 200 or response <= 299: - encoding = response.info().get_content_charset('utf-8') + encoding = response.info().get_content_charset("utf-8") result = json.loads(response.read().decode(encoding)) - self.keycloak_pem_key = f"-----BEGIN PUBLIC KEY-----\n" \ - f"{result['public_key']}\n" \ - f"-----END PUBLIC KEY-----" + self.keycloak_pem_key = ( + f"-----BEGIN PUBLIC KEY-----\n" + f"{result['public_key']}\n" + f"-----END PUBLIC KEY-----" + ) else: raise web.HTTPError(500, log_message="Could not get key from keycloak.") @@ -92,15 +95,19 @@ def check_jwt_token(self, access_token): # make sure audience is set if not self.keycloak_audience: - raise web.HTTPError(403, log_message="JupyterHub is not correctly configured.") + raise web.HTTPError( + 403, log_message="JupyterHub is not correctly configured." + ) # no token in the cookie if not access_token: raise web.HTTPError(401, log_message="Please login to access Clowder.") # make sure it is a valid token - if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': - raise web.HTTPError(403, log_message="Token format not valid, it has to be bearer xxxx!") + if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != "Bearer": + raise web.HTTPError( + 403, log_message="Token format not valid, it has to be bearer xxxx!" + ) # decode jwt token instead of sending it to userinfo endpoint: access_token = access_token.split(" ")[1] @@ -109,17 +116,27 @@ def check_jwt_token(self, access_token): try: resp_json = jwt.decode(access_token, public_key, audience=audience) except ExpiredSignatureError: - raise web.HTTPError(403, log_message='JWT Expired Signature Error: token signature has expired') + raise web.HTTPError( + 403, + log_message="JWT Expired Signature Error: token signature has expired", + ) except JWTClaimsError: - raise web.HTTPError(403, log_message='JWT Claims Error: token signature is invalid') + raise web.HTTPError( + 403, log_message="JWT Claims Error: token signature is invalid" + ) except JWTError: - raise web.HTTPError(403, log_message='JWT Error: token signature is invalid') - except Exception as e: + raise web.HTTPError( + 403, log_message="JWT Error: token signature is invalid" + ) + except Exception: raise web.HTTPError(403, log_message="Not a valid jwt token!") # make sure we know username if self.auth_username_key not in resp_json.keys(): - raise web.HTTPError(500, log_message=f"Required field {self.auth_username_key} does not exist in jwt token") + raise web.HTTPError( + 500, + log_message=f"Required field {self.auth_username_key} does not exist in jwt token", + ) username = resp_json[self.auth_username_key] # # make sure there is a user id @@ -135,19 +152,20 @@ def check_jwt_token(self, access_token): else: user_roles = [] - self.log.info(f"username={username}") return { - 'name': username, - 'auth_state': { - 'roles': user_roles, + "name": username, + "auth_state": { + "roles": user_roles, }, } async def authenticate(self, handler, data): self.log.info("Authenticate") try: - access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) + access_token = urllib.parse.unquote( + handler.get_cookie(self.auth_cookie_header, "") + ) if not access_token: raise web.HTTPError(401, log_message="Please login to access Clowder.") @@ -156,12 +174,14 @@ async def authenticate(self, handler, data): return user except web.HTTPError as e: if e.log_message: - error_msg = urllib.parse.quote(e.log_message.encode('utf-8')) + error_msg = urllib.parse.quote(e.log_message.encode("utf-8")) else: - error_msg = urllib.parse.quote(f"Error {e}".encode('utf-8')) + ". Please login to access Clowder." + error_msg = ( + urllib.parse.quote(f"Error {e}".encode("utf-8")) + + ". Please login to access Clowder." + ) handler.redirect(f"{self.landing_page_login_url}?error={error_msg}") - # async def pre_spawn_start(self, user, spawner): # auth_state = await user.get_auth_state() # if not auth_state: @@ -185,6 +205,7 @@ async def authenticate(self, handler, data): # spawner.mem_guarantee = "2G" # spawner.mem_limit = "4G" + # # # This is called from the jupyterlab so there is no cookies that this depends on # async def refresh_user(self, user, handler): @@ -205,12 +226,12 @@ async def authenticate(self, handler, data): class CustomTokenLogoutHandler(LogoutHandler): - async def handle_logout(self): # remove clowder token on logout self.log.info("Remove clowder token on logout") - error_msg = "You have logged out of Clowder system from Clowder . Please login again if you want to use " \ - "Clowder components." + error_msg = ( + "You have logged out of Clowder system from Clowder . Please login again if you want to use " + "Clowder components." + ) self.set_cookie(self.authenticator.auth_cookie_header, "") self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") - diff --git a/jupyterhub/authenticator/setup.py b/jupyterhub/authenticator/setup.py index 7e37c3d7a..0314f36d2 100644 --- a/jupyterhub/authenticator/setup.py +++ b/jupyterhub/authenticator/setup.py @@ -1,17 +1,12 @@ from setuptools import setup setup( - name='customauthenticator', - version='0.8.0', - description='Custom Authenticator for JupyterHub', - author='cwang138', - author_email='cwang138@illinois.edu', - license='MPL 2.0', - packages=['customauthenticator'], - install_requires=[ - 'jupyterhub', - 'pyjwt', - 'requests', - 'python-jose' - ] + name="customauthenticator", + version="0.8.0", + description="Custom Authenticator for JupyterHub", + author="cwang138", + author_email="cwang138@illinois.edu", + license="MPL 2.0", + packages=["customauthenticator"], + install_requires=["jupyterhub", "pyjwt", "requests", "python-jose"], ) diff --git a/jupyterhub/authenticator/test_jwt.py b/jupyterhub/authenticator/test_jwt.py index 12df103db..e86f6bc96 100644 --- a/jupyterhub/authenticator/test_jwt.py +++ b/jupyterhub/authenticator/test_jwt.py @@ -1,25 +1,27 @@ -from jose import jwt -from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError -import os -import urllib.request import json +import urllib.request + +from jose import jwt +from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError response = urllib.request.urlopen("") if response.code >= 200 or response <= 299: - encoding = response.info().get_content_charset('utf-8') + encoding = response.info().get_content_charset("utf-8") result = json.loads(response.read().decode(encoding)) - public_key = f"-----BEGIN PUBLIC KEY-----\n" \ - f"{result['public_key']}\n" \ - f"-----END PUBLIC KEY-----" + public_key = ( + f"-----BEGIN PUBLIC KEY-----\n" + f"{result['public_key']}\n" + f"-----END PUBLIC KEY-----" + ) else: print("Could not get key from keycloak.") -access_token ="" +access_token = "" # make sure it is a valid token -if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': +if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != "Bearer": print("Token format not valid, it has to be bearer xxxx!") # decode jwt token instead of sending it to userinfo endpoint: @@ -30,11 +32,10 @@ print(decoded) except ExpiredSignatureError: - print('JWT Expired Signature Error: token signature has expired') + print("JWT Expired Signature Error: token signature has expired") except JWTClaimsError: - print('JWT Claims Error: token signature is invalid') + print("JWT Claims Error: token signature is invalid") except JWTError: - print('JWT Error: token signature is invalid') -except Exception as e: + print("JWT Error: token signature is invalid") +except Exception: print("Not a valid jwt token!") - diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1e704e316..11ce4c850 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -4,6 +4,8 @@ # Configuration file for JupyterHub import os +from customauthenticator.custom import CustomTokenAuthenticator + c = get_config() # noqa: F821 # We rely on environment variables to configure JupyterHub so that we @@ -53,20 +55,24 @@ # c.NativeAuthenticator.open_signup = True # Authenticate with Custom Token Authenticator -from customauthenticator.custom import CustomTokenAuthenticator -c.Spawner.cmd = ['start.sh', 'jupyterhub-singleuser', '--allow-root'] -c.KubeSpawner.args = ['--allow-root'] +c.Spawner.cmd = ["start.sh", "jupyterhub-singleuser", "--allow-root"] +c.KubeSpawner.args = ["--allow-root"] c.JupyterHub.authenticator_class = CustomTokenAuthenticator # TODO:Change this keycloak_url as required -c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % (os.getenv('KEYCLOAK_HOSTNAME'), os.getenv('KEYCLOAK_REALM')) -c.CustomTokenAuthenticator.auth_cookie_header= "Authorization" -c.CustomTokenAuthenticator.auth_username_key= "preferred_username" +c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), +) +c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" +c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True -c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv('KEYCLOAK_HOSTNAME') +c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( + "KEYCLOAK_HOSTNAME" +) -c.JupyterHub.cookie_secret = os.getenv('JUPYTERHUB_CRYPT_KEY') +c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") # Allowed admins admin = os.environ.get("JUPYTERHUB_ADMIN") From ee6ec0061a5145fb66f94f936f0b9058637297f9 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Thu, 23 May 2024 12:27:02 -0500 Subject: [PATCH 03/24] Handling logout --- docker-compose.dev.yml | 4 +++ jupyterhub/.env-example | 2 ++ .../customauthenticator/custom.py | 10 ++++-- jupyterhub/jupyterhub_config.py | 32 +++++++++++++++---- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a6ff21566..61dcc8bf0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -67,6 +67,8 @@ services: postgres: image: postgres + networks: + - clowder2 volumes: - postgres_data:/var/lib/postgresql/data environment: @@ -79,6 +81,8 @@ services: volumes: - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro + networks: + - clowder2 command: - start-dev - --http-relative-path /keycloak diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example index 0df50c06b..dcb467521 100644 --- a/jupyterhub/.env-example +++ b/jupyterhub/.env-example @@ -10,3 +10,5 @@ DOCKER_NETWORK_NAME="clowder2" DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" DOCKER_NOTEBOOK_DIR="/home/jovyan/work" JUPYTERHUB_CRYPT_KEY="" +CLOWDER_URL="localhost:3000" +PROD_DEPLOYMENT="false" diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 6f555a7bb..435def5c1 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -34,6 +34,12 @@ class CustomTokenAuthenticator(Authenticator): help="the landing page login entry", ) + landing_page_logout_url = Unicode( + os.environ.get("LANDING_PAGE_LOGOUT", ""), + config=True, + help="the landing page logout entry", + ) + keycloak_url = Unicode( os.environ.get("KEYCLOAK_URL", ""), config=True, @@ -229,9 +235,9 @@ class CustomTokenLogoutHandler(LogoutHandler): async def handle_logout(self): # remove clowder token on logout self.log.info("Remove clowder token on logout") - error_msg = ( + self.log.info( "You have logged out of Clowder system from Clowder . Please login again if you want to use " "Clowder components." ) self.set_cookie(self.authenticator.auth_cookie_header, "") - self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") + self.redirect(f"{self.authenticator.landing_page_logout_url}") diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 11ce4c850..136f80962 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -59,18 +59,36 @@ c.KubeSpawner.args = ["--allow-root"] c.JupyterHub.authenticator_class = CustomTokenAuthenticator # TODO:Change this keycloak_url as required -c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( - os.getenv("KEYCLOAK_HOSTNAME"), - os.getenv("KEYCLOAK_REALM"), -) + c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True -c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( - "KEYCLOAK_HOSTNAME" -) + +if os.getenv("PROD_DEPLOYMENT") == "true": + c.CustomTokenAuthenticator.keycloak_url = "https://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "https://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) + +else: + c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "http://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "http://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") From ce129fe50eb7020a510016fa132f6027c8b03027 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 11:39:55 -0500 Subject: [PATCH 04/24] Updating docker-compose to keep jupyterhub version static --- docker-compose.jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml index 333ca5555..6179f8937 100644 --- a/docker-compose.jupyter.yml +++ b/docker-compose.jupyter.yml @@ -5,7 +5,7 @@ services: context: jupyterhub dockerfile: Dockerfile.jupyterhub args: - JUPYTERHUB_VERSION: latest + JUPYTERHUB_VERSION: 4 restart: always networks: - clowder2 From 15638b0144e36f1c0a8bd80e2574a54d5b77dcda Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 24 May 2024 15:34:07 -0500 Subject: [PATCH 05/24] Prepopulating attempy --- jupyterhub/Clowder_APIs.ipynb | 151 +++++++++++++++++++++++++++++++ jupyterhub/Dockerfile.jupyterlab | 21 +++++ jupyterhub/jupyterhub_config.py | 17 +++- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 jupyterhub/Clowder_APIs.ipynb create mode 100644 jupyterhub/Dockerfile.jupyterlab diff --git a/jupyterhub/Clowder_APIs.ipynb b/jupyterhub/Clowder_APIs.ipynb new file mode 100644 index 000000000..f975f1bc9 --- /dev/null +++ b/jupyterhub/Clowder_APIs.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6dfcb963-cf69-4505-9871-5cc13471f5dd", + "metadata": {}, + "source": [ + "## Clowder APIs" + ] + }, + { + "cell_type": "markdown", + "id": "310051ad-4262-42fc-ac28-911f92842a7e", + "metadata": {}, + "source": [ + "## Import libraries and setup utility function\n", + "\n", + "We start by importing the rquired libraries, data and setting up some utility functions and variables that we will use below." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "ffec7cb7-1a82-4148-aad1-bb3b1b19915b", + "metadata": {}, + "outputs": [], + "source": [ + "import pyclowder\n", + "import json\n", + "import os\n", + "import pandas as pd\n", + "\n", + "import requests\n", + "\n", + "# Function to download the IRIS dataset\n", + "def download_iris_dataset():\n", + " # URL for the Iris dataset hosted by UCI Machine Learning Repository\n", + " url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data\"\n", + " \n", + " response = requests.get(url)\n", + " \n", + " if response.status_code == 200:\n", + " with open(\"iris.csv\", \"wb\") as f:\n", + " f.write(response.content)\n", + " print(\"The Iris dataset has been downloaded and saved as iris.csv.\")\n", + " else:\n", + " print(\"Failed to download the dataset. Status code:\", response.status_code)\n", + "\n", + "\n", + "CLOWDER_URL = \"http://localhost:8000\"" + ] + }, + { + "cell_type": "markdown", + "id": "f8d5c0ae-659b-4a52-a30a-0fa8b7051694", + "metadata": {}, + "source": [ + "## Token Generation" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "59fca5f8-a5d6-419d-835a-023a73c5a1d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzcwNTgsImlhdCI6MTcxNjU3Njc1OCwianRpIjoiZjBjYmZlYzYtZjJlMS00NDg0LTg0YWMtNzFjMGMwZWNmNTNlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiIzOTgyN2EyNy1kMDA5LTRiY2YtOGIzYS1jZTk1MGM3YTI4OWQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiMzk4MjdhMjctZDAwOS00YmNmLThiM2EtY2U5NTBjN2EyODlkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.Z3upy5xkoXfTwILujF7C8vpE_XaL8F32vuRd67V2TVqUK4qEVuN_5yTsyCcXyte9osG2jeZTHlJXreekOrpTvOW4Qbel1JAHnCQfGmlri_io-XFE57aT379HZrLGVlrpiHOGTqKG5mDVxIbaHEqdWhJkAEWY55rDqpkCScHUP7_SoJgAUj9Xx_LSS1hDHLwBCYRvGPFEoMADM40F-P3qxkW5Qsv9gPqP5ChpfG_7KZeBMRhtFXH_xk3M18qjNsSiQ9QKBFC2IrLFnT89bCK_slaUfUWLyiONt6ASPkAw85aajD3jc8RKq_AM42a2-wFVF7vERGPnoMscOpvjp9lBiQ'}\n" + ] + } + ], + "source": [ + "user_login_json = {\n", + " \"email\": \"mohanar2@illinois.edu\",\n", + " \"password\": \"password\"\n", + "}\n", + "login_url = CLOWDER_URL + \"/api/v2/login\"\n", + "response = requests.post(login_url, json = user_login_json)\n", + "token = response.json()[\"token\"]\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "id": "2b273e10-8efc-46a8-b220-b63354735c49", + "metadata": {}, + "source": [ + "## Creating Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "91639e11-b553-4b57-a0df-6e91c661662f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '6650e202fcd9057e97ae2d8a', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True}, 'created': '2024-05-24T18:52:50.457552', 'modified': '2024-05-24T18:52:50.457556', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" + ] + } + ], + "source": [ + "dataset_json = {\n", + " \"name\": \"Flower Dataset\",\n", + " \"description\": \"Dataset for Flower Data\",\n", + " \"status\": \"PRIVATE\",\n", + "}\n", + "dataset_params = {\n", + " \"license_id\": \"CC BY\"\n", + "}\n", + "\n", + "token = \"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzY5NjcsImlhdCI6MTcxNjU3NjY2NywiYXV0aF90aW1lIjoxNzE2NTc2Mzc1LCJqdGkiOiI5NzU3NTM1Mi04Y2FhLTRhMmUtODZhZC0wM2JkNDcwODhhZjkiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAva2V5Y2xvYWsvcmVhbG1zL2Nsb3dkZXIiLCJzdWIiOiJmODRjYmM2ZC1jMTNkLTQyZWYtYWE3Yy0xZDgyYWNjNWFlZWIiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjbG93ZGVyMi1iYWNrZW5kIiwic2Vzc2lvbl9zdGF0ZSI6IjdhMjJjZDFkLTQ3NDctNDAyNS1iNDk2LTUyYzNhOWUwZDhiYyIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0OjgwMDAiXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImRlZmF1bHQtcm9sZXMtY2xvd2RlciIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXX0sInNjb3BlIjoib3BlbmlkIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3YTIyY2QxZC00NzQ3LTQwMjUtYjQ5Ni01MmMzYTllMGQ4YmMiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwibmFtZSI6IlZpc21heWFrIE1vaGFuYXJhamFuIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1IiwiZ2l2ZW5fbmFtZSI6IlZpc21heWFrIiwiZmFtaWx5X25hbWUiOiJNb2hhbmFyYWphbiIsImVtYWlsIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1In0.AHqeZ79L8OUW2zTYEaDeUmuhHAeDMcqlgR8CE7ScGBRt-4gj8QFipbkiiMt0LYbMNi97dv0O_-iojrMtnlaX1OubnEwI_mMcoGckpXc5CzCoAQ_Qri2ZnpfnClDa9wPu-G_aGL-Sv4UpAglSTCbpBxsE99EiBubMb0T3NpP8p0k_lJTSvAZPJdMJ2sPpBo4BPUUIZq9JBXAZF7YPL9ZzUOoQKJvpNlA_7fHBHDjdSKRbVjP5cCNldjHJ70D2j1HM4JJUwMirNWZ8SFaJiny5a7NoZ2fa_JQkA3ZRrwGPHxGh6JThhu_F-a-pv2CDSytPCSU5DJxnslNJ4ePLVKDWpQ\"\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", + "\n", + "\n", + "dataset_url = CLOWDER_URL + \"/api/v2/datasets\"\n", + "response = requests.post(dataset_url, json = dataset_json, headers = headers, params = dataset_params)\n", + "print(response.json())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab new file mode 100644 index 000000000..ee0bb22f6 --- /dev/null +++ b/jupyterhub/Dockerfile.jupyterlab @@ -0,0 +1,21 @@ +# Base Image +FROM quay.io/jupyter/base-notebook:latest + +# Install additional packages +USER root +RUN apt-get -qq update && apt-get install -y --no-install-recommends \ + curl \ + git \ + zip unzip \ + nano \ + vim-tiny \ + lsof && \ + rm -rf /var/lib/apt/lists/* + +USER $NB_USER + +# Install Python packages +RUN pip install --no-cache-dir \ + requests \ + pyclowder \ + pandas diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 136f80962..58105897f 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -3,6 +3,8 @@ # Configuration file for JupyterHub import os +import shutil +import logging from customauthenticator.custom import CustomTokenAuthenticator @@ -61,7 +63,7 @@ # TODO:Change this keycloak_url as required c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" -c.CustomTokenAuthenticator.auth_username_key = "preferred_username" +c.CustomTokenAuthenticator.auth_username_key = "email" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True @@ -92,7 +94,20 @@ c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") +# Allow all users to access +c.Authenticator.allow_all = True + # Allowed admins admin = os.environ.get("JUPYTERHUB_ADMIN") if admin: c.Authenticator.admin_users = [admin] + + +# Pre spawn hook +# def pre_spawn_hook(spawner): +# # Git clone +# +# + + +# c.Spawner.post_stop_hook = pre_spawn_hook From f04103ad6ef9ac17a78d9854e27eaf7be78cfb2c Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Sun, 26 May 2024 19:16:59 -0500 Subject: [PATCH 06/24] Not working --- jupyterhub/Dockerfile.jupyterhub | 3 +-- jupyterhub/Dockerfile.jupyterlab | 3 +++ jupyterhub/jupyterhub_config.py | 18 ++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index 349782ad7..a27ff8c65 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -1,5 +1,4 @@ -ARG JUPYTERHUB_VERSION -FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:latest # Install dockerspawner, # hadolint ignore=DL3013 diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index ee0bb22f6..06afd0deb 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -19,3 +19,6 @@ RUN pip install --no-cache-dir \ requests \ pyclowder \ pandas + +# Copy Clowder API notebook +COPY Clowder_APIs.ipynb /etc/jupyter/Clowder_APIs.ipynb diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 58105897f..1e4cdb99d 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -1,10 +1,10 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. + # Configuration file for JupyterHub import os import shutil -import logging from customauthenticator.custom import CustomTokenAuthenticator @@ -31,11 +31,13 @@ # We follow the same convention. notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work") c.DockerSpawner.notebook_dir = notebook_dir +c.Spawner.args = ["--NotebookApp.default_url=/notebooks/Welcome.ipynb"] # Mount the real user's Docker volume on the host to the notebook user's # notebook directory in the container c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} + # Remove containers once they are stopped c.DockerSpawner.remove = True @@ -104,10 +106,14 @@ # Pre spawn hook -# def pre_spawn_hook(spawner): -# # Git clone -# -# +def post_spawn_hook(spawner, auth_state): + username = spawner.user.name + spawner.environment["GREETING"] = f"Hello Master {username}" + + target_file_path = f"/home/jovyan/work/Clowder_APIs.ipynb" + + if not os.path.exists(target_file_path): + shutil.copy2("/etc/jupyter/Clowder_APIs.ipynb", target_file_path) -# c.Spawner.post_stop_hook = pre_spawn_hook +# c.Spawner.auth_state_hook = post_spawn_hook From 6718d5a67cd0635c8374a6a1ed4047ee74c5483e Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 31 May 2024 09:29:18 -0500 Subject: [PATCH 07/24] not working --- jupyterhub/Dockerfile.jupyterlab | 7 +++-- jupyterhub/jupyterhub_config.py | 48 +++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index 06afd0deb..84d08cc7f 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -14,11 +14,12 @@ RUN apt-get -qq update && apt-get install -y --no-install-recommends \ USER $NB_USER +COPY Clowder_APIs.ipynb /home/jovyan/work/ + +COPY Clowder_APIs.ipynb /home/jovyan/ + # Install Python packages RUN pip install --no-cache-dir \ requests \ pyclowder \ pandas - -# Copy Clowder API notebook -COPY Clowder_APIs.ipynb /etc/jupyter/Clowder_APIs.ipynb diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1e4cdb99d..1321a18a8 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -4,7 +4,6 @@ # Configuration file for JupyterHub import os -import shutil from customauthenticator.custom import CustomTokenAuthenticator @@ -14,6 +13,38 @@ # avoid having to rebuild the JupyterHub container every time we change a # configuration parameter. + +def create_dir_hook(spawner): + username = spawner.user.name # get the username + + volume_path = os.path.join("/volumes/jupyterhub/", username) + print(f"Checking if {volume_path} exists…") + + if not os.path.exists(volume_path): + print(f"{volume_path} does not exist. Creating directory...") + # create a directory with umask 0755 + # hub and container user must have the same UID to be writeable + # still readable by other users on the system + try: + # Attempt to create the directory + os.mkdir(volume_path) + print(f"Directory {volume_path} created.") + # Now, do whatever you think your user needs + # ... + except OSError as e: + print(f"Error creating directory {volume_path}: {e}") + + try: + os.chown(volume_path, 1003, 1003) + print("Ownership of changed successfully.") + except OSError as e: + print(f"Error changing ownership of: {e}") + # now do whatever you think your user needs + + +# attach the hook function to the spawner +c.Spawner.pre_spawn_hook = create_dir_hook + # Spawn single-user servers as Docker containers c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" @@ -37,7 +68,6 @@ # notebook directory in the container c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} - # Remove containers once they are stopped c.DockerSpawner.remove = True @@ -103,17 +133,3 @@ admin = os.environ.get("JUPYTERHUB_ADMIN") if admin: c.Authenticator.admin_users = [admin] - - -# Pre spawn hook -def post_spawn_hook(spawner, auth_state): - username = spawner.user.name - spawner.environment["GREETING"] = f"Hello Master {username}" - - target_file_path = f"/home/jovyan/work/Clowder_APIs.ipynb" - - if not os.path.exists(target_file_path): - shutil.copy2("/etc/jupyter/Clowder_APIs.ipynb", target_file_path) - - -# c.Spawner.auth_state_hook = post_spawn_hook From fcf9d470f29fca1e06c1e9260f14c5d078aa307b Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:00:13 -0500 Subject: [PATCH 08/24] Updating lab dockerfile --- jupyterhub/Dockerfile.jupyterlab | 2 -- 1 file changed, 2 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterlab b/jupyterhub/Dockerfile.jupyterlab index 84d08cc7f..dd9c2e4f4 100644 --- a/jupyterhub/Dockerfile.jupyterlab +++ b/jupyterhub/Dockerfile.jupyterlab @@ -16,8 +16,6 @@ USER $NB_USER COPY Clowder_APIs.ipynb /home/jovyan/work/ -COPY Clowder_APIs.ipynb /home/jovyan/ - # Install Python packages RUN pip install --no-cache-dir \ requests \ From be74dbe11e15b794c67fab3f73d2177b90ed7446 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:00:45 -0500 Subject: [PATCH 09/24] Updating config --- jupyterhub/jupyterhub_config.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1321a18a8..1cdd33b84 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -13,38 +13,6 @@ # avoid having to rebuild the JupyterHub container every time we change a # configuration parameter. - -def create_dir_hook(spawner): - username = spawner.user.name # get the username - - volume_path = os.path.join("/volumes/jupyterhub/", username) - print(f"Checking if {volume_path} exists…") - - if not os.path.exists(volume_path): - print(f"{volume_path} does not exist. Creating directory...") - # create a directory with umask 0755 - # hub and container user must have the same UID to be writeable - # still readable by other users on the system - try: - # Attempt to create the directory - os.mkdir(volume_path) - print(f"Directory {volume_path} created.") - # Now, do whatever you think your user needs - # ... - except OSError as e: - print(f"Error creating directory {volume_path}: {e}") - - try: - os.chown(volume_path, 1003, 1003) - print("Ownership of changed successfully.") - except OSError as e: - print(f"Error changing ownership of: {e}") - # now do whatever you think your user needs - - -# attach the hook function to the spawner -c.Spawner.pre_spawn_hook = create_dir_hook - # Spawn single-user servers as Docker containers c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" From 4f368c9a76309d7f598dcae06c4fb2d0e5099c46 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:04:49 -0500 Subject: [PATCH 10/24] Incorporating feedback --- docs/docs/devs/getstarted.md | 2 ++ frontend/src/app.config.ts | 5 +++++ frontend/src/components/Layout.tsx | 4 +++- .../customauthenticator/custom.py | 20 +------------------ 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/docs/devs/getstarted.md b/docs/docs/devs/getstarted.md index bbf8f1633..6eb3b72aa 100644 --- a/docs/docs/devs/getstarted.md +++ b/docs/docs/devs/getstarted.md @@ -47,6 +47,8 @@ section below). - Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers, provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`. - Running `./docker-dev.sh down` brings down the required services. +- If you want to run the jupyterhub, you can run `./docker-dev.sh jupyter up`. The jupyterhub will be available at + `http://localhost:8765`. You can bring it down using `./docker-dev.sh jupyter down`. **Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`. diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index f06933279..b0a91bf63 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -9,6 +9,7 @@ interface Config { hostname: string; apikey: string; GHIssueBaseURL: string; + jupyterHubURL: string; KeycloakBaseURL: string; KeycloakLogin: string; KeycloakLogout: string; @@ -62,6 +63,10 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`; // elasticsearch config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`; +// jupterhub +const localJupyterhubURL: string = "http://localhost:8765"; +config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL; + // refresh token time interval config["refreshTokenInterval"] = 1000 * 60; // 1 minute // updated extractor logs diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index cc485aff9..d02c64d54 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -41,6 +41,8 @@ import ManageAccountsIcon from "@mui/icons-material/ManageAccounts"; import AdminPanelSettingsIcon from "@mui/icons-material/AdminPanelSettings"; import { Footer } from "./navigation/Footer"; +import config from "../app.config"; + const drawerWidth = 240; const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{ @@ -426,7 +428,7 @@ export default function PersistentDrawerLeft(props) { diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 435def5c1..7dfb05962 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -145,26 +145,8 @@ def check_jwt_token(self, access_token): ) username = resp_json[self.auth_username_key] - # # make sure there is a user id - # if self.auth_uid_number_key not in resp_json.keys(): - # raise web.HTTPError(500, log_message=f"Required field {self.auth_uid_number_key} does not exist in jwt token") - # uid = resp_json[self.auth_uid_number_key] - # - # get the groups/roles for the user - if "roles" in resp_json: - user_roles = resp_json.get("roles", []) - elif "realm_access" in resp_json: - user_roles = resp_json["realm_access"].get("roles", []) - else: - user_roles = [] - self.log.info(f"username={username}") - return { - "name": username, - "auth_state": { - "roles": user_roles, - }, - } + return {"name": username} async def authenticate(self, handler, data): self.log.info("Authenticate") From 494d7d92eb72e32e4564f7550d4ccf6838017664 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Tue, 21 May 2024 10:25:47 -0500 Subject: [PATCH 11/24] Working Jupyterhub Integration --- docker-compose.dev.yml | 1 + docker-compose.jupyter.yml | 30 +++ docker-dev.sh | 9 + frontend/src/components/Layout.tsx | 14 ++ jupyterhub/.env-example | 12 + jupyterhub/Dockerfile.jupyterhub | 14 ++ .../customauthenticator/__init__.py | 0 .../customauthenticator/custom.py | 216 ++++++++++++++++++ jupyterhub/authenticator/setup.py | 17 ++ jupyterhub/authenticator/test_jwt.py | 40 ++++ jupyterhub/jupyterhub_config.py | 74 ++++++ 11 files changed, 427 insertions(+) create mode 100644 docker-compose.jupyter.yml create mode 100644 jupyterhub/.env-example create mode 100644 jupyterhub/Dockerfile.jupyterhub create mode 100644 jupyterhub/authenticator/customauthenticator/__init__.py create mode 100644 jupyterhub/authenticator/customauthenticator/custom.py create mode 100644 jupyterhub/authenticator/setup.py create mode 100644 jupyterhub/authenticator/test_jwt.py create mode 100644 jupyterhub/jupyterhub_config.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 92df388f8..a6ff21566 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -172,6 +172,7 @@ services: networks: clowder2: + name: clowder2 ## By default this config uses default local driver, ## For custom volumes replace with volume driver configuration. diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml new file mode 100644 index 000000000..333ca5555 --- /dev/null +++ b/docker-compose.jupyter.yml @@ -0,0 +1,30 @@ +version: '3' +services: + jupyterhub: + build: + context: jupyterhub + dockerfile: Dockerfile.jupyterhub + args: + JUPYTERHUB_VERSION: latest + restart: always + networks: + - clowder2 + volumes: + # The JupyterHub configuration file + - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro + # Bind Docker socket on the hostso we can connect to the daemon from + # within the container + - /var/run/docker.sock:/var/run/docker.sock:rw + # Bind Docker volume on host for JupyterHub database and cookie secrets + - jupyterhub-data:/data + ports: + - "8765:8000" + env_file: + - jupyterhub/.env + command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py + + depends_on: + - keycloak + +volumes: + jupyterhub-data: diff --git a/docker-dev.sh b/docker-dev.sh index 44f50afab..2c3fd1a00 100755 --- a/docker-dev.sh +++ b/docker-dev.sh @@ -7,3 +7,12 @@ if [ "$1" = "down" ] then docker-compose -f docker-compose.dev.yml -p clowder2-dev down fi +if [ "$1" = "jupyter" ] & [ "$2" = "up" ] +then + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev up -d --build +fi + +if [ "$1" = "jupyter" ] & [ "$2" = "down" ] +then + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev down +fi \ No newline at end of file diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 103af8f6b..4bd1dc9a8 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -23,6 +23,7 @@ import { RootState } from "../types/data"; import { AddBox, Explore } from "@material-ui/icons"; import HistoryIcon from "@mui/icons-material/History"; import GroupIcon from "@mui/icons-material/Group"; +import MenuBookIcon from '@mui/icons-material/MenuBook'; import Gravatar from "react-gravatar"; import PersonIcon from "@mui/icons-material/Person"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; @@ -435,6 +436,19 @@ export default function PersistentDrawerLeft(props) { + + + + + {/*TODO: Need to make link dynamic */} + + + + + + + diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example new file mode 100644 index 000000000..8092bde32 --- /dev/null +++ b/jupyterhub/.env-example @@ -0,0 +1,12 @@ +# Example configuration file for Clowder JupyterHub +KEYCLOAK_HOSTNAME="keycloak:8080/keycloak" +# Development mode use the following line instead +#KEYCLOAK_HOSTNAME="host.docker.internal:8080/keycloak" +KEYCLOAK_AUDIENCE="clowder" +KEYCLOAK_REALM="clowder" +JUPYTERHUB_ADMIN="admin" +#Change network name to the one created by docker-compose +DOCKER_NETWORK_NAME="clowder2" +DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" +DOCKER_NOTEBOOK_DIR="/home/jovyan/work" +JUPYTERHUB_CRYPT_KEY="" \ No newline at end of file diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub new file mode 100644 index 000000000..9956c7ab8 --- /dev/null +++ b/jupyterhub/Dockerfile.jupyterhub @@ -0,0 +1,14 @@ +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION + +# Install dockerspawner, +# hadolint ignore=DL3013 +RUN python3 -m pip install --no-cache-dir \ + dockerspawner + +# Install custom authenticator +WORKDIR /tmp/authenticator/ +COPY authenticator /tmp/authenticator/ +RUN pip3 install /tmp/authenticator + +CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] \ No newline at end of file diff --git a/jupyterhub/authenticator/customauthenticator/__init__.py b/jupyterhub/authenticator/customauthenticator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py new file mode 100644 index 000000000..1d9b8b101 --- /dev/null +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -0,0 +1,216 @@ +import os +import urllib.parse +import json + +from jose import jwt +from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +from jupyterhub.auth import Authenticator +from jupyterhub.handlers import LoginHandler, LogoutHandler +from traitlets import Unicode +from tornado import web +import requests + + +class CustomTokenAuthenticator(Authenticator): + """ + Accept the authenticated Access Token from cookie. + """ + auth_cookie_header = Unicode( + os.environ.get('AUTH_COOKIE_HEADER', ''), + config=True, + help="the cookie header we put in browser to retrieve token", + ) + + auth_username_key = Unicode( + os.environ.get('AUTH_USERNAME_KEY', ''), + config=True, + help="the key to retreive username from the json", + ) + + + + landing_page_login_url = Unicode( + os.environ.get('LANDING_PAGE_LOGIN_URL', ''), + config=True, + help="the landing page login entry", + ) + + keycloak_url = Unicode( + os.environ.get('KEYCLOAK_URL', ''), + config=True, + help="the URL where keycloak is installed", + ) + + keycloak_audience = Unicode( + os.environ.get('KEYCLOAK_AUDIENCE', ''), + config=True, + help="the audience for keycloak to check", + ) + + keycloak_pem_key = Unicode( + os.environ.get('KEYCLOAK_PEM_KEY', ''), + config=True, + help="the RSA pem key with proper header and footer (deprecated)", + ) + + space_service_url = Unicode( + os.environ.get('SPACE_SERVICE_URL', ''), + config=True, + help="the internal space service url" + ) + + quotas = None + + def get_handlers(self, app): + return [ + (r'/', LoginHandler), + (r'/user', LoginHandler), + (r'/lab', LoginHandler), + (r'/login', LoginHandler), + (r'/logout', CustomTokenLogoutHandler), + ] + + def get_keycloak_pem(self): + if not self.keycloak_url: + raise web.HTTPError(500, log_message="JupyterHub is not correctly configured.") + + # fetch the key + response = urllib.request.urlopen(self.keycloak_url) + if response.code >= 200 or response <= 299: + encoding = response.info().get_content_charset('utf-8') + result = json.loads(response.read().decode(encoding)) + self.keycloak_pem_key = f"-----BEGIN PUBLIC KEY-----\n" \ + f"{result['public_key']}\n" \ + f"-----END PUBLIC KEY-----" + else: + raise web.HTTPError(500, log_message="Could not get key from keycloak.") + + def check_jwt_token(self, access_token): + # make sure we have the pem cert + if not self.keycloak_pem_key: + self.get_keycloak_pem() + + # make sure audience is set + if not self.keycloak_audience: + raise web.HTTPError(403, log_message="JupyterHub is not correctly configured.") + + # no token in the cookie + if not access_token: + raise web.HTTPError(401, log_message="Please login to access Clowder.") + + # make sure it is a valid token + if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': + raise web.HTTPError(403, log_message="Token format not valid, it has to be bearer xxxx!") + + # decode jwt token instead of sending it to userinfo endpoint: + access_token = access_token.split(" ")[1] + public_key = self.keycloak_pem_key + audience = self.keycloak_audience + try: + resp_json = jwt.decode(access_token, public_key, audience=audience) + except ExpiredSignatureError: + raise web.HTTPError(403, log_message='JWT Expired Signature Error: token signature has expired') + except JWTClaimsError: + raise web.HTTPError(403, log_message='JWT Claims Error: token signature is invalid') + except JWTError: + raise web.HTTPError(403, log_message='JWT Error: token signature is invalid') + except Exception as e: + raise web.HTTPError(403, log_message="Not a valid jwt token!") + + # make sure we know username + if self.auth_username_key not in resp_json.keys(): + raise web.HTTPError(500, log_message=f"Required field {self.auth_username_key} does not exist in jwt token") + username = resp_json[self.auth_username_key] + + # # make sure there is a user id + # if self.auth_uid_number_key not in resp_json.keys(): + # raise web.HTTPError(500, log_message=f"Required field {self.auth_uid_number_key} does not exist in jwt token") + # uid = resp_json[self.auth_uid_number_key] + # + # get the groups/roles for the user + if "roles" in resp_json: + user_roles = resp_json.get("roles", []) + elif "realm_access" in resp_json: + user_roles = resp_json["realm_access"].get("roles", []) + else: + user_roles = [] + + + self.log.info(f"username={username}") + return { + 'name': username, + 'auth_state': { + 'roles': user_roles, + }, + } + + async def authenticate(self, handler, data): + self.log.info("Authenticate") + try: + access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) + if not access_token: + raise web.HTTPError(401, log_message="Please login to access Clowder.") + + # check token and authorization + user = self.check_jwt_token(access_token) + return user + except web.HTTPError as e: + if e.log_message: + error_msg = urllib.parse.quote(e.log_message.encode('utf-8')) + else: + error_msg = urllib.parse.quote(f"Error {e}".encode('utf-8')) + ". Please login to access Clowder." + handler.redirect(f"{self.landing_page_login_url}?error={error_msg}") + + + # async def pre_spawn_start(self, user, spawner): + # auth_state = await user.get_auth_state() + # if not auth_state: + # self.log.error("No auth state") + # return + # + # spawner.environment['NB_USER'] = user.name + # spawner.environment['NB_UID'] = str(auth_state['uid']) + # + # quota = self.find_quota(user, auth_state) + # if "cpu" in quota: + # spawner.cpu_guarantee = quota["cpu"][0] + # spawner.cpu_limit = quota["cpu"][1] + # else: + # spawner.cpu_guarantee = 1 + # spawner.cpu_limit = 2 + # if "mem" in quota: + # spawner.mem_guarantee = f"{quota['mem'][0]}G" + # spawner.mem_limit = f"{quota['mem'][1]}G" + # else: + # spawner.mem_guarantee = "2G" + # spawner.mem_limit = "4G" + +# +# # This is called from the jupyterlab so there is no cookies that this depends on +# async def refresh_user(self, user, handler): +# self.log.info("Refresh User") +# try: +# access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) +# # if no token present +# if not access_token: +# return False +# +# # if token present, check token and authorization +# if self.check_jwt_token(access_token): +# True +# return False +# except: +# self.log.exception("Error in refresh user") +# return False + + +class CustomTokenLogoutHandler(LogoutHandler): + + async def handle_logout(self): + # remove clowder token on logout + self.log.info("Remove clowder token on logout") + error_msg = "You have logged out of Clowder system from Clowder . Please login again if you want to use " \ + "Clowder components." + self.set_cookie(self.authenticator.auth_cookie_header, "") + self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") + diff --git a/jupyterhub/authenticator/setup.py b/jupyterhub/authenticator/setup.py new file mode 100644 index 000000000..7e37c3d7a --- /dev/null +++ b/jupyterhub/authenticator/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +setup( + name='customauthenticator', + version='0.8.0', + description='Custom Authenticator for JupyterHub', + author='cwang138', + author_email='cwang138@illinois.edu', + license='MPL 2.0', + packages=['customauthenticator'], + install_requires=[ + 'jupyterhub', + 'pyjwt', + 'requests', + 'python-jose' + ] +) diff --git a/jupyterhub/authenticator/test_jwt.py b/jupyterhub/authenticator/test_jwt.py new file mode 100644 index 000000000..12df103db --- /dev/null +++ b/jupyterhub/authenticator/test_jwt.py @@ -0,0 +1,40 @@ +from jose import jwt +from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +import os +import urllib.request +import json + +response = urllib.request.urlopen("") + +if response.code >= 200 or response <= 299: + encoding = response.info().get_content_charset('utf-8') + result = json.loads(response.read().decode(encoding)) + public_key = f"-----BEGIN PUBLIC KEY-----\n" \ + f"{result['public_key']}\n" \ + f"-----END PUBLIC KEY-----" +else: + print("Could not get key from keycloak.") + + +access_token ="" + +# make sure it is a valid token +if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': + print("Token format not valid, it has to be bearer xxxx!") + +# decode jwt token instead of sending it to userinfo endpoint: +access_token = access_token.split(" ")[1] + +try: + decoded = jwt.decode(access_token, public_key, audience="clowder") + print(decoded) + +except ExpiredSignatureError: + print('JWT Expired Signature Error: token signature has expired') +except JWTClaimsError: + print('JWT Claims Error: token signature is invalid') +except JWTError: + print('JWT Error: token signature is invalid') +except Exception as e: + print("Not a valid jwt token!") + diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py new file mode 100644 index 000000000..1e704e316 --- /dev/null +++ b/jupyterhub/jupyterhub_config.py @@ -0,0 +1,74 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Configuration file for JupyterHub +import os + +c = get_config() # noqa: F821 + +# We rely on environment variables to configure JupyterHub so that we +# avoid having to rebuild the JupyterHub container every time we change a +# configuration parameter. + +# Spawn single-user servers as Docker containers +c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" + +# Spawn containers from this image +c.DockerSpawner.image = os.environ["DOCKER_NOTEBOOK_IMAGE"] + +# Connect containers to this Docker network +network_name = os.environ["DOCKER_NETWORK_NAME"] +c.DockerSpawner.use_internal_ip = True +c.DockerSpawner.network_name = network_name + +# Explicitly set notebook directory because we'll be mounting a volume to it. +# Most `jupyter/docker-stacks` *-notebook images run the Notebook server as +# user `jovyan`, and set the notebook directory to `/home/jovyan/work`. +# We follow the same convention. +notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work") +c.DockerSpawner.notebook_dir = notebook_dir + +# Mount the real user's Docker volume on the host to the notebook user's +# notebook directory in the container +c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} + +# Remove containers once they are stopped +c.DockerSpawner.remove = True + +# For debugging arguments passed to spawned containers +c.DockerSpawner.debug = True + +# User containers will access hub by container name on the Docker network +c.JupyterHub.hub_ip = "jupyterhub" +c.JupyterHub.hub_port = 8080 + +# Persist hub data on volume mounted inside container +# c.JupyterHub.cookie_secret_file = "/data/jupyterhub_cookie_secret" +c.JupyterHub.db_url = "sqlite:////data/jupyterhub.sqlite" + +# # Authenticate users with Native Authenticator +# c.JupyterHub.authenticator_class = "nativeauthenticator.NativeAuthenticator" +# +# # Allow anyone to sign-up without approval +# c.NativeAuthenticator.open_signup = True + +# Authenticate with Custom Token Authenticator +from customauthenticator.custom import CustomTokenAuthenticator +c.Spawner.cmd = ['start.sh', 'jupyterhub-singleuser', '--allow-root'] +c.KubeSpawner.args = ['--allow-root'] +c.JupyterHub.authenticator_class = CustomTokenAuthenticator +# TODO:Change this keycloak_url as required +c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % (os.getenv('KEYCLOAK_HOSTNAME'), os.getenv('KEYCLOAK_REALM')) +c.CustomTokenAuthenticator.auth_cookie_header= "Authorization" +c.CustomTokenAuthenticator.auth_username_key= "preferred_username" +c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" +c.CustomTokenAuthenticator.enable_auth_state = True +c.CustomTokenAuthenticator.auto_login = True +c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv('KEYCLOAK_HOSTNAME') + +c.JupyterHub.cookie_secret = os.getenv('JUPYTERHUB_CRYPT_KEY') + +# Allowed admins +admin = os.environ.get("JUPYTERHUB_ADMIN") +if admin: + c.Authenticator.admin_users = [admin] From aaa505a97e1e6032db658815d910266f41c73914 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Wed, 22 May 2024 11:45:55 -0500 Subject: [PATCH 12/24] Linting --- docker-dev.sh | 6 +- frontend/src/components/Layout.tsx | 10 +- jupyterhub/.env-example | 2 +- jupyterhub/Dockerfile.jupyterhub | 2 +- .../customauthenticator/custom.py | 113 +++++++++++------- jupyterhub/authenticator/setup.py | 21 ++-- jupyterhub/authenticator/test_jwt.py | 31 ++--- jupyterhub/jupyterhub_config.py | 22 ++-- 8 files changed, 117 insertions(+), 90 deletions(-) diff --git a/docker-dev.sh b/docker-dev.sh index 2c3fd1a00..6e08082b4 100755 --- a/docker-dev.sh +++ b/docker-dev.sh @@ -7,12 +7,12 @@ if [ "$1" = "down" ] then docker-compose -f docker-compose.dev.yml -p clowder2-dev down fi -if [ "$1" = "jupyter" ] & [ "$2" = "up" ] +if [ "$1" = "jupyter" ] && [ "$2" = "up" ] then docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev up -d --build fi -if [ "$1" = "jupyter" ] & [ "$2" = "down" ] +if [ "$1" = "jupyter" ] && [ "$2" = "down" ] then docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev down -fi \ No newline at end of file +fi diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 4bd1dc9a8..893dfd7b7 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -23,7 +23,7 @@ import { RootState } from "../types/data"; import { AddBox, Explore } from "@material-ui/icons"; import HistoryIcon from "@mui/icons-material/History"; import GroupIcon from "@mui/icons-material/Group"; -import MenuBookIcon from '@mui/icons-material/MenuBook'; +import MenuBookIcon from "@mui/icons-material/MenuBook"; import Gravatar from "react-gravatar"; import PersonIcon from "@mui/icons-material/Person"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; @@ -443,8 +443,12 @@ export default function PersistentDrawerLeft(props) { {/*TODO: Need to make link dynamic */} - + diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example index 8092bde32..0df50c06b 100644 --- a/jupyterhub/.env-example +++ b/jupyterhub/.env-example @@ -9,4 +9,4 @@ JUPYTERHUB_ADMIN="admin" DOCKER_NETWORK_NAME="clowder2" DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" DOCKER_NOTEBOOK_DIR="/home/jovyan/work" -JUPYTERHUB_CRYPT_KEY="" \ No newline at end of file +JUPYTERHUB_CRYPT_KEY="" diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index 9956c7ab8..349782ad7 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -11,4 +11,4 @@ WORKDIR /tmp/authenticator/ COPY authenticator /tmp/authenticator/ RUN pip3 install /tmp/authenticator -CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] \ No newline at end of file +CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 1d9b8b101..6f555a7bb 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -1,87 +1,90 @@ +import json import os import urllib.parse -import json from jose import jwt -from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError +from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError +from tornado import web +from traitlets import Unicode + from jupyterhub.auth import Authenticator from jupyterhub.handlers import LoginHandler, LogoutHandler -from traitlets import Unicode -from tornado import web -import requests class CustomTokenAuthenticator(Authenticator): """ - Accept the authenticated Access Token from cookie. + Accept the authenticated Access Token from cookie. """ + auth_cookie_header = Unicode( - os.environ.get('AUTH_COOKIE_HEADER', ''), + os.environ.get("AUTH_COOKIE_HEADER", ""), config=True, help="the cookie header we put in browser to retrieve token", ) auth_username_key = Unicode( - os.environ.get('AUTH_USERNAME_KEY', ''), + os.environ.get("AUTH_USERNAME_KEY", ""), config=True, help="the key to retreive username from the json", ) - - landing_page_login_url = Unicode( - os.environ.get('LANDING_PAGE_LOGIN_URL', ''), + os.environ.get("LANDING_PAGE_LOGIN_URL", ""), config=True, help="the landing page login entry", ) keycloak_url = Unicode( - os.environ.get('KEYCLOAK_URL', ''), + os.environ.get("KEYCLOAK_URL", ""), config=True, help="the URL where keycloak is installed", ) keycloak_audience = Unicode( - os.environ.get('KEYCLOAK_AUDIENCE', ''), + os.environ.get("KEYCLOAK_AUDIENCE", ""), config=True, help="the audience for keycloak to check", ) keycloak_pem_key = Unicode( - os.environ.get('KEYCLOAK_PEM_KEY', ''), + os.environ.get("KEYCLOAK_PEM_KEY", ""), config=True, help="the RSA pem key with proper header and footer (deprecated)", ) space_service_url = Unicode( - os.environ.get('SPACE_SERVICE_URL', ''), + os.environ.get("SPACE_SERVICE_URL", ""), config=True, - help="the internal space service url" + help="the internal space service url", ) quotas = None def get_handlers(self, app): return [ - (r'/', LoginHandler), - (r'/user', LoginHandler), - (r'/lab', LoginHandler), - (r'/login', LoginHandler), - (r'/logout', CustomTokenLogoutHandler), + (r"/", LoginHandler), + (r"/user", LoginHandler), + (r"/lab", LoginHandler), + (r"/login", LoginHandler), + (r"/logout", CustomTokenLogoutHandler), ] def get_keycloak_pem(self): if not self.keycloak_url: - raise web.HTTPError(500, log_message="JupyterHub is not correctly configured.") + raise web.HTTPError( + 500, log_message="JupyterHub is not correctly configured." + ) # fetch the key response = urllib.request.urlopen(self.keycloak_url) if response.code >= 200 or response <= 299: - encoding = response.info().get_content_charset('utf-8') + encoding = response.info().get_content_charset("utf-8") result = json.loads(response.read().decode(encoding)) - self.keycloak_pem_key = f"-----BEGIN PUBLIC KEY-----\n" \ - f"{result['public_key']}\n" \ - f"-----END PUBLIC KEY-----" + self.keycloak_pem_key = ( + f"-----BEGIN PUBLIC KEY-----\n" + f"{result['public_key']}\n" + f"-----END PUBLIC KEY-----" + ) else: raise web.HTTPError(500, log_message="Could not get key from keycloak.") @@ -92,15 +95,19 @@ def check_jwt_token(self, access_token): # make sure audience is set if not self.keycloak_audience: - raise web.HTTPError(403, log_message="JupyterHub is not correctly configured.") + raise web.HTTPError( + 403, log_message="JupyterHub is not correctly configured." + ) # no token in the cookie if not access_token: raise web.HTTPError(401, log_message="Please login to access Clowder.") # make sure it is a valid token - if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': - raise web.HTTPError(403, log_message="Token format not valid, it has to be bearer xxxx!") + if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != "Bearer": + raise web.HTTPError( + 403, log_message="Token format not valid, it has to be bearer xxxx!" + ) # decode jwt token instead of sending it to userinfo endpoint: access_token = access_token.split(" ")[1] @@ -109,17 +116,27 @@ def check_jwt_token(self, access_token): try: resp_json = jwt.decode(access_token, public_key, audience=audience) except ExpiredSignatureError: - raise web.HTTPError(403, log_message='JWT Expired Signature Error: token signature has expired') + raise web.HTTPError( + 403, + log_message="JWT Expired Signature Error: token signature has expired", + ) except JWTClaimsError: - raise web.HTTPError(403, log_message='JWT Claims Error: token signature is invalid') + raise web.HTTPError( + 403, log_message="JWT Claims Error: token signature is invalid" + ) except JWTError: - raise web.HTTPError(403, log_message='JWT Error: token signature is invalid') - except Exception as e: + raise web.HTTPError( + 403, log_message="JWT Error: token signature is invalid" + ) + except Exception: raise web.HTTPError(403, log_message="Not a valid jwt token!") # make sure we know username if self.auth_username_key not in resp_json.keys(): - raise web.HTTPError(500, log_message=f"Required field {self.auth_username_key} does not exist in jwt token") + raise web.HTTPError( + 500, + log_message=f"Required field {self.auth_username_key} does not exist in jwt token", + ) username = resp_json[self.auth_username_key] # # make sure there is a user id @@ -135,19 +152,20 @@ def check_jwt_token(self, access_token): else: user_roles = [] - self.log.info(f"username={username}") return { - 'name': username, - 'auth_state': { - 'roles': user_roles, + "name": username, + "auth_state": { + "roles": user_roles, }, } async def authenticate(self, handler, data): self.log.info("Authenticate") try: - access_token = urllib.parse.unquote(handler.get_cookie(self.auth_cookie_header, "")) + access_token = urllib.parse.unquote( + handler.get_cookie(self.auth_cookie_header, "") + ) if not access_token: raise web.HTTPError(401, log_message="Please login to access Clowder.") @@ -156,12 +174,14 @@ async def authenticate(self, handler, data): return user except web.HTTPError as e: if e.log_message: - error_msg = urllib.parse.quote(e.log_message.encode('utf-8')) + error_msg = urllib.parse.quote(e.log_message.encode("utf-8")) else: - error_msg = urllib.parse.quote(f"Error {e}".encode('utf-8')) + ". Please login to access Clowder." + error_msg = ( + urllib.parse.quote(f"Error {e}".encode("utf-8")) + + ". Please login to access Clowder." + ) handler.redirect(f"{self.landing_page_login_url}?error={error_msg}") - # async def pre_spawn_start(self, user, spawner): # auth_state = await user.get_auth_state() # if not auth_state: @@ -185,6 +205,7 @@ async def authenticate(self, handler, data): # spawner.mem_guarantee = "2G" # spawner.mem_limit = "4G" + # # # This is called from the jupyterlab so there is no cookies that this depends on # async def refresh_user(self, user, handler): @@ -205,12 +226,12 @@ async def authenticate(self, handler, data): class CustomTokenLogoutHandler(LogoutHandler): - async def handle_logout(self): # remove clowder token on logout self.log.info("Remove clowder token on logout") - error_msg = "You have logged out of Clowder system from Clowder . Please login again if you want to use " \ - "Clowder components." + error_msg = ( + "You have logged out of Clowder system from Clowder . Please login again if you want to use " + "Clowder components." + ) self.set_cookie(self.authenticator.auth_cookie_header, "") self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") - diff --git a/jupyterhub/authenticator/setup.py b/jupyterhub/authenticator/setup.py index 7e37c3d7a..0314f36d2 100644 --- a/jupyterhub/authenticator/setup.py +++ b/jupyterhub/authenticator/setup.py @@ -1,17 +1,12 @@ from setuptools import setup setup( - name='customauthenticator', - version='0.8.0', - description='Custom Authenticator for JupyterHub', - author='cwang138', - author_email='cwang138@illinois.edu', - license='MPL 2.0', - packages=['customauthenticator'], - install_requires=[ - 'jupyterhub', - 'pyjwt', - 'requests', - 'python-jose' - ] + name="customauthenticator", + version="0.8.0", + description="Custom Authenticator for JupyterHub", + author="cwang138", + author_email="cwang138@illinois.edu", + license="MPL 2.0", + packages=["customauthenticator"], + install_requires=["jupyterhub", "pyjwt", "requests", "python-jose"], ) diff --git a/jupyterhub/authenticator/test_jwt.py b/jupyterhub/authenticator/test_jwt.py index 12df103db..e86f6bc96 100644 --- a/jupyterhub/authenticator/test_jwt.py +++ b/jupyterhub/authenticator/test_jwt.py @@ -1,25 +1,27 @@ -from jose import jwt -from jose.exceptions import JWTError, ExpiredSignatureError, JWTClaimsError -import os -import urllib.request import json +import urllib.request + +from jose import jwt +from jose.exceptions import ExpiredSignatureError, JWTClaimsError, JWTError response = urllib.request.urlopen("") if response.code >= 200 or response <= 299: - encoding = response.info().get_content_charset('utf-8') + encoding = response.info().get_content_charset("utf-8") result = json.loads(response.read().decode(encoding)) - public_key = f"-----BEGIN PUBLIC KEY-----\n" \ - f"{result['public_key']}\n" \ - f"-----END PUBLIC KEY-----" + public_key = ( + f"-----BEGIN PUBLIC KEY-----\n" + f"{result['public_key']}\n" + f"-----END PUBLIC KEY-----" + ) else: print("Could not get key from keycloak.") -access_token ="" +access_token = "" # make sure it is a valid token -if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != 'Bearer': +if len(access_token.split(" ")) != 2 or access_token.split(" ")[0] != "Bearer": print("Token format not valid, it has to be bearer xxxx!") # decode jwt token instead of sending it to userinfo endpoint: @@ -30,11 +32,10 @@ print(decoded) except ExpiredSignatureError: - print('JWT Expired Signature Error: token signature has expired') + print("JWT Expired Signature Error: token signature has expired") except JWTClaimsError: - print('JWT Claims Error: token signature is invalid') + print("JWT Claims Error: token signature is invalid") except JWTError: - print('JWT Error: token signature is invalid') -except Exception as e: + print("JWT Error: token signature is invalid") +except Exception: print("Not a valid jwt token!") - diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1e704e316..11ce4c850 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -4,6 +4,8 @@ # Configuration file for JupyterHub import os +from customauthenticator.custom import CustomTokenAuthenticator + c = get_config() # noqa: F821 # We rely on environment variables to configure JupyterHub so that we @@ -53,20 +55,24 @@ # c.NativeAuthenticator.open_signup = True # Authenticate with Custom Token Authenticator -from customauthenticator.custom import CustomTokenAuthenticator -c.Spawner.cmd = ['start.sh', 'jupyterhub-singleuser', '--allow-root'] -c.KubeSpawner.args = ['--allow-root'] +c.Spawner.cmd = ["start.sh", "jupyterhub-singleuser", "--allow-root"] +c.KubeSpawner.args = ["--allow-root"] c.JupyterHub.authenticator_class = CustomTokenAuthenticator # TODO:Change this keycloak_url as required -c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % (os.getenv('KEYCLOAK_HOSTNAME'), os.getenv('KEYCLOAK_REALM')) -c.CustomTokenAuthenticator.auth_cookie_header= "Authorization" -c.CustomTokenAuthenticator.auth_username_key= "preferred_username" +c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), +) +c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" +c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True -c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv('KEYCLOAK_HOSTNAME') +c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( + "KEYCLOAK_HOSTNAME" +) -c.JupyterHub.cookie_secret = os.getenv('JUPYTERHUB_CRYPT_KEY') +c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") # Allowed admins admin = os.environ.get("JUPYTERHUB_ADMIN") From 85ec5ee8a63142af14156d31b75bc09fd6a6220a Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Thu, 23 May 2024 12:27:02 -0500 Subject: [PATCH 13/24] Handling logout --- docker-compose.dev.yml | 4 +++ jupyterhub/.env-example | 2 ++ .../customauthenticator/custom.py | 10 ++++-- jupyterhub/jupyterhub_config.py | 32 +++++++++++++++---- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a6ff21566..61dcc8bf0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -67,6 +67,8 @@ services: postgres: image: postgres + networks: + - clowder2 volumes: - postgres_data:/var/lib/postgresql/data environment: @@ -79,6 +81,8 @@ services: volumes: - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro + networks: + - clowder2 command: - start-dev - --http-relative-path /keycloak diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example index 0df50c06b..dcb467521 100644 --- a/jupyterhub/.env-example +++ b/jupyterhub/.env-example @@ -10,3 +10,5 @@ DOCKER_NETWORK_NAME="clowder2" DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" DOCKER_NOTEBOOK_DIR="/home/jovyan/work" JUPYTERHUB_CRYPT_KEY="" +CLOWDER_URL="localhost:3000" +PROD_DEPLOYMENT="false" diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 6f555a7bb..435def5c1 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -34,6 +34,12 @@ class CustomTokenAuthenticator(Authenticator): help="the landing page login entry", ) + landing_page_logout_url = Unicode( + os.environ.get("LANDING_PAGE_LOGOUT", ""), + config=True, + help="the landing page logout entry", + ) + keycloak_url = Unicode( os.environ.get("KEYCLOAK_URL", ""), config=True, @@ -229,9 +235,9 @@ class CustomTokenLogoutHandler(LogoutHandler): async def handle_logout(self): # remove clowder token on logout self.log.info("Remove clowder token on logout") - error_msg = ( + self.log.info( "You have logged out of Clowder system from Clowder . Please login again if you want to use " "Clowder components." ) self.set_cookie(self.authenticator.auth_cookie_header, "") - self.redirect(f"{self.authenticator.landing_page_login_url}?error={error_msg}") + self.redirect(f"{self.authenticator.landing_page_logout_url}") diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 11ce4c850..136f80962 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -59,18 +59,36 @@ c.KubeSpawner.args = ["--allow-root"] c.JupyterHub.authenticator_class = CustomTokenAuthenticator # TODO:Change this keycloak_url as required -c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( - os.getenv("KEYCLOAK_HOSTNAME"), - os.getenv("KEYCLOAK_REALM"), -) + c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True -c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( - "KEYCLOAK_HOSTNAME" -) + +if os.getenv("PROD_DEPLOYMENT") == "true": + c.CustomTokenAuthenticator.keycloak_url = "https://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "https://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) + +else: + c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "http://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "http://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") From f547bc7feff3fa25d8ea1ae0113a3dc585c04844 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 11:39:55 -0500 Subject: [PATCH 14/24] Updating docker-compose to keep jupyterhub version static --- docker-compose.jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml index 333ca5555..6179f8937 100644 --- a/docker-compose.jupyter.yml +++ b/docker-compose.jupyter.yml @@ -5,7 +5,7 @@ services: context: jupyterhub dockerfile: Dockerfile.jupyterhub args: - JUPYTERHUB_VERSION: latest + JUPYTERHUB_VERSION: 4 restart: always networks: - clowder2 From 4c512650ab7c1e5de220928f77564ffc1ad5c6be Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 13:04:49 -0500 Subject: [PATCH 15/24] Incorporating feedback --- docs/docs/devs/getstarted.md | 2 ++ frontend/src/app.config.ts | 5 +++++ frontend/src/components/Layout.tsx | 4 +++- .../customauthenticator/custom.py | 20 +------------------ 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/docs/devs/getstarted.md b/docs/docs/devs/getstarted.md index bbf8f1633..6eb3b72aa 100644 --- a/docs/docs/devs/getstarted.md +++ b/docs/docs/devs/getstarted.md @@ -47,6 +47,8 @@ section below). - Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers, provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`. - Running `./docker-dev.sh down` brings down the required services. +- If you want to run the jupyterhub, you can run `./docker-dev.sh jupyter up`. The jupyterhub will be available at + `http://localhost:8765`. You can bring it down using `./docker-dev.sh jupyter down`. **Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`. diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index 83a1e77a0..3c62e8839 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -9,6 +9,7 @@ interface Config { hostname: string; apikey: string; GHIssueBaseURL: string; + jupyterHubURL: string; KeycloakBaseURL: string; KeycloakLogin: string; KeycloakLogout: string; @@ -64,6 +65,10 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`; config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`; config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`; +// jupterhub +const localJupyterhubURL: string = "http://localhost:8765"; +config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL; + // refresh token time interval config["refreshTokenInterval"] = 1000 * 60; // 1 minute // updated extractor logs diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 893dfd7b7..cab52319d 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -42,6 +42,8 @@ import AdminPanelSettingsIcon from "@mui/icons-material/AdminPanelSettings"; import { Footer } from "./navigation/Footer"; import BuildIcon from "@mui/icons-material/Build"; +import config from "../app.config"; + const drawerWidth = 240; const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{ @@ -445,7 +447,7 @@ export default function PersistentDrawerLeft(props) { diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py index 435def5c1..7dfb05962 100644 --- a/jupyterhub/authenticator/customauthenticator/custom.py +++ b/jupyterhub/authenticator/customauthenticator/custom.py @@ -145,26 +145,8 @@ def check_jwt_token(self, access_token): ) username = resp_json[self.auth_username_key] - # # make sure there is a user id - # if self.auth_uid_number_key not in resp_json.keys(): - # raise web.HTTPError(500, log_message=f"Required field {self.auth_uid_number_key} does not exist in jwt token") - # uid = resp_json[self.auth_uid_number_key] - # - # get the groups/roles for the user - if "roles" in resp_json: - user_roles = resp_json.get("roles", []) - elif "realm_access" in resp_json: - user_roles = resp_json["realm_access"].get("roles", []) - else: - user_roles = [] - self.log.info(f"username={username}") - return { - "name": username, - "auth_state": { - "roles": user_roles, - }, - } + return {"name": username} async def authenticate(self, handler, data): self.log.info("Authenticate") From 1169759cbe6321b912ba53d2d7ba4250c799242a Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 16:11:35 -0500 Subject: [PATCH 16/24] Fix conflict --- frontend/src/components/Layout.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index cab52319d..70c1391d2 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -438,10 +438,10 @@ export default function PersistentDrawerLeft(props) { - - - - + + + + {/*TODO: Need to make link dynamic */} From 5ef860f12880c1946a0554bb7ed66759a2239f76 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 7 Jun 2024 16:37:54 -0500 Subject: [PATCH 17/24] Fix config --- frontend/src/app.config.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index 7e5664c3d..3c62e8839 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -69,10 +69,6 @@ config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`; const localJupyterhubURL: string = "http://localhost:8765"; config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL; -// jupterhub -const localJupyterhubURL: string = "http://localhost:8765"; -config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL; - // refresh token time interval config["refreshTokenInterval"] = 1000 * 60; // 1 minute // updated extractor logs From d269f614952693fdd19a68fe388cb97c2f6fba97 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 21 Jun 2024 12:06:54 -0500 Subject: [PATCH 18/24] Creating a traefik setup for prod docker-compose --- docker-compose.jupyter-dev.yml | 30 ++++++++++++++++++++++++++++++ docker-compose.jupyter.yml | 11 +++++++---- docker-dev.sh | 4 ++-- frontend/src/app.config.ts | 2 +- frontend/src/components/Layout.tsx | 2 +- jupyterhub/jupyterhub_config.py | 9 +++++++++ 6 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 docker-compose.jupyter-dev.yml diff --git a/docker-compose.jupyter-dev.yml b/docker-compose.jupyter-dev.yml new file mode 100644 index 000000000..6179f8937 --- /dev/null +++ b/docker-compose.jupyter-dev.yml @@ -0,0 +1,30 @@ +version: '3' +services: + jupyterhub: + build: + context: jupyterhub + dockerfile: Dockerfile.jupyterhub + args: + JUPYTERHUB_VERSION: 4 + restart: always + networks: + - clowder2 + volumes: + # The JupyterHub configuration file + - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro + # Bind Docker socket on the hostso we can connect to the daemon from + # within the container + - /var/run/docker.sock:/var/run/docker.sock:rw + # Bind Docker volume on host for JupyterHub database and cookie secrets + - jupyterhub-data:/data + ports: + - "8765:8000" + env_file: + - jupyterhub/.env + command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py + + depends_on: + - keycloak + +volumes: + jupyterhub-data: diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml index 6179f8937..b8f02659c 100644 --- a/docker-compose.jupyter.yml +++ b/docker-compose.jupyter.yml @@ -16,15 +16,18 @@ services: # within the container - /var/run/docker.sock:/var/run/docker.sock:rw # Bind Docker volume on host for JupyterHub database and cookie secrets - - jupyterhub-data:/data - ports: - - "8765:8000" + - jupyterhub_data:/data env_file: - jupyterhub/.env + labels: + - "traefik.enable=true" + - "traefik.http.routers.jupyterhub.rule=PathPrefix(`/jupyterhub`)" + - "traefik.http.services.jupyterhub.loadbalancer.server.port=8000" + command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py depends_on: - keycloak volumes: - jupyterhub-data: + jupyterhub_data: diff --git a/docker-dev.sh b/docker-dev.sh index 6e08082b4..eb775df60 100755 --- a/docker-dev.sh +++ b/docker-dev.sh @@ -9,10 +9,10 @@ then fi if [ "$1" = "jupyter" ] && [ "$2" = "up" ] then - docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev up -d --build + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev up -d --build fi if [ "$1" = "jupyter" ] && [ "$2" = "down" ] then - docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter.yml -p clowder2-dev down + docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev down fi diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts index 73cc04cda..fc33bdb6d 100644 --- a/frontend/src/app.config.ts +++ b/frontend/src/app.config.ts @@ -67,7 +67,7 @@ config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`; config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`; // jupterhub -const localJupyterhubURL: string = "http://localhost:8765"; +const localJupyterhubURL: string = `${config.hostname}/jupyterhub`; config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL; // refresh token time interval diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 9d54fd2fc..fea4e64b6 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -455,7 +455,7 @@ export default function PersistentDrawerLeft(props) { - + diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 136f80962..20242be38 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -12,6 +12,15 @@ # avoid having to rebuild the JupyterHub container every time we change a # configuration parameter. +# Base URL of the Hub +c.JupyterHub.base_url = "/jupyterhub" + + +# Important proxy settings to work with Traefik +c.JupyterHub.proxy_class = "jupyterhub.proxy.ConfigurableHTTPProxy" +c.ConfigurableHTTPProxy.command = ["configurable-http-proxy"] + + # Spawn single-user servers as Docker containers c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" From bf48b8cee5e190b704039c81921b7daf89d9daad Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 21 Jun 2024 14:19:59 -0500 Subject: [PATCH 19/24] Make it work for dev set up --- .gitignore | 1 + docker-compose.jupyter-dev.yml | 4 +- frontend/package.json | 2 +- frontend/webpack.config.dev.js | 4 ++ jupyterhub/jupyterhub_config.dev.py | 98 +++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 jupyterhub/jupyterhub_config.dev.py diff --git a/.gitignore b/.gitignore index 986f39ae1..aa73ebb11 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,4 @@ secrets.yaml # faker official.csv fact.png +jupyterhub/.env-dev diff --git a/docker-compose.jupyter-dev.yml b/docker-compose.jupyter-dev.yml index 6179f8937..006a778b4 100644 --- a/docker-compose.jupyter-dev.yml +++ b/docker-compose.jupyter-dev.yml @@ -11,7 +11,7 @@ services: - clowder2 volumes: # The JupyterHub configuration file - - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro + - ./jupyterhub/jupyterhub_config.dev.py:/srv/jupyterhub/jupyterhub_config.py:ro # Bind Docker socket on the hostso we can connect to the daemon from # within the container - /var/run/docker.sock:/var/run/docker.sock:rw @@ -20,7 +20,7 @@ services: ports: - "8765:8000" env_file: - - jupyterhub/.env + - jupyterhub/.env-dev command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py depends_on: diff --git a/frontend/package.json b/frontend/package.json index 0c9f543b4..2681aa035 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,7 +12,7 @@ "start-message": "babel-node tools/startMessage.js", "prestart": "npm-run-all --parallel start-message", "start": "npm-run-all --parallel open:src", - "start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && npm run start", + "start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && export JUPYTERHUB_URL=http://localhost:8765 && npm run start", "open:src": "babel-node tools/srcServer.js", "open:dist": "babel-node tools/distServer.js", "lint:watch": "npm run lint --watch", diff --git a/frontend/webpack.config.dev.js b/frontend/webpack.config.dev.js index 688ca512b..6511e871f 100644 --- a/frontend/webpack.config.dev.js +++ b/frontend/webpack.config.dev.js @@ -8,6 +8,9 @@ import ESLintPlugin from "eslint-webpack-plugin"; console.log( `the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}` ); +console.log( + `the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}` +) export default { mode: "development", @@ -40,6 +43,7 @@ export default { CLOWDER_REMOTE_HOSTNAME: JSON.stringify( process.env.CLOWDER_REMOTE_HOSTNAME ), + JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL), APIKEY: JSON.stringify(process.env.APIKEY), KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL), }, diff --git a/jupyterhub/jupyterhub_config.dev.py b/jupyterhub/jupyterhub_config.dev.py new file mode 100644 index 000000000..136f80962 --- /dev/null +++ b/jupyterhub/jupyterhub_config.dev.py @@ -0,0 +1,98 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Configuration file for JupyterHub +import os + +from customauthenticator.custom import CustomTokenAuthenticator + +c = get_config() # noqa: F821 + +# We rely on environment variables to configure JupyterHub so that we +# avoid having to rebuild the JupyterHub container every time we change a +# configuration parameter. + +# Spawn single-user servers as Docker containers +c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner" + +# Spawn containers from this image +c.DockerSpawner.image = os.environ["DOCKER_NOTEBOOK_IMAGE"] + +# Connect containers to this Docker network +network_name = os.environ["DOCKER_NETWORK_NAME"] +c.DockerSpawner.use_internal_ip = True +c.DockerSpawner.network_name = network_name + +# Explicitly set notebook directory because we'll be mounting a volume to it. +# Most `jupyter/docker-stacks` *-notebook images run the Notebook server as +# user `jovyan`, and set the notebook directory to `/home/jovyan/work`. +# We follow the same convention. +notebook_dir = os.environ.get("DOCKER_NOTEBOOK_DIR", "/home/jovyan/work") +c.DockerSpawner.notebook_dir = notebook_dir + +# Mount the real user's Docker volume on the host to the notebook user's +# notebook directory in the container +c.DockerSpawner.volumes = {"jupyterhub-user-{username}": notebook_dir} + +# Remove containers once they are stopped +c.DockerSpawner.remove = True + +# For debugging arguments passed to spawned containers +c.DockerSpawner.debug = True + +# User containers will access hub by container name on the Docker network +c.JupyterHub.hub_ip = "jupyterhub" +c.JupyterHub.hub_port = 8080 + +# Persist hub data on volume mounted inside container +# c.JupyterHub.cookie_secret_file = "/data/jupyterhub_cookie_secret" +c.JupyterHub.db_url = "sqlite:////data/jupyterhub.sqlite" + +# # Authenticate users with Native Authenticator +# c.JupyterHub.authenticator_class = "nativeauthenticator.NativeAuthenticator" +# +# # Allow anyone to sign-up without approval +# c.NativeAuthenticator.open_signup = True + +# Authenticate with Custom Token Authenticator +c.Spawner.cmd = ["start.sh", "jupyterhub-singleuser", "--allow-root"] +c.KubeSpawner.args = ["--allow-root"] +c.JupyterHub.authenticator_class = CustomTokenAuthenticator +# TODO:Change this keycloak_url as required + +c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" +c.CustomTokenAuthenticator.auth_username_key = "preferred_username" +c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" +c.CustomTokenAuthenticator.enable_auth_state = True +c.CustomTokenAuthenticator.auto_login = True + +if os.getenv("PROD_DEPLOYMENT") == "true": + c.CustomTokenAuthenticator.keycloak_url = "https://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "https://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "https://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) + +else: + c.CustomTokenAuthenticator.keycloak_url = "http://%s/realms/%s/" % ( + os.getenv("KEYCLOAK_HOSTNAME"), + os.getenv("KEYCLOAK_REALM"), + ) + c.CustomTokenAuthenticator.landing_page_login_url = "http://" + os.getenv( + "KEYCLOAK_HOSTNAME" + ) + c.CustomTokenAuthenticator.landing_page_logout_url = ( + "http://" + os.getenv("CLOWDER_URL") + "/auth/logout" + ) + +c.JupyterHub.cookie_secret = os.getenv("JUPYTERHUB_CRYPT_KEY") + +# Allowed admins +admin = os.environ.get("JUPYTERHUB_ADMIN") +if admin: + c.Authenticator.admin_users = [admin] From ce483a98963bc09e24432fb557fbdac325e36500 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 21 Jun 2024 14:29:44 -0500 Subject: [PATCH 20/24] Updating .env ecample --- jupyterhub/.env-example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example index dcb467521..949459b1f 100644 --- a/jupyterhub/.env-example +++ b/jupyterhub/.env-example @@ -1,14 +1,14 @@ # Example configuration file for Clowder JupyterHub KEYCLOAK_HOSTNAME="keycloak:8080/keycloak" # Development mode use the following line instead -#KEYCLOAK_HOSTNAME="host.docker.internal:8080/keycloak" +#KEYCLOAK_HOSTNAME="keycloak:8080/keycloak" KEYCLOAK_AUDIENCE="clowder" KEYCLOAK_REALM="clowder" JUPYTERHUB_ADMIN="admin" #Change network name to the one created by docker-compose -DOCKER_NETWORK_NAME="clowder2" +DOCKER_NETWORK_NAME="clowder2_clowder2" DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest" DOCKER_NOTEBOOK_DIR="/home/jovyan/work" JUPYTERHUB_CRYPT_KEY="" -CLOWDER_URL="localhost:3000" +CLOWDER_URL="localhost" PROD_DEPLOYMENT="false" From 4644855e40c7eeeaf9eb2acf7f0cfa097b50b19a Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Tue, 25 Jun 2024 14:05:59 -0500 Subject: [PATCH 21/24] better file names --- docker-compose.jupyter-dev.yml | 2 +- .../{jupyterhub_config.dev.py => jupyterhub_dev_config.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename jupyterhub/{jupyterhub_config.dev.py => jupyterhub_dev_config.py} (100%) diff --git a/docker-compose.jupyter-dev.yml b/docker-compose.jupyter-dev.yml index 006a778b4..bea03e898 100644 --- a/docker-compose.jupyter-dev.yml +++ b/docker-compose.jupyter-dev.yml @@ -11,7 +11,7 @@ services: - clowder2 volumes: # The JupyterHub configuration file - - ./jupyterhub/jupyterhub_config.dev.py:/srv/jupyterhub/jupyterhub_config.py:ro + - ./jupyterhub/jupyterhub_dev_config.py:/srv/jupyterhub/jupyterhub_config.py:ro # Bind Docker socket on the hostso we can connect to the daemon from # within the container - /var/run/docker.sock:/var/run/docker.sock:rw diff --git a/jupyterhub/jupyterhub_config.dev.py b/jupyterhub/jupyterhub_dev_config.py similarity index 100% rename from jupyterhub/jupyterhub_config.dev.py rename to jupyterhub/jupyterhub_dev_config.py From 378f89f2dcad7023bc0b34b4d3ca96de642a5169 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 28 Jun 2024 17:01:43 -0500 Subject: [PATCH 22/24] Updated prod webpack and a shell script for prod setup --- README.md | 4 +++- docker-prod.sh | 10 ++++++++++ frontend/webpack.config.prod.js | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100755 docker-prod.sh diff --git a/README.md b/README.md index e779590d0..425aa2dc4 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ There is a few other documentation links available on the [website](https://clow ## Installation The easiest way of running Clowder v2 is checking out the [code](https://github.com/clowder-framework/clowder2) -and running `docker compose up` in the main directory. +and running `docker compose up` in the main directory. If you would like to run Clowder with JupyterHub, +you can use our script `docker-prod.sh` to start the services. Run `./docker-prod.sh prod up` to start the services +and `./docker-prod.sh prod down` to stop them. Helm charts are available for running Clowder v2 on Kubernetes. See the [helm](https://github.com/clowder-framework/clowder2/tree/main/deployments/kubernetes/charts) directory for more information. diff --git a/docker-prod.sh b/docker-prod.sh new file mode 100755 index 000000000..e661a9a31 --- /dev/null +++ b/docker-prod.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +if [ "$1" = "prod" ] && [ "$2" = "up" ] +then + docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml up -d +fi + +if [ "$1" = "prod" ] && [ "$2" = "down" ] +then + docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml down +fi diff --git a/frontend/webpack.config.prod.js b/frontend/webpack.config.prod.js index 8102de89c..67ccf827b 100644 --- a/frontend/webpack.config.prod.js +++ b/frontend/webpack.config.prod.js @@ -11,6 +11,9 @@ import TerserPlugin from "terser-webpack-plugin"; console.log( `the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}` ); +console.log( + `the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}` +) export default { mode: "production", @@ -47,6 +50,7 @@ export default { CLOWDER_REMOTE_HOSTNAME: JSON.stringify( process.env.CLOWDER_REMOTE_HOSTNAME ), + JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL), APIKEY: JSON.stringify(process.env.APIKEY), KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL), }, From 20ca74a3cd31a7eea71d99e7db19571ec39519a5 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Fri, 28 Jun 2024 17:24:02 -0500 Subject: [PATCH 23/24] reverting changes --- jupyterhub/Dockerfile.jupyterhub | 3 ++- jupyterhub/jupyterhub_config.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index a27ff8c65..349782ad7 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -1,4 +1,5 @@ -FROM quay.io/jupyterhub/jupyterhub:latest +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION # Install dockerspawner, # hadolint ignore=DL3013 diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 85c7dbcaf..5f0b19b99 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -72,7 +72,7 @@ # TODO:Change this keycloak_url as required c.CustomTokenAuthenticator.auth_cookie_header = "Authorization" -c.CustomTokenAuthenticator.auth_username_key = "email" +c.CustomTokenAuthenticator.auth_username_key = "preferred_username" c.CustomTokenAuthenticator.auth_uid_number_key = "uid_number" c.CustomTokenAuthenticator.enable_auth_state = True c.CustomTokenAuthenticator.auto_login = True From a1f4161627bb1f451a08f7930d79e0a1df4e12a3 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Tue, 2 Jul 2024 09:23:21 -0500 Subject: [PATCH 24/24] Preloading is working, working on notebook --- jupyterhub/Clowder_APIs.ipynb | 68 ++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/jupyterhub/Clowder_APIs.ipynb b/jupyterhub/Clowder_APIs.ipynb index f975f1bc9..14beefba8 100644 --- a/jupyterhub/Clowder_APIs.ipynb +++ b/jupyterhub/Clowder_APIs.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 43, "id": "ffec7cb7-1a82-4148-aad1-bb3b1b19915b", "metadata": {}, "outputs": [], @@ -40,9 +40,7 @@ " response = requests.get(url)\n", " \n", " if response.status_code == 200:\n", - " with open(\"iris.csv\", \"wb\") as f:\n", - " f.write(response.content)\n", - " print(\"The Iris dataset has been downloaded and saved as iris.csv.\")\n", + " return response.content\n", " else:\n", " print(\"Failed to download the dataset. Status code:\", response.status_code)\n", "\n", @@ -60,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 44, "id": "59fca5f8-a5d6-419d-835a-023a73c5a1d7", "metadata": {}, "outputs": [ @@ -68,7 +66,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzcwNTgsImlhdCI6MTcxNjU3Njc1OCwianRpIjoiZjBjYmZlYzYtZjJlMS00NDg0LTg0YWMtNzFjMGMwZWNmNTNlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiIzOTgyN2EyNy1kMDA5LTRiY2YtOGIzYS1jZTk1MGM3YTI4OWQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiMzk4MjdhMjctZDAwOS00YmNmLThiM2EtY2U5NTBjN2EyODlkIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.Z3upy5xkoXfTwILujF7C8vpE_XaL8F32vuRd67V2TVqUK4qEVuN_5yTsyCcXyte9osG2jeZTHlJXreekOrpTvOW4Qbel1JAHnCQfGmlri_io-XFE57aT379HZrLGVlrpiHOGTqKG5mDVxIbaHEqdWhJkAEWY55rDqpkCScHUP7_SoJgAUj9Xx_LSS1hDHLwBCYRvGPFEoMADM40F-P3qxkW5Qsv9gPqP5ChpfG_7KZeBMRhtFXH_xk3M18qjNsSiQ9QKBFC2IrLFnT89bCK_slaUfUWLyiONt6ASPkAw85aajD3jc8RKq_AM42a2-wFVF7vERGPnoMscOpvjp9lBiQ'}\n" + "{'token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTk5MzAzNTMsImlhdCI6MTcxOTkzMDA1MywianRpIjoiYjZhYTAwNjUtZmY1OS00M2QyLWJiOGYtNzliNGI0MTgwNzNmIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2tleWNsb2FrL3JlYWxtcy9jbG93ZGVyIiwic3ViIjoiZjg0Y2JjNmQtYzEzZC00MmVmLWFhN2MtMWQ4MmFjYzVhZWViIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiY2xvd2RlcjItYmFja2VuZCIsInNlc3Npb25fc3RhdGUiOiJlZDM0MzA3Ny1lZmNiLTRlOGMtYjM4OS0zN2JlMzk0MmUyNzAiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cDovL2xvY2FsaG9zdDo4MDAwIl0sInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJkZWZhdWx0LXJvbGVzLWNsb3dkZXIiLCJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6Im9wZW5pZCBwcm9maWxlIGVtYWlsIiwic2lkIjoiZWQzNDMwNzctZWZjYi00ZThjLWIzODktMzdiZTM5NDJlMjcwIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJWaXNtYXlhayBNb2hhbmFyYWphbiIsInByZWZlcnJlZF91c2VybmFtZSI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSIsImdpdmVuX25hbWUiOiJWaXNtYXlhayIsImZhbWlseV9uYW1lIjoiTW9oYW5hcmFqYW4iLCJlbWFpbCI6Im1vaGFuYXIyQGlsbGlub2lzLmVkdSJ9.NAZs_sDtIdmy02bIjILG59jeRK99bMPdODQqEkyn-jC0YP_949LAPhWMSvxG5-eVdhHOeGILlbPmWgzU9tuw4YOkn-mNdVIBh16EicMJJ6zXeYzVj5RuUfhtZYJ3LfoknjuPBABI44dqo-Ixqh760m6HQKyZUfW62Lg-nGLYTvfoGtlResziGZ7u4L6vdsmmr_05SKGWdXxpVQUzwBNiXI2SO3FbWNk2uwR_qXL8WHyajHKCXvgHYHUbnAKJ1SRKO0xNfYDDlGKXuEn4fZfVmICq9693Z8emwydlcp8BVOaVuFqbsWwoRmEoSnK4bzLAH-CM-PFESPfbTrnu0Scnnw'}\n" ] } ], @@ -80,6 +78,9 @@ "login_url = CLOWDER_URL + \"/api/v2/login\"\n", "response = requests.post(login_url, json = user_login_json)\n", "token = response.json()[\"token\"]\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", "print(response.json())" ] }, @@ -93,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 45, "id": "91639e11-b553-4b57-a0df-6e91c661662f", "metadata": {}, "outputs": [ @@ -101,7 +102,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '6650e202fcd9057e97ae2d8a', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True}, 'created': '2024-05-24T18:52:50.457552', 'modified': '2024-05-24T18:52:50.457556', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" + "{'name': 'Flower Dataset', 'description': 'Dataset for Flower Data', 'status': 'PRIVATE', 'id': '66840cc55713c91cd9b89483', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True, 'read_only_user': False}, 'created': '2024-07-02T14:20:53.503934', 'modified': '2024-07-02T14:20:53.503942', 'user_views': 0, 'downloads': 0, 'thumbnail_id': None, 'standard_license': True, 'license_id': 'CC BY'}\n" ] } ], @@ -115,16 +116,57 @@ " \"license_id\": \"CC BY\"\n", "}\n", "\n", - "token = \"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI2dUVlQ0xOc1hTQXZUN1VDek1FRVk2VmI4ajJnY1RhWlFESUpnbnFGSHVJIn0.eyJleHAiOjE3MTY1NzY5NjcsImlhdCI6MTcxNjU3NjY2NywiYXV0aF90aW1lIjoxNzE2NTc2Mzc1LCJqdGkiOiI5NzU3NTM1Mi04Y2FhLTRhMmUtODZhZC0wM2JkNDcwODhhZjkiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAva2V5Y2xvYWsvcmVhbG1zL2Nsb3dkZXIiLCJzdWIiOiJmODRjYmM2ZC1jMTNkLTQyZWYtYWE3Yy0xZDgyYWNjNWFlZWIiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjbG93ZGVyMi1iYWNrZW5kIiwic2Vzc2lvbl9zdGF0ZSI6IjdhMjJjZDFkLTQ3NDctNDAyNS1iNDk2LTUyYzNhOWUwZDhiYyIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwOi8vbG9jYWxob3N0OjgwMDAiXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImRlZmF1bHQtcm9sZXMtY2xvd2RlciIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXX0sInNjb3BlIjoib3BlbmlkIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3YTIyY2QxZC00NzQ3LTQwMjUtYjQ5Ni01MmMzYTllMGQ4YmMiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwibmFtZSI6IlZpc21heWFrIE1vaGFuYXJhamFuIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1IiwiZ2l2ZW5fbmFtZSI6IlZpc21heWFrIiwiZmFtaWx5X25hbWUiOiJNb2hhbmFyYWphbiIsImVtYWlsIjoibW9oYW5hcjJAaWxsaW5vaXMuZWR1In0.AHqeZ79L8OUW2zTYEaDeUmuhHAeDMcqlgR8CE7ScGBRt-4gj8QFipbkiiMt0LYbMNi97dv0O_-iojrMtnlaX1OubnEwI_mMcoGckpXc5CzCoAQ_Qri2ZnpfnClDa9wPu-G_aGL-Sv4UpAglSTCbpBxsE99EiBubMb0T3NpP8p0k_lJTSvAZPJdMJ2sPpBo4BPUUIZq9JBXAZF7YPL9ZzUOoQKJvpNlA_7fHBHDjdSKRbVjP5cCNldjHJ70D2j1HM4JJUwMirNWZ8SFaJiny5a7NoZ2fa_JQkA3ZRrwGPHxGh6JThhu_F-a-pv2CDSytPCSU5DJxnslNJ4ePLVKDWpQ\"\n", - "headers = {\n", - " 'Authorization': f'Bearer {token}'\n", - "}\n", - "\n", "\n", "dataset_url = CLOWDER_URL + \"/api/v2/datasets\"\n", "response = requests.post(dataset_url, json = dataset_json, headers = headers, params = dataset_params)\n", + "dataset_id = response.json()['id']\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "id": "51ad8ad2-bcb6-40d4-9587-44f6a1a7c09d", + "metadata": {}, + "source": [ + "## Uploading File" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "ed02e036-edd0-403b-8292-9600946bc156", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'file', 'status': 'PRIVATE', 'id': '66840ced5713c91cd9b89485', 'creator': {'email': 'mohanar2@illinois.edu', 'first_name': 'Vismayak', 'last_name': 'Mohanarajan', 'id': '663a45d5b75ca83d17ac6564', 'admin': True, 'admin_mode': True, 'read_only_user': False}, 'created': '2024-07-02T14:21:33.733027', 'version_id': '2a4bbb53-8149-4384-9d55-2696c9d587ed', 'version_num': 1, 'dataset_id': '66840cc55713c91cd9b89483', 'folder_id': None, 'views': 0, 'downloads': 0, 'bytes': 4551, 'content_type': {'content_type': 'application/octet-stream', 'main_type': 'application'}, 'thumbnail_id': None, 'storage_type': 'minio', 'storage_path': None, 'object_type': 'file'}\n" + ] + } + ], + "source": [ + "\n", + "file = download_iris_dataset()\n", + "file_json = {\n", + " \"file\": file,\n", + " \"mediaType\": 'multipart/form-data'\n", + "}\n", + "save_file_url = CLOWDER_URL + \"/api/v2/datasets/\" + dataset_id + '/files'\n", + "headers = {\n", + " 'Authorization': f'Bearer {token}'\n", + "}\n", + "response = requests.post(save_file_url, files = file_json, headers = headers)\n", "print(response.json())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08c6673d-70d2-4879-948f-8db76eda4cde", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {