diff --git a/.github/actions/init-local/action.yml b/.github/actions/init-local/action.yml index 306876973a..af8b057fe0 100644 --- a/.github/actions/init-local/action.yml +++ b/.github/actions/init-local/action.yml @@ -75,6 +75,37 @@ runs: size: 1Gi EOM + # Add a jhub startup app for testing + cat >> '${{ steps.metadata.outputs.config }}' <<- EOM + jhub_apps: + enabled: true + overrides: { + startup_apps: [ + { + "username": "service-account-jupyterhub", + "servername": "my-startup-server", + "user_options": { + "display_name": "My Startup Server-", + "description": "description", + "thumbnail": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mNkYPhfz0AEYBxVSF+FAP5FDvcfRYWgAAAAAElFTkSuQmCC", + "filepath": "panel_basic.py", + "framework": "panel", + "public": False, + "keep_alive": False, + "env": {"MY_ENV_VAR": "MY_VALUE"}, + "repository": {"url": "https://github.com/nebari-dev/jhub-apps-from-git-repo-example.git"}, + "conda_env": "global-mypanel", + "profile": "small-instance", + "share_with": { + "users": [], + "groups": ["/admin"] + }, + }, + }, + ] + } + EOM + - shell: bash run: | # Display Nebari config diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml index 7d79efc428..56d01cc715 100644 --- a/.github/workflows/test_local_integration.yaml +++ b/.github/workflows/test_local_integration.yaml @@ -99,7 +99,7 @@ jobs: - name: Create example-user working-directory: ${{ steps.init.outputs.directory }} run: | - nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }} + nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --groups developer --groups admin --config ${{ steps.init.outputs.config }} nebari keycloak listusers --config ${{ steps.init.outputs.config }} - name: Await Workloads diff --git a/src/_nebari/keycloak.py b/src/_nebari/keycloak.py index 6bfea9b8b3..0873a667f2 100644 --- a/src/_nebari/keycloak.py +++ b/src/_nebari/keycloak.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def do_keycloak(config: schema.Main, *args): +def do_keycloak(config: schema.Main, command, **kwargs): # suppress insecure warnings import urllib3 @@ -21,19 +21,17 @@ def do_keycloak(config: schema.Main, *args): keycloak_admin = get_keycloak_admin_from_config(config) - if args[0] == "adduser": - if len(args) < 2: + if command == "adduser": + if "password" not in kwargs: raise ValueError( "keycloak command 'adduser' requires `username [password]`" ) - username = args[1] - password = args[2] if len(args) >= 3 else None - create_user(keycloak_admin, username, password, domain=config.domain) - elif args[0] == "listusers": + create_user(keycloak_admin, **kwargs, domain=config.domain) + elif command == "listusers": list_users(keycloak_admin) else: - raise ValueError(f"unknown keycloak command {args[0]}") + raise ValueError(f"unknown keycloak command {command}") def create_user( diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf index 8a29bc2d41..5931b258b3 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/server.tf @@ -70,9 +70,9 @@ module "conda-store-openid-client" { "https://${var.external-url}/conda-store/oauth_callback" ] service-accounts-enabled = true - service-account-roles = [ - "view-realm", "view-users", "view-clients" - ] + service-account-roles = { + "realm-management" : ["view-realm", "view-users", "view-clients"] + } } diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py index 2557a497a7..5836885e43 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/02-spawner.py @@ -2,7 +2,6 @@ import json import kubernetes.client.models -from tornado import gen kubernetes.client.models.V1EndpointPort = ( kubernetes.client.models.CoreV1EndpointPort @@ -14,9 +13,8 @@ DEFAULT_PAGE_SIZE_LIMIT = 100 -@gen.coroutine -def get_username_hook(spawner): - auth_state = yield spawner.user.get_auth_state() +async def get_username_hook(spawner): + auth_state = await spawner.user.get_auth_state() username = auth_state["oauth_user"]["preferred_username"] spawner.environment.update( @@ -26,6 +24,13 @@ def get_username_hook(spawner): ) +async def pre_spawn_hook(spawner): + # if we are starting a service account pod, set/update auth_state + if spawner.user.name == spawner.authenticator.JHUB_SERVICE_ACCOUNT_NAME: + await spawner.authenticator.set_jhub_service_account_auth_state(spawner.user) + await get_username_hook(spawner) + + def get_total_records(url: str, token: str) -> int: import urllib3 @@ -88,7 +93,7 @@ def get_conda_store_environments(user_info: dict): return [f"{env['namespace']['name']}-{env['name']}" for env in env_data] -c.Spawner.pre_spawn_hook = get_username_hook +c.Spawner.pre_spawn_hook = pre_spawn_hook c.JupyterHub.allow_named_servers = False c.JupyterHub.spawner_class = KubeSpawner diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index b298ae5ae1..5741ab89f9 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -554,7 +554,6 @@ def render_profiles(spawner): # userinfo request to have the groups in the key # "auth_state.oauth_user.groups" auth_state = yield spawner.user.get_auth_state() - username = auth_state["oauth_user"]["preferred_username"] # only return the lowest level group name diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py index 2694b2a34e..0cc61035cd 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/files/jupyterhub/04-auth.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import os import time import urllib @@ -8,7 +9,7 @@ from jupyterhub import scopes from jupyterhub.traitlets import Callable from oauthenticator.generic import GenericOAuthenticator -from traitlets import Bool, Unicode, Union +from traitlets import Bool, Unicode, Union, default class KeyCloakOAuthenticator(GenericOAuthenticator): @@ -18,6 +19,13 @@ class KeyCloakOAuthenticator(GenericOAuthenticator): feature added in JupyterHub 5.0 (https://github.com/jupyterhub/jupyterhub/pull/4748). """ + JHUB_SERVICE_ACCOUNT_NAME = Unicode() + + # Keycloak currently dictates service account name format as `service-account-` See https://github.com/keycloak/keycloak/blob/5e6bb9f7bd2c83febd12668f2605aa8ecbdcf130/docs/documentation/server_admin/topics/admin-cli.adoc?plain=1#L1008 for more info. + @default("JHUB_SERVICE_ACCOUNT_NAME") + def _default_jhub_service_account_name(self): + return f"service-account-{self.client_id}" + claim_roles_key = Union( [Unicode(os.environ.get("OAUTH2_ROLES_KEY", "groups")), Callable()], config=True, @@ -30,6 +38,39 @@ class KeyCloakOAuthenticator(GenericOAuthenticator): reset_managed_roles_on_startup = Bool(True) + async def set_jhub_service_account_auth_state(self, user): + if user.name != self.JHUB_SERVICE_ACCOUNT_NAME: + raise ValueError( + f'User name "{user.name}" does not match service account name "{self.JHUB_SERVICE_ACCOUNT_NAME}"' + ) + auth_model = await self.authenticate_service_account() + await user.save_auth_state(auth_model["auth_state"]) + logging.info(f'Auth state set for service account: "{user.name}"') + + async def authenticate_service_account(self): + # We mimic what OAuthenticator currently does in `authenticate` method, but the logic may change in the future + # Currently, the logic is based on https://github.com/jupyterhub/oauthenticator/blob/d31bb193e84e7cda58b16f2f5d385c9b8affda4f/oauthenticator/oauth2.py#L1436 + + token_info = await self._get_token_info() + + # Get user info using the access token + user_info = await self.token_to_user(token_info) + + # Get/set username + username = self.user_info_to_username(user_info) + username = self.normalize_username(username) + + # Build auth model similar to OAuth flow + auth_model = { + "name": username, + "admin": True if username in self.admin_users else None, + "auth_state": self.build_auth_state_dict(token_info, user_info), + } + + auth_model = await self.update_auth_model(auth_model) + + return auth_model + async def update_auth_model(self, auth_model): """Updates and returns the auth_model dict. This function is called every time a user authenticates with JupyterHub, as in @@ -46,15 +87,15 @@ async def update_auth_model(self, auth_model): user_id = auth_model["auth_state"]["oauth_user"]["sub"] token = await self._get_token() - jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token) + jupyterhub_client_uuid = await self._get_jupyterhub_client_uuid(token=token) user_info = auth_model["auth_state"][self.user_auth_state_key] user_roles_from_claims = self._get_user_roles(user_info=user_info) keycloak_api_call_start = time.time() user_roles = await self._get_client_roles_for_user( - user_id=user_id, client_id=jupyterhub_client_id, token=token + user_id=user_id, client_id=jupyterhub_client_uuid, token=token ) user_roles_rich = await self._get_roles_with_attributes( - roles=user_roles, client_id=jupyterhub_client_id, token=token + roles=user_roles, client_id=jupyterhub_client_uuid, token=token ) # Include which groups have permission to mount shared directories (user by @@ -63,7 +104,7 @@ async def update_auth_model(self, auth_model): await self.get_client_groups_with_mount_permissions( user_groups=auth_model["auth_state"]["oauth_user"]["groups"], user_roles=user_roles_rich, - client_id=jupyterhub_client_id, + client_id=jupyterhub_client_uuid, token=token, ) ) @@ -114,7 +155,7 @@ async def _get_jupyterhub_client_roles(self, jupyterhub_client_id, token): ) return client_roles_rich - async def _get_jupyterhub_client_id(self, token): + async def _get_jupyterhub_client_uuid(self, token): # Get the clients list to find the "id" of "jupyterhub" client. clients_data = await self._fetch_api(endpoint="clients/", token=token) jupyterhub_clients = [ @@ -131,9 +172,9 @@ async def load_managed_roles(self): "Managed roles can only be loaded when `manage_roles` is True" ) token = await self._get_token() - jupyterhub_client_id = await self._get_jupyterhub_client_id(token=token) + jupyterhub_client_uuid = await self._get_jupyterhub_client_uuid(token=token) client_roles_rich = await self._get_jupyterhub_client_roles( - jupyterhub_client_id=jupyterhub_client_id, token=token + jupyterhub_client_id=jupyterhub_client_uuid, token=token ) # Includes roles like "default-roles-nebari", "offline_access", "uma_authorization" @@ -168,7 +209,7 @@ async def load_managed_roles(self): await self._get_users_and_groups_for_role( role_name, token=token, - client_id=jupyterhub_client_id, + client_id=jupyterhub_client_uuid, ) ) @@ -307,7 +348,7 @@ def _get_user_roles(self, user_info): ) return set() - async def _get_token(self) -> str: + async def _get_token_info(self) -> str: http = self.http_client body = urllib.parse.urlencode( @@ -322,8 +363,12 @@ async def _get_token(self) -> str: method="POST", body=body, ) - data = json.loads(response.body) - return data["access_token"] # type: ignore[no-any-return] + token_info = json.loads(response.body) + return token_info + + async def _get_token(self) -> str: + token_info = await self._get_token_info() + return token_info["access_token"] # type: ignore[no-any-return] async def _fetch_api(self, endpoint: str, token: str): response = await self.http_client.fetch( diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf index 9a0675fc85..54ca96b655 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf @@ -344,9 +344,10 @@ module "jupyterhub-openid-client" { ] jupyterlab_profiles_mapper = true service-accounts-enabled = true - service-account-roles = [ - "view-realm", "view-users", "view-clients" - ] + service-account-roles = { + "realm-management" : ["view-realm", "view-users", "view-clients"], + "jupyterhub" : ["allow-app-sharing-role"] + } } diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf new file mode 100644 index 0000000000..0ddb981e5e --- /dev/null +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + keycloak = { + source = "mrparkers/keycloak" + version = "3.7.0" + } + } + required_version = ">= 1.0" +} diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf index e23aeb13c8..20a37e580c 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/main.tf @@ -67,29 +67,54 @@ data "keycloak_realm" "master" { realm = "nebari" } -data "keycloak_openid_client" "realm_management" { - realm_id = var.realm_id - client_id = "realm-management" -} -data "keycloak_role" "main-service" { - for_each = toset(var.service-account-roles) +# Get client data for each service account client +data "keycloak_openid_client" "service_clients" { + for_each = var.service-account-roles - realm_id = data.keycloak_realm.master.id - client_id = data.keycloak_openid_client.realm_management.id - name = each.key -} + realm_id = var.realm_id + client_id = each.key + depends_on = [keycloak_openid_client.main] +} + +# Get role data for each client's roles +data "keycloak_role" "client_roles" { + for_each = { + for pair in flatten([ + for client, roles in var.service-account-roles : [ + for role in roles : { + key = "${client}-${role}" + client = client + role = role + } + ] + ]) : pair.key => pair + } -resource "keycloak_openid_client_service_account_role" "main" { - for_each = toset(var.service-account-roles) + realm_id = var.realm_id + client_id = data.keycloak_openid_client.service_clients[each.value.client].id + name = each.value.role +} + +resource "keycloak_openid_client_service_account_role" "client_roles" { + for_each = { + for pair in flatten([ + for client, roles in var.service-account-roles : [ + for role in roles : { + key = "${client}-${role}" + client = client + role = role + } + ] + ]) : pair.key => pair + } realm_id = var.realm_id service_account_user_id = keycloak_openid_client.main.service_account_user_id - client_id = data.keycloak_openid_client.realm_management.id - role = data.keycloak_role.main-service[each.key].name + client_id = data.keycloak_openid_client.service_clients[each.value.client].id + role = data.keycloak_role.client_roles[each.key].name } - resource "keycloak_role" "main" { for_each = toset(flatten(values(var.role_mapping))) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf index 6077c22b0e..8f87eaf108 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/outputs.tf @@ -12,3 +12,10 @@ output "config" { callback_urls = var.callback-url-paths } } + +output "client_role_ids" { + description = "Map of role names to their IDs" + value = { + for role_key, role in keycloak_role.default_client_roles : role_key => role.id + } +} diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf index 7626cc2b93..08ce465dff 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/keycloak-client/variables.tf @@ -22,10 +22,17 @@ variable "service-accounts-enabled" { default = false } + variable "service-account-roles" { - description = "Roles to be granted to the service account. Requires setting service-accounts-enabled to true." - type = list(string) - default = [] + description = <<-EOT + Map of client to client-roles to be granted to the service account client. Requires setting service-accounts-enabled to true. + + e.g. { + "my-client": ["my-client-role"], + } + EOT + type = map(list(string)) + default = {} } diff --git a/src/_nebari/subcommands/keycloak.py b/src/_nebari/subcommands/keycloak.py index 8f57d34175..d59b08232f 100644 --- a/src/_nebari/subcommands/keycloak.py +++ b/src/_nebari/subcommands/keycloak.py @@ -1,11 +1,14 @@ import json import pathlib -from typing import Tuple +from typing import List, Tuple import typer from _nebari.config import read_configuration -from _nebari.keycloak import do_keycloak, export_keycloak_users +from _nebari.keycloak import ( + do_keycloak, + export_keycloak_users, +) from nebari.hookspecs import hookimpl @@ -30,6 +33,9 @@ def add_user( add_users: Tuple[str, str] = typer.Option( ..., "--user", help="Provide both: " ), + groups: List[str] = typer.Option( + None, "-g", "--groups", help="Provide existing groups to add user to" + ), config_filename: pathlib.Path = typer.Option( ..., "-c", @@ -40,10 +46,12 @@ def add_user( """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" from nebari.plugins import nebari_plugin_manager - args = ["adduser", add_users[0], add_users[1]] + kwargs = {"username": add_users[0], "password": add_users[1], "groups": groups} + config_schema = nebari_plugin_manager.config_schema config = read_configuration(config_filename, config_schema) - do_keycloak(config, *args) + + do_keycloak(config, command="adduser", **kwargs) @app_keycloak.command(name="listusers") def list_users( @@ -57,10 +65,9 @@ def list_users( """List the users in Keycloak.""" from nebari.plugins import nebari_plugin_manager - args = ["listusers"] config_schema = nebari_plugin_manager.config_schema config = read_configuration(config_filename, config_schema) - do_keycloak(config, *args) + do_keycloak(config, command="listusers") @app_keycloak.command(name="export-users") def export_users( diff --git a/tests/tests_deployment/test_jupyterhub_api.py b/tests/tests_deployment/test_jupyterhub_api.py index aaeaf535ac..33330c593d 100644 --- a/tests/tests_deployment/test_jupyterhub_api.py +++ b/tests/tests_deployment/test_jupyterhub_api.py @@ -1,3 +1,5 @@ +from typing import Set + import pytest import requests @@ -14,30 +16,56 @@ from tests.tests_deployment.utils import get_refresh_jupyterhub_token +@pytest.mark.parametrize( + "username,expected_roles", + [ + ( + constants.KEYCLOAK_USERNAME, + { + "user", + "manage-account", + "jupyterhub_developer", + "argo-developer", + "dask_gateway_developer", + "grafana_viewer", + "conda_store_developer", + "argo-viewer", + "grafana_developer", + "manage-account-links", + "view-profile", + "allow-read-access-to-services-role", + "allow-group-directory-creation-role", + # admin roles + "admin", + "grafana_admin", + "conda_store_admin", + "argo-admin", + "manage-users", + "query-groups", + "query-users", + "jupyterhub_admin", + "dask_gateway_admin", + }, + ), + ( + "service-account-jupyterhub", + {"allow-app-sharing-role", "default-roles-nebari", "user"}, + ), + ], + ids=["test-user", "jupyterhub-service-account"], +) @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") -def test_jupyterhub_loads_roles_from_keycloak(jupyterhub_access_token): +def test_jupyterhub_loads_roles_from_keycloak( + jupyterhub_access_token: str, username: str, expected_roles: Set[str] +): + """Test that JupyterHub correctly loads roles from Keycloak for different users""" response = requests.get( - url=f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}", + url=f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{username}", headers={"Authorization": f"Bearer {jupyterhub_access_token}"}, verify=False, ) - user = response.json() - assert set(user["roles"]) == { - "user", - "manage-account", - "jupyterhub_developer", - "argo-developer", - "dask_gateway_developer", - "grafana_viewer", - "conda_store_developer", - "argo-viewer", - "grafana_developer", - "manage-account-links", - "view-profile", - # default roles - "allow-read-access-to-services-role", - "allow-group-directory-creation-role", - } + actual_roles = set(response.json()["roles"]) + assert actual_roles == expected_roles @token_parameterized(note="get-default-scopes") @@ -151,4 +179,18 @@ def test_jupyterhub_loads_groups_from_keycloak(jupyterhub_access_token): verify=False, ) user = response.json() - assert set(user["groups"]) == {"/analyst", "/developer", "/users"} + assert set(user["groups"]) == {"/analyst", "/developer", "/admin", "/users"} + + +@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") +def test_startup_apps_created(jupyterhub_access_token): + response = requests.get( + f"https://{constants.NEBARI_HOSTNAME}/hub/api/users/{constants.KEYCLOAK_USERNAME}/shared", + params={"include_stopped_servers": True}, + headers={"Authorization": f"Bearer {jupyterhub_access_token}"}, + verify=False, + ) + shared_servers = response.json() + assert "my-startup-server" in { + item["server"]["name"] for item in shared_servers["items"] + }