From cdec44c234ea07c82e10b54c4ea3bae468275d34 Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Tue, 7 Oct 2025 11:49:04 +0200 Subject: [PATCH] LCORE-598: Add authorization e2e tests This commit adds end-to-end tests for the authorization functionality. # Implementation details - Modified `.gitignore` to ignore leftover config backup files (unrelated to this change but useful) - Updated testing documentation to include instructions for running e2e tests - Added a JWK server container to `docker-compose.yaml` which is needed for e2e testing JWK auth - Added a `tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml` config file for lightspeed-stack with JWK auth enabled to be used in the JWK e2e tests - Removed hard dependency on the `docker` command in e2e tests and instead use the `CONTAINER_CMD` environment variable if set (to allow using `podman` instead of `docker`) - Added `tests/e2e/features/authorization_jwk.feature` which contains the actual e2e tests for JWK authz - Added `tests/e2e/features/steps/jwk_auth.py` which implements the steps for the JWK authz tests - Modified `tests/e2e/features/environment.py` to handle the new JWK authz tests, including creating a temporary JWK key pair for the tests and writing the public key to a file served by the JWK server container (which lightspeed-stack is directed to access through the config file mentioned above) functionality to functions that can be reused in the e2e tests - Added `tests/e2e/configuration/test_jwk/.gitignore` to ignore generated JWK files - Updated `tests/e2e/test_list.txt` to include the new JWK authz tests --- .gitignore | 3 + docker-compose.yaml | 17 +++++ docs/testing.md | 33 +++++++++ .../lightspeed-stack-auth-jwk.yaml | 52 ++++++++++++++ tests/e2e/configuration/test_jwk/.gitignore | 2 + tests/e2e/features/authorization_jwk.feature | 39 +++++++++++ tests/e2e/features/environment.py | 23 ++++++- tests/e2e/features/steps/health.py | 13 +++- tests/e2e/features/steps/jwk_auth.py | 41 ++++++++++++ tests/e2e/test_list.txt | 1 + tests/e2e/utils/utils.py | 4 +- tests/unit/authentication/test_jwk_token.py | 67 ++++++++++++------- 12 files changed, 264 insertions(+), 31 deletions(-) create mode 100644 tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml create mode 100644 tests/e2e/configuration/test_jwk/.gitignore create mode 100644 tests/e2e/features/authorization_jwk.feature create mode 100644 tests/e2e/features/steps/jwk_auth.py diff --git a/.gitignore b/.gitignore index 771821714..a233f70be 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,6 @@ dev/ # Database files *.sqlite + +# A file sometimes leftover by the e2e tests +lightspeed-stack.yaml.backup diff --git a/docker-compose.yaml b/docker-compose.yaml index b36245260..7a96bcc21 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -42,6 +42,23 @@ services: retries: 3 # how many times to retry before marking as unhealthy start_period: 5s # time to wait before starting checks + # This server is used for the e2e JWK tests + test-jwk-server: + # Conveniently uses the same Containerfile as lightspeed-stack as it has + # Python installed + build: + context: . + dockerfile: Containerfile + container_name: test-jwk-server + ports: + - "16161:16161" + networks: + - lightspeednet + working_dir: /app-root/test_jwk + entrypoint: ["python3", "-m", "http.server", "16161"] + volumes: + - ./tests/e2e/configuration/test_jwk:/app-root/test_jwk:Z + networks: lightspeednet: driver: bridge diff --git a/docs/testing.md b/docs/testing.md index 1e7bc4b43..bd288a88f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -141,7 +141,40 @@ End to end tests are based on [Behave](https://behave.readthedocs.io/en/stable/) * Defined in [tests/e2e](https://github.com/lightspeed-core/lightspeed-stack/tree/main/tests/e2e) +### Prerequisites for E2E tests +E2E tests require running services. You may run them using `podman-compose` or `docker-compose`. + +Some tests also require an OpenAI API key. You can run specific tests that do not require an OpenAI API key without it. + +First you need to build the images: + +```bash +podman-compose build +``` + +Then to start the services: + +```bash +OPENAI_API_KEY=your-api-key podman-compose up +``` + +And finally to run the actual tests: + +```bash +make test-e2e +``` + +If using `podman`, set `CONTAINER_CMD` accordingly: + +```bash +CONTAINER_CMD=podman make test-e2e +``` + +**Example of running a particular test with podman and live output:** +```bash +CONTAINER_CMD=podman uv run behave tests/e2e/features/authorization_jwk.feature --verbose --no-capture --no-capture-stderr +``` ## Tips and hints diff --git a/tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml b/tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml new file mode 100644 index 000000000..b56bad498 --- /dev/null +++ b/tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml @@ -0,0 +1,52 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Uses a remote llama-stack service + # The instance would have already been started with a llama-stack-run.yaml file + use_as_library_client: false + # Alternative for "as library use" + # use_as_library_client: true + # library_client_config_path: + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" + +authentication: + module: "jwk-token" + jwk_config: + url: "http://test-jwk-server:16161/jwk.json" + jwt_configuration: + user_id_claim: "user_id" + username_claim: "username" + role_rules: + - jsonpath: "$.roles[*]" + operator: "contains" + value: "admin" + roles: ["administrator"] + - jsonpath: "$.roles[*]" + operator: "contains" + value: "config" + roles: ["config"] + - jsonpath: "$.roles[*]" + operator: "contains" + value: "readonly" + roles: ["readonly"] + +authorization: + access_rules: + - role: "administrator" + actions: ["admin"] + - role: "config" + actions: ["get_config", "info"] + - role: "readonly" + actions: ["info"] diff --git a/tests/e2e/configuration/test_jwk/.gitignore b/tests/e2e/configuration/test_jwk/.gitignore new file mode 100644 index 000000000..4f83fad83 --- /dev/null +++ b/tests/e2e/configuration/test_jwk/.gitignore @@ -0,0 +1,2 @@ +# Ignore JWKs created for e2e tests +*.json diff --git a/tests/e2e/features/authorization_jwk.feature b/tests/e2e/features/authorization_jwk.feature new file mode 100644 index 000000000..e229f66d2 --- /dev/null +++ b/tests/e2e/features/authorization_jwk.feature @@ -0,0 +1,39 @@ +@JWKAuth +Feature: JWK authorization enforcement + + Background: + Given The service is started locally + And REST API service hostname is localhost + And REST API service port is 8080 + And REST API service prefix is /v1 + + Scenario: A user with the admin role can access the info endpoint + Given I have a valid JWT token with the admin role + When I access REST API endpoint "info" using HTTP GET method + Then The status code of the response is 200 + + Scenario: A user with the admin role can access the config endpoint + Given I have a valid JWT token with the admin role + When I access REST API endpoint "config" using HTTP GET method + Then The status code of the response is 200 + + Scenario: A user with the config role can access the config endpoint + Given I have a valid JWT token with the config role + When I access REST API endpoint "config" using HTTP GET method + Then The status code of the response is 200 + + Scenario: A user with the config role can access the info endpoint + Given I have a valid JWT token with the config role + When I access REST API endpoint "info" using HTTP GET method + Then The status code of the response is 200 + + Scenario: A user with the readonly role can access the info endpoint + Given I have a valid JWT token with the readonly role + When I access REST API endpoint "info" using HTTP GET method + Then The status code of the response is 200 + + Scenario: A user with the readonly role can't access the config endpoint + Given I have a valid JWT token with the readonly role + When I access REST API endpoint "config" using HTTP GET method + Then The status code of the response is 403 + And The body of the response contains Insufficient permissions diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index c46311601..3086f2d34 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -7,9 +7,11 @@ 4. after_scenario """ +import os import requests import subprocess import time +import json from behave.model import Scenario, Feature from behave.runner import Context @@ -25,6 +27,8 @@ except ImportError as e: print("Warning: unable to import module:", e) +from tests.unit.authentication.test_jwk_token import create_jwks_keys, make_key + def before_all(context: Context) -> None: """Run before and after the whole shooting match.""" @@ -55,7 +59,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None: try: # Start the llama-stack container again subprocess.run( - ["docker", "start", "llama-stack"], check=True, capture_output=True + [os.getenv("CONTAINER_CMD", "docker"), "start", "llama-stack"], ) # Wait for the service to be healthy @@ -67,7 +71,7 @@ def after_scenario(context: Context, scenario: Scenario) -> None: try: result = subprocess.run( [ - "docker", + os.getenv("CONTAINER_CMD", "docker"), "exec", "llama-stack", "curl", @@ -108,13 +112,26 @@ def before_feature(context: Context, feature: Feature) -> None: switch_config(context.feature_config) restart_container("lightspeed-stack") + elif "JWKAuth" in feature.tags: + context.feature_config = ( + "tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml" + ) + with open("tests/e2e/configuration/test_jwk/jwk.json", "w") as f: + context.test_key = make_key() + keys = create_jwks_keys([context.test_key], ["RS256"]) + f.write(json.dumps(keys)) + + context.default_config_backup = create_config_backup("lightspeed-stack.yaml") + switch_config("tests/e2e/configuration/lightspeed-stack-auth-jwk.yaml") + restart_container("lightspeed-stack") + if "Feedback" in feature.tags: context.feedback_conversations = [] def after_feature(context: Context, feature: Feature) -> None: """Run after each feature file is exercised.""" - if "Authorized" in feature.tags: + if "Authorized" in feature.tags or "JWKAuth" in feature.tags: switch_config(context.default_config_backup) restart_container("lightspeed-stack") remove_config_backup(context.default_config_backup) diff --git a/tests/e2e/features/steps/health.py b/tests/e2e/features/steps/health.py index 5b82daf84..77a1930b7 100644 --- a/tests/e2e/features/steps/health.py +++ b/tests/e2e/features/steps/health.py @@ -1,5 +1,6 @@ """Implementation of common test steps.""" +import os import subprocess import time from behave import given # pyright: ignore[reportAttributeAccessIssue] @@ -14,7 +15,13 @@ def llama_stack_connection_broken(context: Context) -> None: try: result = subprocess.run( - ["docker", "inspect", "-f", "{{.State.Running}}", "llama-stack"], + [ + os.getenv("CONTAINER_CMD", "docker"), + "inspect", + "-f", + "{{.State.Running}}", + "llama-stack", + ], capture_output=True, text=True, check=True, @@ -23,7 +30,9 @@ def llama_stack_connection_broken(context: Context) -> None: if result.stdout.strip(): context.llama_stack_was_running = True subprocess.run( - ["docker", "stop", "llama-stack"], check=True, capture_output=True + [os.getenv("CONTAINER_CMD", "docker"), "stop", "llama-stack"], + check=True, + capture_output=True, ) # Wait a moment for the connection to be fully disrupted diff --git a/tests/e2e/features/steps/jwk_auth.py b/tests/e2e/features/steps/jwk_auth.py new file mode 100644 index 000000000..c7a9b133a --- /dev/null +++ b/tests/e2e/features/steps/jwk_auth.py @@ -0,0 +1,41 @@ +"""JWK auth steps - reusing unit test primitives.""" + +from pathlib import Path +import sys + +from behave import given # pyright: ignore[reportAttributeAccessIssue] +from behave.runner import Context + + +from authlib.jose import JsonWebToken + +sys.path.append(str(Path(__file__).resolve().parents[4])) + +# Import at runtime to avoid module load issues +from tests.unit.authentication.test_jwk_token import ( + create_token_header, + create_token_payload, +) + + +@given("I have a valid JWT token with the {role} role") +def create_role_token(context: Context, role: str) -> None: + """Create token with role using the shared test key.""" + test_key = context.test_key + + header = create_token_header(test_key["kid"]) + payload = create_token_payload() + + # This works thanks to the definitions in lightspeed-stack-auth-jwk.yaml + payload["roles"] = [role] # Add role to existing payload + + token = ( + JsonWebToken(algorithms=["RS256"]) + .encode(header, payload, test_key["private_key"]) + .decode() + ) + + if not hasattr(context, "auth_headers"): + context.auth_headers = {} + + context.auth_headers["Authorization"] = f"Bearer {token}" diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 9d7cd0c8b..15872b62f 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -8,3 +8,4 @@ features/info.feature features/query.feature features/streaming_query.feature features/rest_api.feature +features/authorization_jwk.feature diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 54407350a..aa9d0af2d 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -37,7 +37,7 @@ def wait_for_container_health(container_name: str, max_attempts: int = 3) -> Non try: result = subprocess.run( [ - "docker", + os.getenv("CONTAINER_CMD", "docker"), "inspect", "--format={{.State.Health.Status}}", container_name, @@ -130,7 +130,7 @@ def restart_container(container_name: str) -> None: """Restart a Docker container by name and wait until it is healthy.""" try: subprocess.run( - ["docker", "restart", container_name], + [os.getenv("CONTAINER_CMD", "docker"), "restart", container_name], capture_output=True, text=True, check=True, diff --git a/tests/unit/authentication/test_jwk_token.py b/tests/unit/authentication/test_jwk_token.py index 0dc16f1ad..731393446 100644 --- a/tests/unit/authentication/test_jwk_token.py +++ b/tests/unit/authentication/test_jwk_token.py @@ -3,6 +3,7 @@ """Unit tests for functions defined in authentication/jwk_token.py""" import time +from typing import Any import pytest from fastapi import HTTPException, Request @@ -16,16 +17,22 @@ TEST_USER_ID = "test-user-123" TEST_USER_NAME = "testuser" +GeneratedKey = dict[str, Any] + + +def create_token_header(key_id) -> dict[str, str]: + """Create a sample token header.""" + return {"alg": "RS256", "typ": "JWT", "kid": key_id} + @pytest.fixture def token_header(single_key_set): """A sample token header.""" - return {"alg": "RS256", "typ": "JWT", "kid": single_key_set[0]["kid"]} + return create_token_header(single_key_set[0]["kid"]) -@pytest.fixture -def token_payload(): - """A sample token payload with the default user_id and username claims.""" +def create_token_payload() -> dict[str, Any]: + """Create a sample token payload with the default user_id and username claims.""" return { "user_id": TEST_USER_ID, "username": TEST_USER_NAME, @@ -34,7 +41,13 @@ def token_payload(): } -def make_key(): +@pytest.fixture +def token_payload(): + """A sample token payload with the default user_id and username claims.""" + return create_token_payload() + + +def make_key() -> GeneratedKey: """Generate a key pair for testing purposes.""" key = JsonWebKey.generate_key("RSA", 2048, is_private=True) return { @@ -45,19 +58,19 @@ def make_key(): @pytest.fixture -def single_key_set(): +def single_key_set() -> list[GeneratedKey]: """Default single-key set for signing tokens.""" return [make_key()] @pytest.fixture -def another_single_key_set(): +def another_single_key_set() -> list[GeneratedKey]: """Same as single_key_set, but generates a different key pair by being its own fixture.""" return [make_key()] @pytest.fixture -def valid_token(single_key_set, token_header, token_payload): +def valid_token(single_key_set, token_header, token_payload) -> str: """A token that is valid and signed with the signing keys.""" jwt_instance = JsonWebToken(algorithms=["RS256"]) return jwt_instance.encode( @@ -73,23 +86,29 @@ def clear_jwk_cache(): _jwk_cache.clear() +def create_jwks_keys( + key_set: list[GeneratedKey], algorithms: list[str] +) -> dict[str, Any]: + """Create JWK keys dict from key set and algorithms.""" + return { + "keys": [ + { + **key["private_key"].as_dict(private=False), + "kid": key["kid"], + "alg": alg, + } + for alg, key in zip(algorithms, key_set) + ], + } + + def make_signing_server(mocker, key_set, algorithms): """A fake server to serve our signing keys as JWKs.""" mock_session_class = mocker.patch("aiohttp.ClientSession") mock_response = mocker.AsyncMock() # Create JWK dict from private key as public key - keys = [ - { - **key["private_key"].as_dict(private=False), - "kid": key["kid"], - "alg": alg, - } - for alg, key in zip(algorithms, key_set) - ] - mock_response.json.return_value = { - "keys": keys, - } + mock_response.json.return_value = create_jwks_keys(key_set, algorithms) mock_response.raise_for_status = mocker.MagicMock(return_value=None) # Create mock session instance that acts as async context manager @@ -117,7 +136,7 @@ def mocked_signing_keys_server(mocker, single_key_set): @pytest.fixture -def default_jwk_configuration(): +def default_jwk_configuration() -> JwkConfiguration: """Default JwkConfiguration for testing.""" return JwkConfiguration( url=AnyHttpUrl("https://this#isgonnabemocked.com/jwks.json"), @@ -128,7 +147,7 @@ def default_jwk_configuration(): ) -def dummy_request(token): +def dummy_request(token: str) -> Request: """Generate a dummy request with a given token.""" return Request( scope={ @@ -140,7 +159,7 @@ def dummy_request(token): @pytest.fixture -def no_token_request(): +def no_token_request() -> Request: """Dummy request with no token.""" return Request( scope={ @@ -152,7 +171,7 @@ def no_token_request(): @pytest.fixture -def not_bearer_token_request(): +def not_bearer_token_request() -> Request: """Dummy request with no token.""" return Request( scope={ @@ -163,7 +182,7 @@ def not_bearer_token_request(): ) -def set_auth_header(request: Request, token: str): +def set_auth_header(request: Request, token: str) -> None: """Helper function to set the Authorization header in a request.""" new_headers = [ (k, v) for k, v in request.scope["headers"] if k.lower() != b"authorization"