Address review

- avoid encryption tasks on non-encrypted builds
- retrieve configured provider from conn settings
- fail loudly & avoid uppercased values in settings

Author: aclark4life

.evergreen/config.yml

@@ -59,6 +59,17 @@ functions:
         args:
           - ./.evergreen/run-tests.sh
 
+  "run encryption tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
+        args:
+          - ./.evergreen/run-tests.sh
+          - encryption
+
   "teardown":
     - command: subprocess.exec
       params:
@@ -80,6 +91,10 @@ tasks:
     commands:
       - func: "run unit tests"
 
+  - name: run-encryption-tests
+    commands:
+      - func: "run encryption tests"
+
 buildvariants:
   - name: tests-6-noauth-nossl
     display_name: Run Tests 6.0 NoAuth NoSSL
@@ -125,12 +140,22 @@ buildvariants:
     tasks:
       - name: run-tests
 
-  - name: tests-8-qe
-    display_name: Run Tests 8.2 QE
+  - name: tests-8-qe-local
+    display_name: Run Tests 8.2 QE local KMS
     run_on: rhel87-small
     expansions:
       MONGODB_VERSION: "8.2"
       TOPOLOGY: replica_set
-      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+      DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
     tasks:
-      - name: run-tests
+      - name: run-encryption-tests
+
+  - name: tests-8-qe-aws
+    display_name: Run Tests 8.2 QE aws KMS
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "aws_kms_encrypted_settings"
+    tasks:
+      - name: run-encryption-tests

.evergreen/run-tests.sh

@@ -3,13 +3,21 @@
 set -eux
 
 # Export secrets as environment variables
-. ../secrets-export.sh
+if [[ "${1:-}" == "encryption" ]]; then
+    . ../secrets-export.sh
+fi
 
-# Install django-mongodb-backend
+# Set up virtual environment
 /opt/python/3.10/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
-pip install -e '.[encryption]'
+
+# Conditionally install encryption extra if "encryption" arg is passed
+if [[ "${1:-}" == "encryption" ]]; then
+    pip install -e '.[encryption]'
+else
+    pip install -e .
+fi
 
 # Install django and test dependencies
 git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo

.github/workflows/aws_kms_encrypted_settings.py

@@ -0,0 +1,26 @@
+from local_kms_encrypted_settings import *  # noqa: F403
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(  # noqa: F405
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers={
+                "aws": {
+                    "accessKeyId": os.environ.get("FLE_AWS_KEY"),  # noqa: F405
+                    "secretAccessKey": os.environ.get("FLE_AWS_SECRET"),  # noqa: F405
+                }
+            },
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],  # noqa: F405
+            crypt_shared_lib_required=True,
+        ),
+        "directConnection": True,
+    },
+    "KMS_CREDENTIALS": {
+        "aws": {
+            "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+            "region": "us-east-1",
+        }
+    },
+}

.github/workflows/encrypted_settings.py …orkflows/local_kms_encrypted_settings.py.github/workflows/encrypted_settings.py renamed to .github/workflows/local_kms_encrypted_settings.py .github/workflows/encrypted_settings.py renamed to .github/workflows/local_kms_encrypted_settings.py

@@ -7,38 +7,18 @@
 
 os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)
 
-AWS_CREDS = {
-    "accessKeyId": os.environ.get("FLE_AWS_KEY", ""),
-    "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""),
-}
-
-_USE_AWS_KMS = any(AWS_CREDS.values())
-
-if _USE_AWS_KMS:
-    _AWS_REGION = os.environ.get("FLE_AWS_KMS_REGION", "us-east-1")
-    _AWS_KEY_ARN = os.environ.get(
-        "FLE_AWS_KMS_KEY_ARN",
-        "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-    )
-    KMS_PROVIDERS = {"aws": AWS_CREDS}
-    KMS_CREDENTIALS = {"aws": {"key": _AWS_KEY_ARN, "region": _AWS_REGION}}
-else:
-    KMS_PROVIDERS = {"local": {"key": os.urandom(96)}}
-    KMS_CREDENTIALS = {"local": {}}
-
 DATABASES["encrypted"] = {  # noqa: F405
     "ENGINE": "django_mongodb_backend",
     "NAME": "djangotests_encrypted",
     "OPTIONS": {
         "auto_encryption_opts": AutoEncryptionOpts(
             key_vault_namespace="djangotests_encrypted.__keyVault",
-            kms_providers=KMS_PROVIDERS,
+            kms_providers={"local": {"key": os.urandom(96)}},
             crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
-            crypt_shared_lib_required=True,
         ),
         "directConnection": True,
     },
-    "KMS_CREDENTIALS": KMS_CREDENTIALS,
+    "KMS_CREDENTIALS": {"local": {}},
 }
 
 

.github/workflows/test-python-atlas.yml

@@ -61,5 +61,5 @@ jobs:
     permissions:
         contents: read
     env:
-      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+      DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
       CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"

tests/encryption_/test_base.py

@@ -1,5 +1,3 @@
-import os
-
 import pymongo
 from bson.binary import Binary
 from django.conf import settings
@@ -21,18 +19,3 @@ def assertEncrypted(self, model, field):
             collection = db[model._meta.db_table]
             data = collection.find_one({}, {field: 1, "_id": 0})
             self.assertIsInstance(data[field], Binary)
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        AWS_CREDS = {
-            "accessKeyId": os.environ.get("FLE_AWS_KEY", ""),
-            "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""),
-        }
-        _USE_AWS_KMS = any(AWS_CREDS.values())
-
-        if _USE_AWS_KMS:
-            self.DEFAULT_KMS_PROVIDER = "aws"
-        else:
-            # Local-only fallback
-            self.DEFAULT_KMS_PROVIDER = "local"

tests/encryption_/test_management.py

@@ -114,6 +114,9 @@ def test_show_encrypted_fields_map(self):
                 self._compare_output(expected, command_output[model_key])
 
     def test_missing_key(self):
+        connection = connections["encrypted"]
+        auto_encryption_opts = connection.connection._options.auto_encryption_opts
+        kms_providers = auto_encryption_opts._kms_providers
         test_key = "encryption__patient.patient_record.ssn"
         msg = (
             f"Encryption key {test_key} not found. Have migrated the "
@@ -125,11 +128,10 @@ def test_missing_key(self):
                 call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0)
         finally:
             # Replace the deleted key.
-            master_key = connections["encrypted"].settings_dict["KMS_CREDENTIALS"][
-                self.DEFAULT_KMS_PROVIDER
-            ]
-            connections["encrypted"].client_encryption.create_data_key(
-                kms_provider=self.DEFAULT_KMS_PROVIDER,
+            kms_provider = next(iter(kms_providers.keys()))
+            master_key = connection.settings_dict["KMS_CREDENTIALS"][kms_provider]
+            connection.client_encryption.create_data_key(
+                kms_provider=kms_provider,
                 master_key=master_key,
                 key_alt_names=[test_key],
             )