Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
f260ecb
Remove $facet in top level group stages
WaVEV Nov 20, 2025
4dcf235
Wrap query for global aggregation
WaVEV Nov 25, 2025
d747310
[temp] remove Django tests for faster CI
timgraham Nov 8, 2025
fc3915b
INTPYTHON-527 Add Queryable Encryption support
aclark4life Jun 25, 2025
0e4c9ef
[wip] updates to field docs
timgraham Oct 30, 2025
913e669
add QuerySet tests
timgraham Oct 30, 2025
f5dfa89
use shared library instead of mongocryptd
timgraham Nov 5, 2025
233fa08
adjust message and add test for missing auto_encryption_opts error
timgraham Nov 7, 2025
9050bd0
polish howto
timgraham Nov 11, 2025
98dfa80
doc query limitations + docs polish + todos
timgraham Nov 12, 2025
93737ec
edit "Dynamic library path configuration"
timgraham Nov 12, 2025
fc30ab2
combine topic guide with howto
timgraham Nov 13, 2025
3e2ae81
Remove QE from 5.2.0 beta 2 release notes
aclark4life Nov 13, 2025
ce41b74
Add "start csfle servers" func to evergreen config
aclark4life Nov 15, 2025
8ca71d2
remove support for multiple kms providers
timgraham Nov 15, 2025
a64e494
fix join tests following foreignField/localField refactor
timgraham Nov 15, 2025
0f43b90
Add tests-8-qe to evergreen buildvariants
aclark4life Nov 17, 2025
34b710b
fix less than lookup on encrypted fields
timgraham Nov 18, 2025
8bc86ad
simplify "Configuring the Automatic Encryption Shared Library" to rem…
timgraham Nov 20, 2025
20c6c9a
reorder "Configuring the Automatic Encryption Shared Library" to make…
timgraham Nov 20, 2025
f83c854
update docs/tests for $facet removal
timgraham Nov 22, 2025
0bbb296
Combine crypt shared w/installation & db setup
aclark4life Dec 3, 2025
d47e067
`versionadded` updated from 5.2.3 to 6.0.0.
aclark4life Dec 3, 2025
060b315
Configure AWS KMS for testing on evergreen
aclark4life Dec 5, 2025
ffe0b17
doc edits
aclark4life Dec 7, 2025
2f73363
Address review
aclark4life Dec 10, 2025
8344284
Address UAT feedback
aclark4life Dec 10, 2025
08ef851
Address review
aclark4life Dec 11, 2025
723a46f
Update versionadded 6.0.0 -> 6.0.1
aclark4life Dec 11, 2025
78af9b4
Address UAT feedback
aclark4life Dec 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,17 @@ post_error_fails_task: true
post_timeout_secs: 1800 # 5 minutes

functions:
"start csfle servers":
- command: ec2.assume_role
params:
role_arn: ${aws_test_secrets_role}
- command: subprocess.exec
params:
binary: bash
include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"]
args:
- ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh

"setup":
- command: git.get_project
params:
Expand Down Expand Up @@ -44,19 +55,32 @@ functions:
params:
binary: bash
working_dir: "src"
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
args:
- ./.evergreen/run-tests.sh

"run encryption tests":
- command: subprocess.exec
type: test
params:
binary: bash
working_dir: "src"
include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
args:
- ./.evergreen/run-tests.sh
- encryption

"teardown":
- command: subprocess.exec
params:
binary: bash
args:
- ${DRIVERS_TOOLS}/.evergreen/teardown.sh
- ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh

pre:
- func: setup
- func: start csfle servers
- func: bootstrap mongo-orchestration

post:
Expand All @@ -67,6 +91,10 @@ tasks:
commands:
- func: "run unit tests"

- name: run-encryption-tests
commands:
- func: "run encryption tests"

buildvariants:
- name: tests-7-noauth-nossl
display_name: Run Tests 7.0 NoAuth NoSSL
Expand Down Expand Up @@ -111,3 +139,23 @@ buildvariants:
SSL: "ssl"
tasks:
- name: run-tests

- name: tests-8-qe-local
display_name: Run Tests 8.2 QE local KMS
run_on: rhel87-small
expansions:
MONGODB_VERSION: "8.2"
Copy link
Collaborator

@timgraham timgraham Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My impression is that we only want to test with dot zero MongoDB's, but lets get clarification on this.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use 8 then we get this:


2025/12/09 22:29:18.949] ======================================================================
[2025/12/09 22:29:18.949] ERROR: test_join (encryption_.test_fields.QueryTests)
[2025/12/09 22:29:18.949] ----------------------------------------------------------------------
[2025/12/09 22:29:18.949] Traceback (most recent call last):
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongo/synchronous/encryption.py", line 124, in _wrap_encryption_errors
[2025/12/09 22:29:18.949]     yield
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongo/synchronous/encryption.py", line 467, in encrypt
[2025/12/09 22:29:18.949]     encrypted_cmd = self._auto_encrypter.encrypt(database, encoded_cmd)
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/synchronous/auto_encrypter.py", line 43, in encrypt
[2025/12/09 22:29:18.949]     with self.mongocrypt.encryption_context(database, cmd) as ctx:
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 228, in encryption_context
[2025/12/09 22:29:18.949]     return EncryptionContext(
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 426, in __init__
[2025/12/09 22:29:18.949]     self._raise_from_status()
[2025/12/09 22:29:18.949]   File "/data/mci/85bf7b204919d8a8575368e5b4761ad6/src/venv/lib/python3.10/site-packages/pymongocrypt/mongocrypt.py", line 355, in _raise_from_status
[2025/12/09 22:29:18.949]     raise exc
[2025/12/09 22:29:18.949] pymongocrypt.errors.MongoCryptError: Encrypting 'aggregate' requires multiple schemas. Detected crypt_shared with version mongo_crypt_v1-dev-8.0.16, but need 8.1. Upgrade crypt_shared to 8.1 or newer.

Presumably because mongo-orchestration installs the corresponding crypt shared version to match the server version. We could --skip-crypt-shared and manually install crypt shared 8.2 similar to what is in GitHub Actions, but I'm not sure if it's worth the effort.

TOPOLOGY: replica_set
DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
tasks:
- name: run-encryption-tests

- name: tests-8-qe-aws
display_name: Run Tests 8.2 QE aws KMS
run_on: rhel87-small
expansions:
MONGODB_VERSION: "8.2"
TOPOLOGY: replica_set
DJANGO_SETTINGS_MODULE: "aws_kms_encrypted_settings"
tasks:
- name: run-encryption-tests
15 changes: 13 additions & 2 deletions .evergreen/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,22 @@

set -eux

# Install django-mongodb-backend
# Export secrets as environment variables
if [[ "${1:-}" == "encryption" ]]; then
. ../secrets-export.sh
fi

# Set up virtual environment
/opt/python/3.12/bin/python3 -m venv venv
. venv/bin/activate
python -m pip install -U pip
pip install -e .

# Conditionally install encryption extra if "encryption" arg is passed
if [[ "${1:-}" == "encryption" ]]; then
pip install -e '.[encryption]'
else
pip install -e .
fi

# Install django and test dependencies
git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo
Expand Down
8 changes: 4 additions & 4 deletions .evergreen/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ DRIVERS_TOOLS="$(dirname "$(pwd)")/drivers-tools"
PROJECT_DIRECTORY="$(pwd)"

if [ "Windows_NT" = "${OS:-}" ]; then
DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS")
PROJECT_DIRECTORY=$(cygpath -m "$PROJECT_DIRECTORY")
fi
export PROJECT_DIRECTORY
export DRIVERS_TOOLS
Expand All @@ -37,8 +37,8 @@ PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
EOT

# Set up drivers-tools with a .env file.
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
cat <<EOT > ${DRIVERS_TOOLS}/.env
git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git "${DRIVERS_TOOLS}"
cat <<EOT > "${DRIVERS_TOOLS}/.env"
CURRENT_VERSION="$CURRENT_VERSION"
DRIVERS_TOOLS="$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
Expand Down
26 changes: 26 additions & 0 deletions .github/workflows/aws_kms_encrypted_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from local_kms_encrypted_settings import * # noqa: F403

DATABASES["encrypted"] = { # noqa: F405
"ENGINE": "django_mongodb_backend",
"NAME": "djangotests_encrypted",
"OPTIONS": {
"auto_encryption_opts": AutoEncryptionOpts( # noqa: F405
key_vault_namespace="djangotests_encrypted.__keyVault",
kms_providers={
"aws": {
"accessKeyId": os.environ.get("FLE_AWS_KEY"), # noqa: F405
"secretAccessKey": os.environ.get("FLE_AWS_SECRET"), # noqa: F405
}
},
crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], # noqa: F405
crypt_shared_lib_required=True,
),
"directConnection": True,
},
"KMS_CREDENTIALS": {
"aws": {
"key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
"region": "us-east-1",
}
},
}
44 changes: 44 additions & 0 deletions .github/workflows/local_kms_encrypted_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Settings for django_mongodb_backend/tests when encryption is supported.
import os
from pathlib import Path

from mongodb_settings import * # noqa: F403
from pymongo.encryption import AutoEncryptionOpts

os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)

DATABASES["encrypted"] = { # noqa: F405
"ENGINE": "django_mongodb_backend",
"NAME": "djangotests_encrypted",
"OPTIONS": {
"auto_encryption_opts": AutoEncryptionOpts(
key_vault_namespace="djangotests_encrypted.__keyVault",
kms_providers={"local": {"key": os.urandom(96)}},
crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
),
"directConnection": True,
},
"KMS_CREDENTIALS": {"local": {}},
}


class EncryptedRouter:
def db_for_read(self, model, **hints):
if model._meta.app_label == "encryption_":
return "encrypted"
return None

db_for_write = db_for_read

def allow_migrate(self, db, app_label, model_name=None, **hints):
# The encryption_ app's models are only created in the encrypted
# database.
if app_label == "encryption_":
return db == "encrypted"
# Don't create other app's models in the encrypted database.
if db == "encrypted":
return False
return None


DATABASE_ROUTERS.append(EncryptedRouter()) # noqa: F405
1 change: 1 addition & 0 deletions .github/workflows/mongodb_settings.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Settings for django_mongodb_backend/tests.
from django_settings import * # noqa: F403

DATABASES["encrypted"] = {} # noqa: F405
DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]
145 changes: 0 additions & 145 deletions .github/workflows/runtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,151 +6,6 @@
from django.core.exceptions import ImproperlyConfigured

test_apps = [
"admin_changelist",
"admin_checks",
"admin_custom_urls",
"admin_docs",
"admin_filters",
"admin_inlines",
"admin_ordering",
"admin_scripts",
"admin_utils",
"admin_views",
"admin_widgets",
"aggregation",
"aggregation_regress",
"annotations",
"apps",
"async",
"auth_tests",
"backends",
"basic",
"bulk_create",
"cache",
"check_framework",
"constraints",
"contenttypes_tests",
"context_processors",
"custom_columns",
"custom_lookups",
"custom_managers",
"custom_pk",
"datatypes",
"dates",
"datetimes",
"db_functions",
"defer",
"defer_regress",
"delete",
"delete_regress",
"empty",
"empty_models",
"expressions",
"expressions_case",
"field_defaults",
"file_storage",
"file_uploads",
"fixtures",
"fixtures_model_package",
"fixtures_regress",
"flatpages_tests",
"force_insert_update",
"foreign_object",
"forms_tests",
"from_db_value",
"generic_inline_admin",
"generic_relations",
"generic_relations_regress",
"generic_views",
"get_earliest_or_latest",
"get_object_or_404",
"get_or_create",
"i18n",
"indexes",
"inline_formsets",
"introspection",
"invalid_models_tests",
"known_related_objects",
"lookup",
"m2m_and_m2o",
"m2m_intermediary",
"m2m_multiple",
"m2m_recursive",
"m2m_regress",
"m2m_signals",
"m2m_through",
"m2m_through_regress",
"m2o_recursive",
"managers_regress",
"many_to_many",
"many_to_one",
"many_to_one_null",
"max_lengths",
"messages_tests",
"migrate_signals",
"migration_test_data_persistence",
"migrations",
"model_fields",
"model_forms",
"model_formsets",
"model_formsets_regress",
"model_indexes",
"model_inheritance",
"model_inheritance_regress",
"model_options",
"model_package",
"model_regress",
"model_utils",
"modeladmin",
"multiple_database",
"mutually_referential",
"nested_foreign_keys",
"null_fk",
"null_fk_ordering",
"null_queries",
"one_to_one",
"or_lookups",
"order_with_respect_to",
"ordering",
"pagination",
"prefetch_related",
"proxy_model_inheritance",
"proxy_models",
"queries",
"queryset_pickle",
"redirects_tests",
"reserved_names",
"reverse_lookup",
"save_delete_hooks",
"schema",
"select_for_update",
"select_related",
"select_related_onetoone",
"select_related_regress",
"serializers",
"servers",
"sessions_tests",
"shortcuts",
"signals",
"sitemaps_tests",
"sites_framework",
"sites_tests",
"string_lookup",
"swappable_models",
"syndication_tests",
"test_client",
"test_client_regress",
"test_runner",
"test_utils",
"timezones",
"transactions",
"unmanaged_models",
"update",
"update_only_fields",
"user_commands",
"validation",
"view_tests",
"xor_lookups",
# Add directories in django_mongodb_backend/tests
*sorted(
[
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/test-python-atlas.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: install django-mongodb-backend
run: |
pip3 install --upgrade pip
pip3 install -e .
pip3 install -e .[encryption]
- name: Checkout Django
uses: actions/checkout@v6
with:
Expand All @@ -51,8 +51,15 @@ jobs:
run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py
- name: Start local Atlas
working-directory: .
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7
run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15
- name: Download crypt shared
run: |
wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so
- name: Run tests
run: python3 django_repo/tests/runtests_.py
permissions:
contents: read
env:
DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings"
CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"
Loading
Loading