From 8b740fd04de97182f01b8821649f56409fc8dd44 Mon Sep 17 00:00:00 2001 From: floris272 Date: Thu, 4 Dec 2025 12:43:42 +0100 Subject: [PATCH 1/7] :sparkles: [#564] add command to import objecttypes from api --- src/objects/core/constants.py | 25 +++ src/objects/core/management/__init__.py | 0 .../core/management/commands/__init__.py | 0 .../management/commands/import_objecttypes.py | 134 +++++++++++ ...metry_objecttype_contact_email_and_more.py | 115 ++++++++++ src/objects/core/models.py | 209 +++++++++++++++++- src/objects/core/tests/factories.py | 20 +- .../core/tests/test_import_objecttypes.py | 157 +++++++++++++ .../tests/test_objecttypeversion_generate.py | 33 +++ src/objects/core/utils.py | 14 +- src/objects/tests/utils.py | 112 ++++++++++ src/objects/utils/client.py | 5 + 12 files changed, 820 insertions(+), 4 deletions(-) create mode 100644 src/objects/core/constants.py create mode 100644 src/objects/core/management/__init__.py create mode 100644 src/objects/core/management/commands/__init__.py create mode 100644 src/objects/core/management/commands/import_objecttypes.py create mode 100644 src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py create mode 100644 src/objects/core/tests/test_import_objecttypes.py create mode 100644 src/objects/core/tests/test_objecttypeversion_generate.py diff --git a/src/objects/core/constants.py b/src/objects/core/constants.py new file mode 100644 index 00000000..d795b06f --- /dev/null +++ b/src/objects/core/constants.py @@ -0,0 +1,25 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class ObjectVersionStatus(models.TextChoices): + published = "published", _("Published") + draft = "draft", _("Draft") + deprecated = "deprecated", _("Deprecated") + + +class DataClassificationChoices(models.TextChoices): + open = "open", _("Open") + intern = "intern", _("Intern") + confidential = "confidential", _("Confidential") + strictly_confidential = "strictly_confidential", _("Strictly confidential") + + +class UpdateFrequencyChoices(models.TextChoices): + real_time = "real_time", _("Real-time") + hourly = "hourly", _("Hourly") + daily = "daily", _("Daily") + weekly = "weekly", _("Weekly") + monthly = "monthly", _("Monthly") + yearly = "yearly", _("Yearly") + unknown = "unknown", _("Unknown") diff --git a/src/objects/core/management/__init__.py b/src/objects/core/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/objects/core/management/commands/__init__.py b/src/objects/core/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/objects/core/management/commands/import_objecttypes.py b/src/objects/core/management/commands/import_objecttypes.py new file mode 100644 index 00000000..e5c9ee39 --- /dev/null +++ b/src/objects/core/management/commands/import_objecttypes.py @@ -0,0 +1,134 @@ +from typing import Any + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from django.utils.translation import gettext as _ + +from djangorestframework_camel_case.util import underscoreize +from packaging.version import Version +from requests.exceptions import HTTPError +from zgw_consumers.models import Service + +from objects.core.models import ObjectType, ObjectTypeVersion +from objects.utils.client import get_objecttypes_client + +MIN_OBJECTTYPES_VERSION = "2.2.2" + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "service_slug", + help=_("Slug of the service"), + ) + + @transaction.atomic + def handle(self, *args, **options): + service_slug = options["service_slug"] + service = self._get_service(service_slug) + + with get_objecttypes_client(service) as client: + try: + self._check_objecttypes_api_version(client) + + objecttypes = client.list_objecttypes() + data = self._parse_objecttype_data(objecttypes, service) + self._bulk_create_or_update_objecttypes(data) + self.stdout.write("Successfully imported %s objecttypes" % len(data)) + + for objecttype in data: + objecttype_versions = client.list_objecttype_versions( + objecttype.uuid + ) + data = self._parse_objectversion_data( + objecttype_versions, objecttype + ) + self._bulk_create_or_update_objecttype_versions(data) + self.stdout.write( + "Successfully imported %s versions for type: %s" + % (len(data), objecttype.name) + ) + + except HTTPError as e: + raise CommandError(_("Request failed: {}").format(e)) + + def _get_service(self, slug): + try: + return Service.objects.get(slug=slug) + except Service.DoesNotExist: + raise CommandError(_("Service '{}' does not exist").format(slug)) + + def _check_objecttypes_api_version(self, client): + api_version = client.get_objecttypes_api_version() + if api_version is None or Version( + client.get_objecttypes_api_version() + ) < Version(MIN_OBJECTTYPES_VERSION): + raise CommandError( + _("Object types API version must be {} or higher.").format( + MIN_OBJECTTYPES_VERSION + ) + ) + + def _bulk_create_or_update_objecttypes(self, data): + ObjectType.objects.bulk_create( + data, + update_conflicts=True, + unique_fields=["uuid", "service"], # TODO remove service + update_fields=[ + "name", + "name_plural", + "description", + "data_classification", + "maintainer_organization", + "maintainer_department", + "contact_person", + "contact_email", + "source", + "update_frequency", + "provider_organization", + "documentation_url", + "labels", + "created_at", + "modified_at", + "allow_geometry", + "linkable_to_zaken", + ], + ) + + def _bulk_create_or_update_objecttype_versions(self, data): + ObjectTypeVersion.objects.bulk_create( + data, + ignore_conflicts=True, + unique_fields=[ + "object_type", + "version", + ], + update_fields=[ + "created_at", + "modified_at", + "published_at", + "json_schema", + "status", + ], + ) + + def _parse_objecttype_data( + self, objecttypes: list[dict[str, Any]], service: Service + ) -> list[ObjectType]: + data = [] + for objecttype in objecttypes: + objecttype.pop("versions") + objecttype.pop("url") + objecttype["service"] = service + data.append(ObjectType(**underscoreize(objecttype))) + return data + + def _parse_objectversion_data( + self, objecttype_versions: list[dict[str, Any]], objecttype + ) -> list[ObjectTypeVersion]: + data = [] + for objecttype_version in objecttype_versions: + objecttype_version.pop("url") + objecttype_version["objectType"] = objecttype + data.append(ObjectTypeVersion(**underscoreize(objecttype_version))) + return data diff --git a/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py b/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py new file mode 100644 index 00000000..cf673ee4 --- /dev/null +++ b/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py @@ -0,0 +1,115 @@ +# Generated by Django 5.2.7 on 2025-12-04 11:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0034_alter_objectrecord__object_type_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='objecttype', + name='allow_geometry', + field=models.BooleanField(default=True, help_text="Shows whether the related objects can have geographic coordinates. If the value is 'false' the related objects are not allowed to have coordinates and the creation/update of objects with `geometry` property will raise an error ", verbose_name='allow geometry'), + ), + migrations.AddField( + model_name='objecttype', + name='contact_email', + field=models.CharField(blank=True, help_text='Email of the person in the organization who can provide information about the object type', max_length=200, verbose_name='contact email'), + ), + migrations.AddField( + model_name='objecttype', + name='contact_person', + field=models.CharField(blank=True, help_text='Name of the person in the organization who can provide information about the object type', max_length=200, verbose_name='contact person'), + ), + migrations.AddField( + model_name='objecttype', + name='created_at', + field=models.DateField(blank=True, help_text='Date when the object type was created', null=True, verbose_name='created at'), + ), + migrations.AddField( + model_name='objecttype', + name='data_classification', + field=models.CharField(choices=[('open', 'Open'), ('intern', 'Intern'), ('confidential', 'Confidential'), ('strictly_confidential', 'Strictly confidential')], default='open', help_text='Confidential level of the object type', max_length=50, verbose_name='data classification'), + ), + migrations.AddField( + model_name='objecttype', + name='description', + field=models.CharField(blank=True, help_text='The description of the object type', max_length=1000, verbose_name='description'), + ), + migrations.AddField( + model_name='objecttype', + name='documentation_url', + field=models.URLField(blank=True, help_text='Link to the documentation for the object type', verbose_name='documentation url'), + ), + migrations.AddField( + model_name='objecttype', + name='labels', + field=models.JSONField(blank=True, default=dict, help_text='Key-value pairs of keywords related for the object type', verbose_name='labels'), + ), + migrations.AddField( + model_name='objecttype', + name='linkable_to_zaken', + field=models.BooleanField(default=False, help_text='Objects of this type can have a link to 1 or more Zaken.\nTrue indicates the lifetime of the object is linked to the lifetime of linked zaken, i.e., when all linked Zaken to an object are archived/destroyed, the object will also be archived/destroyed.', verbose_name='linkable to zaken'), + ), + migrations.AddField( + model_name='objecttype', + name='maintainer_department', + field=models.CharField(blank=True, help_text='Business department which is responsible for the object type', max_length=200, verbose_name='maintainer department'), + ), + migrations.AddField( + model_name='objecttype', + name='maintainer_organization', + field=models.CharField(blank=True, help_text='Organization which is responsible for the object type', max_length=200, verbose_name='maintainer organization'), + ), + migrations.AddField( + model_name='objecttype', + name='modified_at', + field=models.DateField(blank=True, help_text='Last date when the object type was modified', null=True, verbose_name='modified at'), + ), + migrations.AddField( + model_name='objecttype', + name='name', + field=models.CharField(blank=True, help_text='Name of the object type', max_length=100, verbose_name='name'), + ), + migrations.AddField( + model_name='objecttype', + name='name_plural', + field=models.CharField(blank=True, help_text='Plural name of the object type', max_length=100, verbose_name='name plural'), + ), + migrations.AddField( + model_name='objecttype', + name='provider_organization', + field=models.CharField(blank=True, help_text='Organization which is responsible for publication of the object type', max_length=200, verbose_name='provider organization'), + ), + migrations.AddField( + model_name='objecttype', + name='source', + field=models.CharField(blank=True, help_text='Name of the system from which the object type originates', max_length=200, verbose_name='source'), + ), + migrations.AddField( + model_name='objecttype', + name='update_frequency', + field=models.CharField(choices=[('real_time', 'Real-time'), ('hourly', 'Hourly'), ('daily', 'Daily'), ('weekly', 'Weekly'), ('monthly', 'Monthly'), ('yearly', 'Yearly'), ('unknown', 'Unknown')], default='unknown', help_text='Indicates how often the object type is updated', max_length=10, verbose_name='update frequency'), + ), + migrations.CreateModel( + name='ObjectTypeVersion', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('version', models.PositiveSmallIntegerField(help_text='Integer version of the OBJECTTYPE', verbose_name='version')), + ('created_at', models.DateField(blank=True, help_text='Date when the version was created', null=True, verbose_name='created at')), + ('modified_at', models.DateField(blank=True, help_text='Last date when the version was modified', null=True, verbose_name='modified at')), + ('published_at', models.DateField(blank=True, help_text='Date when the version was published', null=True, verbose_name='published_at')), + ('json_schema', models.JSONField(default=dict, help_text='JSON schema for Object validation', verbose_name='JSON schema')), + ('status', models.CharField(choices=[('published', 'Published'), ('draft', 'Draft'), ('deprecated', 'Deprecated')], default='draft', help_text='Status of the object type version', max_length=20, verbose_name='status')), + ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='core.objecttype')), + ], + options={ + 'unique_together': {('object_type', 'version')}, + }, + ), + ] diff --git a/src/objects/core/models.py b/src/objects/core/models.py index 45cb057f..197600b0 100644 --- a/src/objects/core/models.py +++ b/src/objects/core/models.py @@ -15,8 +15,13 @@ from objects.utils.client import get_objecttypes_client +from .constants import ( + DataClassificationChoices, + ObjectVersionStatus, + UpdateFrequencyChoices, +) from .query import ObjectQuerySet, ObjectRecordQuerySet, ObjectTypeQuerySet -from .utils import check_objecttype_cached +from .utils import check_json_schema, check_objecttype_cached class ObjectType(models.Model): @@ -31,6 +36,128 @@ class ObjectType(models.Model): help_text=_("Cached name of the objecttype retrieved from the Objecttype API"), ) + name = models.CharField( + _("name"), + max_length=100, + help_text=_("Name of the object type"), + blank=True, # TODO temp + ) + + name_plural = models.CharField( + _("name plural"), + max_length=100, + help_text=_("Plural name of the object type"), + blank=True, # TODO temp + ) + description = models.CharField( + _("description"), + max_length=1000, + blank=True, + help_text=_("The description of the object type"), + ) + data_classification = models.CharField( + _("data classification"), + max_length=50, + choices=DataClassificationChoices.choices, + default=DataClassificationChoices.open, + help_text=_("Confidential level of the object type"), + ) + maintainer_organization = models.CharField( + _("maintainer organization"), + max_length=200, + blank=True, + help_text=_("Organization which is responsible for the object type"), + ) + maintainer_department = models.CharField( + _("maintainer department"), + max_length=200, + blank=True, + help_text=_("Business department which is responsible for the object type"), + ) + contact_person = models.CharField( + _("contact person"), + max_length=200, + blank=True, + help_text=_( + "Name of the person in the organization who can provide information about the object type" + ), + ) + contact_email = models.CharField( + _("contact email"), + max_length=200, + blank=True, + help_text=_( + "Email of the person in the organization who can provide information about the object type" + ), + ) + source = models.CharField( + _("source"), + max_length=200, + blank=True, + help_text=_("Name of the system from which the object type originates"), + ) + update_frequency = models.CharField( + _("update frequency"), + max_length=10, + choices=UpdateFrequencyChoices.choices, + default=UpdateFrequencyChoices.unknown, + help_text=_("Indicates how often the object type is updated"), + ) + provider_organization = models.CharField( + _("provider organization"), + max_length=200, + blank=True, + help_text=_( + "Organization which is responsible for publication of the object type" + ), + ) + documentation_url = models.URLField( + _("documentation url"), + blank=True, + help_text=_("Link to the documentation for the object type"), + ) + labels = models.JSONField( + _("labels"), + help_text=_("Key-value pairs of keywords related for the object type"), + default=dict, + blank=True, + ) + created_at = models.DateField( + _("created at"), + auto_now_add=False, # TODO temp + blank=True, # TODO temp + null=True, # TODO temp + help_text=_("Date when the object type was created"), + ) + modified_at = models.DateField( + _("modified at"), + auto_now=False, # TODO temp + blank=True, # TODO temp + null=True, # TODO temp + help_text=_("Last date when the object type was modified"), + ) + allow_geometry = models.BooleanField( + _("allow geometry"), + default=True, + help_text=_( + "Shows whether the related objects can have geographic coordinates. " + "If the value is 'false' the related objects are not allowed to " + "have coordinates and the creation/update of objects with " + "`geometry` property will raise an error " + ), + ) + linkable_to_zaken = models.BooleanField( + _("linkable to zaken"), + default=False, + help_text=_( + # TODO Document: how and where these links should be created/maintained + "Objects of this type can have a link to 1 or more Zaken.\n" + "True indicates the lifetime of the object is linked to the lifetime " + "of linked zaken, i.e., when all linked Zaken to an object are " + "archived/destroyed, the object will also be archived/destroyed." + ), + ) + objects = ObjectTypeQuerySet.as_manager() class Meta: @@ -66,6 +193,86 @@ def clean_fields(self, exclude: Iterable[str] | None = None) -> None: self._name = object_type_data["name"] +class ObjectTypeVersion(models.Model): + object_type = models.ForeignKey( + ObjectType, on_delete=models.CASCADE, related_name="versions" + ) + version = models.PositiveSmallIntegerField( + _("version"), help_text=_("Integer version of the OBJECTTYPE") + ) + created_at = models.DateField( + _("created at"), + auto_now_add=False, # TODO temp + blank=True, # TODO temp + null=True, # TODO temp + help_text=_("Date when the version was created"), + ) + modified_at = models.DateField( + _("modified at"), + auto_now=False, # TODO temp + blank=True, # TODO temp + null=True, # TODO temp + help_text=_("Last date when the version was modified"), + ) + published_at = models.DateField( + _("published_at"), + null=True, + blank=True, + help_text=_("Date when the version was published"), + ) + json_schema = models.JSONField( + _("JSON schema"), help_text=_("JSON schema for Object validation"), default=dict + ) + status = models.CharField( + _("status"), + max_length=20, + choices=ObjectVersionStatus.choices, + default=ObjectVersionStatus.draft, + help_text=_("Status of the object type version"), + ) + + class Meta: + unique_together = ("object_type", "version") + + def __str__(self): + return f"{self.object_type} v.{self.version}" + + def clean(self): + super().clean() + + check_json_schema(self.json_schema) + + def save(self, *args, **kwargs): + if not self.version: + self.version = self.generate_version_number() + + # save published_at + previous_status = ( + ObjectTypeVersion.objects.get(id=self.id).status if self.id else None + ) + if ( + self.status == ObjectVersionStatus.published + and previous_status != self.status + ): + self.published_at = datetime.date.today() + + super().save(*args, **kwargs) + + def generate_version_number(self) -> int: + existed_versions = ObjectTypeVersion.objects.filter( + object_type=self.object_type + ) + + max_version = 0 + if existed_versions.exists(): + max_version = existed_versions.aggregate(models.Max("version"))[ + "version__max" + ] + + version_number = max_version + 1 + return version_number + + class Object(models.Model): uuid = models.UUIDField( unique=True, default=uuid.uuid4, help_text=_("Unique identifier (UUID4)") diff --git a/src/objects/core/tests/factories.py b/src/objects/core/tests/factories.py index 685819be..427bce95 100644 --- a/src/objects/core/tests/factories.py +++ b/src/objects/core/tests/factories.py @@ -8,7 +8,7 @@ from factory.fuzzy import BaseFuzzyAttribute from zgw_consumers.test.factories import ServiceFactory -from ..models import Object, ObjectRecord, ObjectType +from ..models import Object, ObjectRecord, ObjectType, ObjectTypeVersion class ObjectTypeFactory(factory.django.DjangoModelFactory): @@ -16,10 +16,28 @@ class ObjectTypeFactory(factory.django.DjangoModelFactory): uuid = factory.LazyFunction(uuid.uuid4) _name = factory.Faker("word") + name = factory.Faker("word") + name_plural = factory.LazyAttribute(lambda x: f"{x.name}s") + description = factory.Faker("bs") + class Meta: model = ObjectType +class ObjectTypeVersionFactory(factory.django.DjangoModelFactory): + object_type = factory.SubFactory(ObjectTypeFactory) + json_schema = { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["diameter"], + "properties": {"diameter": {"type": "integer", "description": "size in cm."}}, + } + + class Meta: + model = ObjectTypeVersion + + class FuzzyPoint(BaseFuzzyAttribute): def fuzz(self): return Point(random.uniform(-180.0, 180.0), random.uniform(-90.0, 90.0)) diff --git a/src/objects/core/tests/test_import_objecttypes.py b/src/objects/core/tests/test_import_objecttypes.py new file mode 100644 index 00000000..2ece6cd5 --- /dev/null +++ b/src/objects/core/tests/test_import_objecttypes.py @@ -0,0 +1,157 @@ +import uuid + +from django.core.management import CommandError, call_command +from django.test import TestCase + +import requests_mock +from zgw_consumers.models import Service + +from objects.core.models import ObjectType, ObjectTypeVersion +from objects.core.tests.factories import ObjectTypeFactory +from objects.tests.utils import ( + mock_objecttype_versions, + mock_objecttypes, +) + + +class TestImportObjectTypesCommand(TestCase): + def setUp(self): + super().setUp() + self.url = "http://127.0.0.1:8000/api/v2/" + + self.m = requests_mock.Mocker() + self.m.start() + + self.service = Service.objects.create(api_root=self.url, slug="objecttypes-api") + self.m.head(self.url, status_code=200, headers={"api-version": "2.2.2"}) + + def tearDown(self): + self.m.stop() + + def _call_command(self): + call_command("import_objecttypes", self.service.slug) + + def test_api_version_is_required(self): + self.m.head(self.url, status_code=200) + + with self.assertRaisesMessage( + CommandError, "API version must be 2.2.2 or higher" + ): + self._call_command() + + def test_api_version_must_be_greater_than_constant(self): + self.m.head(self.url, status_code=200, headers={"api-version": "2.2.1"}) + + with self.assertRaisesMessage( + CommandError, "API version must be 2.2.2 or higher" + ): + self._call_command() + + def test_command_fails_if_http_error(self): + self.m.get(f"{self.url}objecttypes", status_code=404) + with self.assertRaises(CommandError): + self._call_command() + + def test_new_objecttypes_are_created(self): + self.assertEqual(ObjectType.objects.count(), 0) + self.assertEqual(ObjectTypeVersion.objects.count(), 0) + + uuid1 = str(uuid.uuid4()) + uuid2 = str(uuid.uuid4()) + + self.m.get(f"{self.url}objecttypes", json=mock_objecttypes(uuid1, uuid2)) + self.m.get( + f"{self.url}objecttypes/{uuid1}/versions", + json=mock_objecttype_versions(uuid1), + ) + self.m.get( + f"{self.url}objecttypes/{uuid2}/versions", + json=mock_objecttype_versions(uuid2), + ) + + self._call_command() + + self.assertEqual(ObjectType.objects.count(), 2) + + objecttype = ObjectType.objects.get(uuid=uuid1) + self.assertEqual(objecttype.name, "Melding") + self.assertEqual(objecttype.name_plural, "Meldingen") + self.assertEqual(objecttype.description, "") + self.assertEqual(objecttype.data_classification, "intern") + self.assertEqual(objecttype.maintainer_organization, "Dimpact") + self.assertEqual(objecttype.maintainer_department, "") + self.assertEqual(objecttype.contact_person, "Ad Alarm") + self.assertEqual(objecttype.contact_email, "") + self.assertEqual(objecttype.source, "") + self.assertEqual(objecttype.update_frequency, "unknown") + self.assertEqual(objecttype.provider_organization, "") + self.assertEqual(objecttype.documentation_url, "") + self.assertEqual(objecttype.labels, {}) + self.assertEqual(objecttype.linkable_to_zaken, False) + self.assertEqual(str(objecttype.created_at), "2020-12-01") + self.assertEqual(str(objecttype.modified_at), "2020-12-01") + self.assertEqual(objecttype.allow_geometry, True) + + self.assertEqual(ObjectTypeVersion.objects.count(), 4) + + version = ObjectTypeVersion.objects.get(object_type=objecttype, version=1) + self.assertEqual(str(version.created_at), "2020-12-01") + self.assertEqual(str(version.modified_at), "2020-12-01") + self.assertEqual(str(version.published_at), "2020-10-02") + self.assertEqual( + version.json_schema, + { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + ) + self.assertEqual(str(version.status), "published") + + def test_existing_objecttypes_are_updated(self): + objecttype1 = ObjectTypeFactory(service=self.service) + objecttype2 = ObjectTypeFactory(service=self.service) + + self.m.get( + f"{self.url}objecttypes", + json=mock_objecttypes(str(objecttype1.uuid), str(objecttype2.uuid)), + ) + self.m.get( + f"{self.url}objecttypes/{str(objecttype1.uuid)}/versions", + json=mock_objecttype_versions(str(objecttype1.uuid)), + ) + self.m.get( + f"{self.url}objecttypes/{str(objecttype2.uuid)}/versions", + json=mock_objecttype_versions(str(objecttype2.uuid)), + ) + + self._call_command() + self.assertEqual(ObjectType.objects.count(), 2) + self.assertEqual(ObjectTypeVersion.objects.count(), 4) + + objecttype = ObjectType.objects.get(uuid=objecttype1.uuid) + self.assertEqual(objecttype.name, "Melding") + + version = ObjectTypeVersion.objects.get(object_type=objecttype, version=1) + self.assertEqual( + version.json_schema, + { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + ) diff --git a/src/objects/core/tests/test_objecttypeversion_generate.py b/src/objects/core/tests/test_objecttypeversion_generate.py new file mode 100644 index 00000000..49a7c52f --- /dev/null +++ b/src/objects/core/tests/test_objecttypeversion_generate.py @@ -0,0 +1,33 @@ +from django.test import TestCase + +from objects.core.models import ObjectTypeVersion +from objects.core.tests.factories import ObjectTypeFactory, ObjectTypeVersionFactory + +JSON_SCHEMA = { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["diameter"], + "properties": {"diameter": {"type": "integer", "description": "size in cm."}}, +} + + +class GenerateVersionTests(TestCase): + def test_generate_version_for_new_objecttype(self): + object_type = ObjectTypeFactory.create() + + object_version = ObjectTypeVersion.objects.create( + json_schema=JSON_SCHEMA, object_type=object_type + ) + + self.assertEqual(object_version.version, 1) + + def test_generate_version_for_objecttype_with_existed_version(self): + object_type = ObjectTypeFactory.create() + ObjectTypeVersionFactory.create(object_type=object_type, version=1) + + object_version = ObjectTypeVersion.objects.create( + json_schema=JSON_SCHEMA, object_type=object_type + ) + + self.assertEqual(object_version.version, 2) diff --git a/src/objects/core/utils.py b/src/objects/core/utils.py index ff6e65ca..3879d711 100644 --- a/src/objects/core/utils.py +++ b/src/objects/core/utils.py @@ -3,6 +3,8 @@ import jsonschema import requests +from jsonschema.exceptions import SchemaError +from jsonschema.validators import validator_for from objects.core import models from objects.utils.cache import cache @@ -27,8 +29,8 @@ def get_objecttype_version_response(): ) try: - vesion_data = get_objecttype_version_response() - jsonschema.validate(data, vesion_data["jsonSchema"]) + version_data = get_objecttype_version_response() + jsonschema.validate(data, version_data["jsonSchema"]) except KeyError: raise ValidationError( f"{object_type.versions_url} does not appear to be a valid objecttype.", @@ -49,3 +51,11 @@ def can_connect_to_objecttypes() -> bool: if not client.can_connect: return False return True + + +def check_json_schema(json_schema: dict): + schema_validator = validator_for(json_schema) + try: + schema_validator.check_schema(json_schema) + except SchemaError as exc: + raise ValidationError(exc.args[0]) from exc diff --git a/src/objects/tests/utils.py b/src/objects/tests/utils.py index 9f7043f8..9004724f 100644 --- a/src/objects/tests/utils.py +++ b/src/objects/tests/utils.py @@ -55,6 +55,118 @@ def mock_objecttype(url: str, attrs=None) -> dict: return response +def mock_objecttypes(uuid1, uuid2): + return { + "count": 3, + "next": None, + "previous": None, + "results": [ + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}", + "uuid": uuid1, + "name": "Melding", + "namePlural": "Meldingen", + "description": "", + "dataClassification": "intern", + "maintainerOrganization": "Dimpact", + "maintainerDepartment": "", + "contactPerson": "Ad Alarm", + "contactEmail": "", + "source": "", + "updateFrequency": "unknown", + "providerOrganization": "", + "documentationUrl": "", + "labels": {}, + "linkableToZaken": False, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "allowGeometry": True, + "versions": [ + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}/versions/1", + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}/versions/2", + ], + }, + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}", + "uuid": uuid2, + "name": "Straatverlichting", + "namePlural": "Straatverlichting", + "description": "", + "dataClassification": "open", + "maintainerOrganization": "Maykin Media", + "maintainerDepartment": "", + "contactPerson": "Desiree Lumen", + "contactEmail": "", + "source": "", + "updateFrequency": "unknown", + "providerOrganization": "", + "documentationUrl": "", + "labels": {}, + "linkableToZaken": False, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "allowGeometry": True, + "versions": [ + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}/versions/1", + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}/versions/2", + ], + }, + ], + } + + +def mock_objecttype_versions(objecttype_uuid: str): + return { + "count": 3, + "next": None, + "previous": None, + "results": [ + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}/versions/2", + "version": 2, + "objectType": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}", + "status": "published", + "jsonSchema": { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + "createdAt": "2020-11-12", + "modifiedAt": "2020-11-27", + "publishedAt": "2020-11-27", + }, + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}/versions/1", + "version": 1, + "objectType": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}", + "status": "published", + "jsonSchema": { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "publishedAt": "2020-10-02", + }, + ], + } + + def mock_objecttype_version(url: str, attrs=None) -> dict: attrs = attrs or {} response = { diff --git a/src/objects/utils/client.py b/src/objects/utils/client.py index e7989a57..880e3d11 100644 --- a/src/objects/utils/client.py +++ b/src/objects/utils/client.py @@ -80,6 +80,11 @@ def get_objecttype_version( response.raise_for_status() return response.json() + def get_objecttypes_api_version(self) -> str | None: + response = self.head("") + response.raise_for_status() + return response.headers.get("api-version") + def get_objecttypes_client(service) -> ObjecttypesClient: assert service is not None From b5090072d7713d0c003b67b103e443920db2097c Mon Sep 17 00:00:00 2001 From: floris272 Date: Fri, 5 Dec 2025 14:57:00 +0100 Subject: [PATCH 2/7] :sparkles: [#564] add is_imported to objecttype --- .../management/commands/import_objecttypes.py | 2 ++ .../migrations/0036_objecttype_is_imported.py | 18 ++++++++++++++++++ src/objects/core/models.py | 6 ++++++ .../core/tests/test_import_objecttypes.py | 2 ++ 4 files changed, 28 insertions(+) create mode 100644 src/objects/core/migrations/0036_objecttype_is_imported.py diff --git a/src/objects/core/management/commands/import_objecttypes.py b/src/objects/core/management/commands/import_objecttypes.py index e5c9ee39..40ea4e6b 100644 --- a/src/objects/core/management/commands/import_objecttypes.py +++ b/src/objects/core/management/commands/import_objecttypes.py @@ -75,6 +75,7 @@ def _bulk_create_or_update_objecttypes(self, data): update_conflicts=True, unique_fields=["uuid", "service"], # TODO remove service update_fields=[ + "is_imported", "name", "name_plural", "description", @@ -120,6 +121,7 @@ def _parse_objecttype_data( objecttype.pop("versions") objecttype.pop("url") objecttype["service"] = service + objecttype["is_imported"] = True data.append(ObjectType(**underscoreize(objecttype))) return data diff --git a/src/objects/core/migrations/0036_objecttype_is_imported.py b/src/objects/core/migrations/0036_objecttype_is_imported.py new file mode 100644 index 00000000..784b3b8d --- /dev/null +++ b/src/objects/core/migrations/0036_objecttype_is_imported.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.7 on 2025-12-05 13:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0035_objecttype_allow_geometry_objecttype_contact_email_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='objecttype', + name='is_imported', + field=models.BooleanField(default=False, editable=False, verbose_name='Is imported'), + ), + ] diff --git a/src/objects/core/models.py b/src/objects/core/models.py index 197600b0..f025ad1e 100644 --- a/src/objects/core/models.py +++ b/src/objects/core/models.py @@ -36,6 +36,12 @@ class ObjectType(models.Model): help_text=_("Cached name of the objecttype retrieved from the Objecttype API"), ) + is_imported = models.BooleanField( + _("Is imported"), + default=False, + editable=False, + ) # TODO temp + name = models.CharField( _("name"), max_length=100, diff --git a/src/objects/core/tests/test_import_objecttypes.py b/src/objects/core/tests/test_import_objecttypes.py index 2ece6cd5..899b89d3 100644 --- a/src/objects/core/tests/test_import_objecttypes.py +++ b/src/objects/core/tests/test_import_objecttypes.py @@ -74,6 +74,7 @@ def test_new_objecttypes_are_created(self): self.assertEqual(ObjectType.objects.count(), 2) objecttype = ObjectType.objects.get(uuid=uuid1) + self.assertEqual(objecttype.is_imported, True) self.assertEqual(objecttype.name, "Melding") self.assertEqual(objecttype.name_plural, "Meldingen") self.assertEqual(objecttype.description, "") @@ -137,6 +138,7 @@ def test_existing_objecttypes_are_updated(self): self.assertEqual(ObjectTypeVersion.objects.count(), 4) objecttype = ObjectType.objects.get(uuid=objecttype1.uuid) + self.assertEqual(objecttype.is_imported, True) self.assertEqual(objecttype.name, "Melding") version = ObjectTypeVersion.objects.get(object_type=objecttype, version=1) From bd75798e96b08087aa57e50a4e55f92e9d12e7f7 Mon Sep 17 00:00:00 2001 From: floris272 Date: Tue, 16 Dec 2025 12:47:02 +0100 Subject: [PATCH 3/7] :memo: [#564] add migration documentation --- docs/manual/index.rst | 1 + docs/manual/migration.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 docs/manual/migration.rst diff --git a/docs/manual/index.rst b/docs/manual/index.rst index d00424a4..c9bcbcc5 100644 --- a/docs/manual/index.rst +++ b/docs/manual/index.rst @@ -8,3 +8,4 @@ Manual :caption: Further reading scripts + migration diff --git a/docs/manual/migration.rst b/docs/manual/migration.rst new file mode 100644 index 00000000..2ae61a3c --- /dev/null +++ b/docs/manual/migration.rst @@ -0,0 +1,15 @@ +.. _objecttype_migration + +ObjectType API migration +======================== + +In 4.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. + +Importing objecttype data +------------------------ +Before updating to 4.0 all objecttypes need to be imported. This can be done with the `import_objecttypes` command. +This command will fetch all objecttypes and their versions from an objecttype service based on its identifier/slug +and update existing objecttypes or create new ones if they have not been added to the objecttypes API. + +Please note that after the update the objecttypes API is still being used in Objects API <4.0 the command only fetches the data. +From 4.0 onwards it will use the imported objecttypes. From 0f90831959c7af3a2961eafa7539f5548b8ce75c Mon Sep 17 00:00:00 2001 From: floris272 Date: Tue, 16 Dec 2025 12:47:44 +0100 Subject: [PATCH 4/7] :recycle: [#564] make migration todo comments clearer --- .../management/commands/import_objecttypes.py | 23 +++++++++---- src/objects/core/models.py | 32 +++++++++---------- .../core/tests/test_import_objecttypes.py | 8 ++--- src/objects/tests/utils.py | 4 +-- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/objects/core/management/commands/import_objecttypes.py b/src/objects/core/management/commands/import_objecttypes.py index 40ea4e6b..f8a78547 100644 --- a/src/objects/core/management/commands/import_objecttypes.py +++ b/src/objects/core/management/commands/import_objecttypes.py @@ -6,20 +6,24 @@ from djangorestframework_camel_case.util import underscoreize from packaging.version import Version -from requests.exceptions import HTTPError +from requests.exceptions import RequestException from zgw_consumers.models import Service from objects.core.models import ObjectType, ObjectTypeVersion from objects.utils.client import get_objecttypes_client -MIN_OBJECTTYPES_VERSION = "2.2.2" +MIN_OBJECTTYPES_VERSION = "3.4.0" # added boolean field linkable_to_zaken to ObjectType class Command(BaseCommand): + help = ( + "Import ObjectTypes & ObjectTypeVersions from an Objecttypes API based on the service identifier.", + ) + def add_arguments(self, parser): parser.add_argument( "service_slug", - help=_("Slug of the service"), + help=_("Identifier/slug of Objecttypes API service"), ) @transaction.atomic @@ -49,8 +53,12 @@ def handle(self, *args, **options): % (len(data), objecttype.name) ) - except HTTPError as e: - raise CommandError(_("Request failed: {}").format(e)) + except RequestException as e: + raise CommandError( + _( + "Something went wrong while making requests to Objecttypes API: {}" + ).format(e) + ) def _get_service(self, slug): try: @@ -73,7 +81,10 @@ def _bulk_create_or_update_objecttypes(self, data): ObjectType.objects.bulk_create( data, update_conflicts=True, - unique_fields=["uuid", "service"], # TODO remove service + unique_fields=[ + "uuid", + "service", + ], # TODO remove service from unique_fields after objecttype migration since it will no longer be part of the ObjectType model. update_fields=[ "is_imported", "name", diff --git a/src/objects/core/models.py b/src/objects/core/models.py index f025ad1e..dd95aa62 100644 --- a/src/objects/core/models.py +++ b/src/objects/core/models.py @@ -34,26 +34,26 @@ class ObjectType(models.Model): _name = models.CharField( max_length=100, help_text=_("Cached name of the objecttype retrieved from the Objecttype API"), - ) + ) # TODO can be removed after objecttype migration is_imported = models.BooleanField( _("Is imported"), default=False, editable=False, - ) # TODO temp + ) # TODO temp field to track if object was imported, can be removed after objecttype migration name = models.CharField( _("name"), max_length=100, help_text=_("Name of the object type"), - blank=True, # TODO temp + blank=True, # TODO blank=False after objecttype migration ) name_plural = models.CharField( _("name plural"), max_length=100, help_text=_("Plural name of the object type"), - blank=True, # TODO temp + blank=True, # TODO blank=False after objecttype migration ) description = models.CharField( _("description"), @@ -130,16 +130,16 @@ class ObjectType(models.Model): ) created_at = models.DateField( _("created at"), - auto_now_add=False, # TODO temp - blank=True, # TODO temp - null=True, # TODO temp + auto_now_add=False, # TODO auto_now_add=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration help_text=_("Date when the object type was created"), ) modified_at = models.DateField( _("modified at"), - auto_now=False, # TODO temp - blank=True, # TODO temp - null=True, # TODO temp + auto_now=False, # TODO auto_now=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration help_text=_("Last date when the object type was modified"), ) allow_geometry = models.BooleanField( @@ -208,16 +208,16 @@ class ObjectTypeVersion(models.Model): ) created_at = models.DateField( _("created at"), - auto_now_add=False, # TODO temp - blank=True, # TODO temp - null=True, # TODO temp + auto_now_add=False, # TODO auto_now_add=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration help_text=_("Date when the version was created"), ) modified_at = models.DateField( _("modified at"), - auto_now=False, # TODO temp - blank=True, # TODO temp - null=True, # TODO temp + auto_now=False, # TODO auto_now=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration help_text=_("Last date when the version was modified"), ) published_at = models.DateField( diff --git a/src/objects/core/tests/test_import_objecttypes.py b/src/objects/core/tests/test_import_objecttypes.py index 899b89d3..d10c1779 100644 --- a/src/objects/core/tests/test_import_objecttypes.py +++ b/src/objects/core/tests/test_import_objecttypes.py @@ -23,7 +23,7 @@ def setUp(self): self.m.start() self.service = Service.objects.create(api_root=self.url, slug="objecttypes-api") - self.m.head(self.url, status_code=200, headers={"api-version": "2.2.2"}) + self.m.head(self.url, status_code=200, headers={"api-version": "3.4.0"}) def tearDown(self): self.m.stop() @@ -35,15 +35,15 @@ def test_api_version_is_required(self): self.m.head(self.url, status_code=200) with self.assertRaisesMessage( - CommandError, "API version must be 2.2.2 or higher" + CommandError, "API version must be 3.4.0 or higher" ): self._call_command() def test_api_version_must_be_greater_than_constant(self): - self.m.head(self.url, status_code=200, headers={"api-version": "2.2.1"}) + self.m.head(self.url, status_code=200, headers={"api-version": "3.2.0"}) with self.assertRaisesMessage( - CommandError, "API version must be 2.2.2 or higher" + CommandError, "API version must be 3.4.0 or higher" ): self._call_command() diff --git a/src/objects/tests/utils.py b/src/objects/tests/utils.py index 9004724f..1ea4ae4e 100644 --- a/src/objects/tests/utils.py +++ b/src/objects/tests/utils.py @@ -57,7 +57,7 @@ def mock_objecttype(url: str, attrs=None) -> dict: def mock_objecttypes(uuid1, uuid2): return { - "count": 3, + "count": 2, "next": None, "previous": None, "results": [ @@ -117,7 +117,7 @@ def mock_objecttypes(uuid1, uuid2): def mock_objecttype_versions(objecttype_uuid: str): return { - "count": 3, + "count": 2, "next": None, "previous": None, "results": [ From 459b1747ef5e9914cb6fde323204409f0a2fa7a0 Mon Sep 17 00:00:00 2001 From: floris272 Date: Tue, 16 Dec 2025 12:53:01 +0100 Subject: [PATCH 5/7] :memo: [#564] fix migration documentation --- docs/manual/migration.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/manual/migration.rst b/docs/manual/migration.rst index 2ae61a3c..838830ba 100644 --- a/docs/manual/migration.rst +++ b/docs/manual/migration.rst @@ -1,15 +1,15 @@ -.. _objecttype_migration +.. _objecttype_migration: ObjectType API migration ======================== -In 4.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. +In 4.0.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. Importing objecttype data ------------------------- -Before updating to 4.0 all objecttypes need to be imported. This can be done with the `import_objecttypes` command. +------------------------- +Before updating to 4.0.0 all objecttypes need to be imported. This can be done with the `import_objecttypes` command. This command will fetch all objecttypes and their versions from an objecttype service based on its identifier/slug and update existing objecttypes or create new ones if they have not been added to the objecttypes API. -Please note that after the update the objecttypes API is still being used in Objects API <4.0 the command only fetches the data. -From 4.0 onwards it will use the imported objecttypes. +Please note that after the update the objecttypes API is still being used in Objects API <4.0.0 the command only fetches the data. +From 4.0.0 onwards it will use the imported objecttypes. From 156447e3f59be42bb2bec28aad6e8a87af004e9b Mon Sep 17 00:00:00 2001 From: floris272 Date: Tue, 16 Dec 2025 16:07:54 +0100 Subject: [PATCH 6/7] :memo: [#564] update migration documentation --- docs/manual/migration.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/manual/migration.rst b/docs/manual/migration.rst index 838830ba..0df78046 100644 --- a/docs/manual/migration.rst +++ b/docs/manual/migration.rst @@ -1,15 +1,19 @@ .. _objecttype_migration: -ObjectType API migration -======================== +ObjectTypes API migration +========================= -In 4.0.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. +In version 4.0.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. This means that from version 4.0.0 only objecttypes that exist in the database are supported, and no external objecttypes can be used. Importing objecttype data ------------------------- -Before updating to 4.0.0 all objecttypes need to be imported. This can be done with the `import_objecttypes` command. -This command will fetch all objecttypes and their versions from an objecttype service based on its identifier/slug +Before updating to 4.0.0 all objecttypes from the ObjectTypes API instance need to be imported. This can be done with the ``import_objecttypes`` command that can be executed from the Objects container. +This command will fetch all objecttypes and their versions from an objecttype service based on its identifier/slug (which can be found in the admin interface under ``Configuration > Services``) and update existing objecttypes or create new ones if they have not been added to the objecttypes API. -Please note that after the update the objecttypes API is still being used in Objects API <4.0.0 the command only fetches the data. +.. code-block:: bash + + src/manage.py import_objecttypes objecttypes-api + +Please note that after the update the objecttypes API is still being used in Objects API version <4.0.0, the command only fetches and imports the data. From 4.0.0 onwards it will use the imported objecttypes. From 1305ff0ceea95a1132bb98080f9b04c61e97e70d Mon Sep 17 00:00:00 2001 From: floris272 Date: Tue, 16 Dec 2025 16:11:26 +0100 Subject: [PATCH 7/7] :memo: [#564] fix import objecttypes command help text --- src/objects/core/management/commands/import_objecttypes.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/objects/core/management/commands/import_objecttypes.py b/src/objects/core/management/commands/import_objecttypes.py index f8a78547..d9929a79 100644 --- a/src/objects/core/management/commands/import_objecttypes.py +++ b/src/objects/core/management/commands/import_objecttypes.py @@ -16,9 +16,7 @@ class Command(BaseCommand): - help = ( - "Import ObjectTypes & ObjectTypeVersions from an Objecttypes API based on the service identifier.", - ) + help = "Import ObjectTypes & ObjectTypeVersions from an Objecttypes API based on the service identifier." def add_arguments(self, parser): parser.add_argument( @@ -80,7 +78,7 @@ def _check_objecttypes_api_version(self, client): def _bulk_create_or_update_objecttypes(self, data): ObjectType.objects.bulk_create( data, - update_conflicts=True, + update_conflicts=True, # Updates existing Objecttypes based on unique_fields unique_fields=[ "uuid", "service",