diff --git a/docs/manual/index.rst b/docs/manual/index.rst index d00424a4..c9bcbcc5 100644 --- a/docs/manual/index.rst +++ b/docs/manual/index.rst @@ -8,3 +8,4 @@ Manual :caption: Further reading scripts + migration diff --git a/docs/manual/migration.rst b/docs/manual/migration.rst new file mode 100644 index 00000000..0df78046 --- /dev/null +++ b/docs/manual/migration.rst @@ -0,0 +1,19 @@ +.. _objecttype_migration: + +ObjectTypes API migration +========================= + +In version 4.0.0 the ObjectTypes API will be merged into the Objects API so that only one application is needed. This means that from version 4.0.0 only objecttypes that exist in the database are supported, and no external objecttypes can be used. + +Importing objecttype data +------------------------- +Before updating to 4.0.0 all objecttypes from the ObjectTypes API instance need to be imported. This can be done with the ``import_objecttypes`` command that can be executed from the Objects container. +This command will fetch all objecttypes and their versions from an objecttype service based on its identifier/slug (which can be found in the admin interface under ``Configuration > Services``) +and update existing objecttypes or create new ones if they have not been added to the objecttypes API. + +.. code-block:: bash + + src/manage.py import_objecttypes objecttypes-api + +Please note that after the update the objecttypes API is still being used in Objects API version <4.0.0, the command only fetches and imports the data. +From 4.0.0 onwards it will use the imported objecttypes. diff --git a/src/objects/core/constants.py b/src/objects/core/constants.py new file mode 100644 index 00000000..d795b06f --- /dev/null +++ b/src/objects/core/constants.py @@ -0,0 +1,25 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class ObjectVersionStatus(models.TextChoices): + published = "published", _("Published") + draft = "draft", _("Draft") + deprecated = "deprecated", _("Deprecated") + + +class DataClassificationChoices(models.TextChoices): + open = "open", _("Open") + intern = "intern", _("Intern") + confidential = "confidential", _("Confidential") + strictly_confidential = "strictly_confidential", _("Strictly confidential") + + +class UpdateFrequencyChoices(models.TextChoices): + real_time = "real_time", _("Real-time") + hourly = "hourly", _("Hourly") + daily = "daily", _("Daily") + weekly = "weekly", _("Weekly") + monthly = "monthly", _("Monthly") + yearly = "yearly", _("Yearly") + unknown = "unknown", _("Unknown") diff --git a/src/objects/core/management/__init__.py b/src/objects/core/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/objects/core/management/commands/__init__.py b/src/objects/core/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/objects/core/management/commands/import_objecttypes.py b/src/objects/core/management/commands/import_objecttypes.py new file mode 100644 index 00000000..d9929a79 --- /dev/null +++ b/src/objects/core/management/commands/import_objecttypes.py @@ -0,0 +1,145 @@ +from typing import Any + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from django.utils.translation import gettext as _ + +from djangorestframework_camel_case.util import underscoreize +from packaging.version import Version +from requests.exceptions import RequestException +from zgw_consumers.models import Service + +from objects.core.models import ObjectType, ObjectTypeVersion +from objects.utils.client import get_objecttypes_client + +MIN_OBJECTTYPES_VERSION = "3.4.0" # added boolean field linkable_to_zaken to ObjectType + + +class Command(BaseCommand): + help = "Import ObjectTypes & ObjectTypeVersions from an Objecttypes API based on the service identifier." + + def add_arguments(self, parser): + parser.add_argument( + "service_slug", + help=_("Identifier/slug of Objecttypes API service"), + ) + + @transaction.atomic + def handle(self, *args, **options): + service_slug = options["service_slug"] + service = self._get_service(service_slug) + + with get_objecttypes_client(service) as client: + try: + self._check_objecttypes_api_version(client) + + objecttypes = client.list_objecttypes() + data = self._parse_objecttype_data(objecttypes, service) + self._bulk_create_or_update_objecttypes(data) + self.stdout.write("Successfully imported %s objecttypes" % len(data)) + + for objecttype in data: + objecttype_versions = client.list_objecttype_versions( + objecttype.uuid + ) + data = self._parse_objectversion_data( + objecttype_versions, objecttype + ) + self._bulk_create_or_update_objecttype_versions(data) + self.stdout.write( + "Successfully imported %s versions for type: %s" + % (len(data), objecttype.name) + ) + + except RequestException as e: + raise CommandError( + _( + "Something went wrong while making requests to Objecttypes API: {}" + ).format(e) + ) + + def _get_service(self, slug): + try: + return Service.objects.get(slug=slug) + except Service.DoesNotExist: + raise CommandError(_("Service '{}' does not exist").format(slug)) + + def _check_objecttypes_api_version(self, client): + api_version = client.get_objecttypes_api_version() + if api_version is None or Version( + client.get_objecttypes_api_version() + ) < Version(MIN_OBJECTTYPES_VERSION): + raise CommandError( + _("Object types API version must be {} or higher.").format( + MIN_OBJECTTYPES_VERSION + ) + ) + + def _bulk_create_or_update_objecttypes(self, data): + ObjectType.objects.bulk_create( + data, + update_conflicts=True, # Updates existing Objecttypes based on unique_fields + unique_fields=[ + "uuid", + "service", + ], # TODO remove service from unique_fields after objecttype migration since it will no longer be part of the ObjectType model. + update_fields=[ + "is_imported", + "name", + "name_plural", + "description", + "data_classification", + "maintainer_organization", + "maintainer_department", + "contact_person", + "contact_email", + "source", + "update_frequency", + "provider_organization", + "documentation_url", + "labels", + "created_at", + "modified_at", + "allow_geometry", + "linkable_to_zaken", + ], + ) + + def _bulk_create_or_update_objecttype_versions(self, data): + ObjectTypeVersion.objects.bulk_create( + data, + ignore_conflicts=True, + unique_fields=[ + "object_type", + "version", + ], + update_fields=[ + "created_at", + "modified_at", + "published_at", + "json_schema", + "status", + ], + ) + + def _parse_objecttype_data( + self, objecttypes: list[dict[str, Any]], service: Service + ) -> list[ObjectType]: + data = [] + for objecttype in objecttypes: + objecttype.pop("versions") + objecttype.pop("url") + objecttype["service"] = service + objecttype["is_imported"] = True + data.append(ObjectType(**underscoreize(objecttype))) + return data + + def _parse_objectversion_data( + self, objecttype_versions: list[dict[str, Any]], objecttype + ) -> list[ObjectTypeVersion]: + data = [] + for objecttype_version in objecttype_versions: + objecttype_version.pop("url") + objecttype_version["objectType"] = objecttype + data.append(ObjectTypeVersion(**underscoreize(objecttype_version))) + return data diff --git a/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py b/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py new file mode 100644 index 00000000..cf673ee4 --- /dev/null +++ b/src/objects/core/migrations/0035_objecttype_allow_geometry_objecttype_contact_email_and_more.py @@ -0,0 +1,115 @@ +# Generated by Django 5.2.7 on 2025-12-04 11:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0034_alter_objectrecord__object_type_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='objecttype', + name='allow_geometry', + field=models.BooleanField(default=True, help_text="Shows whether the related objects can have geographic coordinates. If the value is 'false' the related objects are not allowed to have coordinates and the creation/update of objects with `geometry` property will raise an error ", verbose_name='allow geometry'), + ), + migrations.AddField( + model_name='objecttype', + name='contact_email', + field=models.CharField(blank=True, help_text='Email of the person in the organization who can provide information about the object type', max_length=200, verbose_name='contact email'), + ), + migrations.AddField( + model_name='objecttype', + name='contact_person', + field=models.CharField(blank=True, help_text='Name of the person in the organization who can provide information about the object type', max_length=200, verbose_name='contact person'), + ), + migrations.AddField( + model_name='objecttype', + name='created_at', + field=models.DateField(blank=True, help_text='Date when the object type was created', null=True, verbose_name='created at'), + ), + migrations.AddField( + model_name='objecttype', + name='data_classification', + field=models.CharField(choices=[('open', 'Open'), ('intern', 'Intern'), ('confidential', 'Confidential'), ('strictly_confidential', 'Strictly confidential')], default='open', help_text='Confidential level of the object type', max_length=50, verbose_name='data classification'), + ), + migrations.AddField( + model_name='objecttype', + name='description', + field=models.CharField(blank=True, help_text='The description of the object type', max_length=1000, verbose_name='description'), + ), + migrations.AddField( + model_name='objecttype', + name='documentation_url', + field=models.URLField(blank=True, help_text='Link to the documentation for the object type', verbose_name='documentation url'), + ), + migrations.AddField( + model_name='objecttype', + name='labels', + field=models.JSONField(blank=True, default=dict, help_text='Key-value pairs of keywords related for the object type', verbose_name='labels'), + ), + migrations.AddField( + model_name='objecttype', + name='linkable_to_zaken', + field=models.BooleanField(default=False, help_text='Objects of this type can have a link to 1 or more Zaken.\nTrue indicates the lifetime of the object is linked to the lifetime of linked zaken, i.e., when all linked Zaken to an object are archived/destroyed, the object will also be archived/destroyed.', verbose_name='linkable to zaken'), + ), + migrations.AddField( + model_name='objecttype', + name='maintainer_department', + field=models.CharField(blank=True, help_text='Business department which is responsible for the object type', max_length=200, verbose_name='maintainer department'), + ), + migrations.AddField( + model_name='objecttype', + name='maintainer_organization', + field=models.CharField(blank=True, help_text='Organization which is responsible for the object type', max_length=200, verbose_name='maintainer organization'), + ), + migrations.AddField( + model_name='objecttype', + name='modified_at', + field=models.DateField(blank=True, help_text='Last date when the object type was modified', null=True, verbose_name='modified at'), + ), + migrations.AddField( + model_name='objecttype', + name='name', + field=models.CharField(blank=True, help_text='Name of the object type', max_length=100, verbose_name='name'), + ), + migrations.AddField( + model_name='objecttype', + name='name_plural', + field=models.CharField(blank=True, help_text='Plural name of the object type', max_length=100, verbose_name='name plural'), + ), + migrations.AddField( + model_name='objecttype', + name='provider_organization', + field=models.CharField(blank=True, help_text='Organization which is responsible for publication of the object type', max_length=200, verbose_name='provider organization'), + ), + migrations.AddField( + model_name='objecttype', + name='source', + field=models.CharField(blank=True, help_text='Name of the system from which the object type originates', max_length=200, verbose_name='source'), + ), + migrations.AddField( + model_name='objecttype', + name='update_frequency', + field=models.CharField(choices=[('real_time', 'Real-time'), ('hourly', 'Hourly'), ('daily', 'Daily'), ('weekly', 'Weekly'), ('monthly', 'Monthly'), ('yearly', 'Yearly'), ('unknown', 'Unknown')], default='unknown', help_text='Indicates how often the object type is updated', max_length=10, verbose_name='update frequency'), + ), + migrations.CreateModel( + name='ObjectTypeVersion', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('version', models.PositiveSmallIntegerField(help_text='Integer version of the OBJECTTYPE', verbose_name='version')), + ('created_at', models.DateField(blank=True, help_text='Date when the version was created', null=True, verbose_name='created at')), + ('modified_at', models.DateField(blank=True, help_text='Last date when the version was modified', null=True, verbose_name='modified at')), + ('published_at', models.DateField(blank=True, help_text='Date when the version was published', null=True, verbose_name='published_at')), + ('json_schema', models.JSONField(default=dict, help_text='JSON schema for Object validation', verbose_name='JSON schema')), + ('status', models.CharField(choices=[('published', 'Published'), ('draft', 'Draft'), ('deprecated', 'Deprecated')], default='draft', help_text='Status of the object type version', max_length=20, verbose_name='status')), + ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='core.objecttype')), + ], + options={ + 'unique_together': {('object_type', 'version')}, + }, + ), + ] diff --git a/src/objects/core/migrations/0036_objecttype_is_imported.py b/src/objects/core/migrations/0036_objecttype_is_imported.py new file mode 100644 index 00000000..784b3b8d --- /dev/null +++ b/src/objects/core/migrations/0036_objecttype_is_imported.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.7 on 2025-12-05 13:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0035_objecttype_allow_geometry_objecttype_contact_email_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='objecttype', + name='is_imported', + field=models.BooleanField(default=False, editable=False, verbose_name='Is imported'), + ), + ] diff --git a/src/objects/core/models.py b/src/objects/core/models.py index 45cb057f..dd95aa62 100644 --- a/src/objects/core/models.py +++ b/src/objects/core/models.py @@ -15,8 +15,13 @@ from objects.utils.client import get_objecttypes_client +from .constants import ( + DataClassificationChoices, + ObjectVersionStatus, + UpdateFrequencyChoices, +) from .query import ObjectQuerySet, ObjectRecordQuerySet, ObjectTypeQuerySet -from .utils import check_objecttype_cached +from .utils import check_json_schema, check_objecttype_cached class ObjectType(models.Model): @@ -29,6 +34,134 @@ class ObjectType(models.Model): _name = models.CharField( max_length=100, help_text=_("Cached name of the objecttype retrieved from the Objecttype API"), + ) # TODO can be removed after objecttype migration + + is_imported = models.BooleanField( + _("Is imported"), + default=False, + editable=False, + ) # TODO temp field to track if object was imported, can be removed after objecttype migration + + name = models.CharField( + _("name"), + max_length=100, + help_text=_("Name of the object type"), + blank=True, # TODO blank=False after objecttype migration + ) + + name_plural = models.CharField( + _("name plural"), + max_length=100, + help_text=_("Plural name of the object type"), + blank=True, # TODO blank=False after objecttype migration + ) + description = models.CharField( + _("description"), + max_length=1000, + blank=True, + help_text=_("The description of the object type"), + ) + data_classification = models.CharField( + _("data classification"), + max_length=50, + choices=DataClassificationChoices.choices, + default=DataClassificationChoices.open, + help_text=_("Confidential level of the object type"), + ) + maintainer_organization = models.CharField( + _("maintainer organization"), + max_length=200, + blank=True, + help_text=_("Organization which is responsible for the object type"), + ) + maintainer_department = models.CharField( + _("maintainer department"), + max_length=200, + blank=True, + help_text=_("Business department which is responsible for the object type"), + ) + contact_person = models.CharField( + _("contact person"), + max_length=200, + blank=True, + help_text=_( + "Name of the person in the organization who can provide information about the object type" + ), + ) + contact_email = models.CharField( + _("contact email"), + max_length=200, + blank=True, + help_text=_( + "Email of the person in the organization who can provide information about the object type" + ), + ) + source = models.CharField( + _("source"), + max_length=200, + blank=True, + help_text=_("Name of the system from which the object type originates"), + ) + update_frequency = models.CharField( + _("update frequency"), + max_length=10, + choices=UpdateFrequencyChoices.choices, + default=UpdateFrequencyChoices.unknown, + help_text=_("Indicates how often the object type is updated"), + ) + provider_organization = models.CharField( + _("provider organization"), + max_length=200, + blank=True, + help_text=_( + "Organization which is responsible for publication of the object type" + ), + ) + documentation_url = models.URLField( + _("documentation url"), + blank=True, + help_text=_("Link to the documentation for the object type"), + ) + labels = models.JSONField( + _("labels"), + help_text=_("Key-value pairs of keywords related for the object type"), + default=dict, + blank=True, + ) + created_at = models.DateField( + _("created at"), + auto_now_add=False, # TODO auto_now_add=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration + help_text=_("Date when the object type was created"), + ) + modified_at = models.DateField( + _("modified at"), + auto_now=False, # TODO auto_now=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration + help_text=_("Last date when the object type was modified"), + ) + allow_geometry = models.BooleanField( + _("allow geometry"), + default=True, + help_text=_( + "Shows whether the related objects can have geographic coordinates. " + "If the value is 'false' the related objects are not allowed to " + "have coordinates and the creation/update of objects with " + "`geometry` property will raise an error " + ), + ) + linkable_to_zaken = models.BooleanField( + _("linkable to zaken"), + default=False, + help_text=_( + # TODO Document: how and where these links should be created/maintained + "Objects of this type can have a link to 1 or more Zaken.\n" + "True indicates the lifetime of the object is linked to the lifetime " + "of linked zaken, i.e., when all linked Zaken to an object are " + "archived/destroyed, the object will also be archived/destroyed." + ), ) objects = ObjectTypeQuerySet.as_manager() @@ -66,6 +199,86 @@ def clean_fields(self, exclude: Iterable[str] | None = None) -> None: self._name = object_type_data["name"] +class ObjectTypeVersion(models.Model): + object_type = models.ForeignKey( + ObjectType, on_delete=models.CASCADE, related_name="versions" + ) + version = models.PositiveSmallIntegerField( + _("version"), help_text=_("Integer version of the OBJECTTYPE") + ) + created_at = models.DateField( + _("created at"), + auto_now_add=False, # TODO auto_now_add=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration + help_text=_("Date when the version was created"), + ) + modified_at = models.DateField( + _("modified at"), + auto_now=False, # TODO auto_now=True after migration + blank=True, # TODO blank=False after migration + null=True, # TODO null=False after migration + help_text=_("Last date when the version was modified"), + ) + published_at = models.DateField( + _("published_at"), + null=True, + blank=True, + help_text=_("Date when the version was published"), + ) + json_schema = models.JSONField( + _("JSON schema"), help_text=_("JSON schema for Object validation"), default=dict + ) + status = models.CharField( + _("status"), + max_length=20, + choices=ObjectVersionStatus.choices, + default=ObjectVersionStatus.draft, + help_text=_("Status of the object type version"), + ) + + class Meta: + unique_together = ("object_type", "version") + + def __str__(self): + return f"{self.object_type} v.{self.version}" + + def clean(self): + super().clean() + + check_json_schema(self.json_schema) + + def save(self, *args, **kwargs): + if not self.version: + self.version = self.generate_version_number() + + # save published_at + previous_status = ( + ObjectTypeVersion.objects.get(id=self.id).status if self.id else None + ) + if ( + self.status == ObjectVersionStatus.published + and previous_status != self.status + ): + self.published_at = datetime.date.today() + + super().save(*args, **kwargs) + + def generate_version_number(self) -> int: + existed_versions = ObjectTypeVersion.objects.filter( + object_type=self.object_type + ) + + max_version = 0 + if existed_versions.exists(): + max_version = existed_versions.aggregate(models.Max("version"))[ + "version__max" + ] + + version_number = max_version + 1 + return version_number + + class Object(models.Model): uuid = models.UUIDField( unique=True, default=uuid.uuid4, help_text=_("Unique identifier (UUID4)") diff --git a/src/objects/core/tests/factories.py b/src/objects/core/tests/factories.py index 685819be..427bce95 100644 --- a/src/objects/core/tests/factories.py +++ b/src/objects/core/tests/factories.py @@ -8,7 +8,7 @@ from factory.fuzzy import BaseFuzzyAttribute from zgw_consumers.test.factories import ServiceFactory -from ..models import Object, ObjectRecord, ObjectType +from ..models import Object, ObjectRecord, ObjectType, ObjectTypeVersion class ObjectTypeFactory(factory.django.DjangoModelFactory): @@ -16,10 +16,28 @@ class ObjectTypeFactory(factory.django.DjangoModelFactory): uuid = factory.LazyFunction(uuid.uuid4) _name = factory.Faker("word") + name = factory.Faker("word") + name_plural = factory.LazyAttribute(lambda x: f"{x.name}s") + description = factory.Faker("bs") + class Meta: model = ObjectType +class ObjectTypeVersionFactory(factory.django.DjangoModelFactory): + object_type = factory.SubFactory(ObjectTypeFactory) + json_schema = { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["diameter"], + "properties": {"diameter": {"type": "integer", "description": "size in cm."}}, + } + + class Meta: + model = ObjectTypeVersion + + class FuzzyPoint(BaseFuzzyAttribute): def fuzz(self): return Point(random.uniform(-180.0, 180.0), random.uniform(-90.0, 90.0)) diff --git a/src/objects/core/tests/test_import_objecttypes.py b/src/objects/core/tests/test_import_objecttypes.py new file mode 100644 index 00000000..d10c1779 --- /dev/null +++ b/src/objects/core/tests/test_import_objecttypes.py @@ -0,0 +1,159 @@ +import uuid + +from django.core.management import CommandError, call_command +from django.test import TestCase + +import requests_mock +from zgw_consumers.models import Service + +from objects.core.models import ObjectType, ObjectTypeVersion +from objects.core.tests.factories import ObjectTypeFactory +from objects.tests.utils import ( + mock_objecttype_versions, + mock_objecttypes, +) + + +class TestImportObjectTypesCommand(TestCase): + def setUp(self): + super().setUp() + self.url = "http://127.0.0.1:8000/api/v2/" + + self.m = requests_mock.Mocker() + self.m.start() + + self.service = Service.objects.create(api_root=self.url, slug="objecttypes-api") + self.m.head(self.url, status_code=200, headers={"api-version": "3.4.0"}) + + def tearDown(self): + self.m.stop() + + def _call_command(self): + call_command("import_objecttypes", self.service.slug) + + def test_api_version_is_required(self): + self.m.head(self.url, status_code=200) + + with self.assertRaisesMessage( + CommandError, "API version must be 3.4.0 or higher" + ): + self._call_command() + + def test_api_version_must_be_greater_than_constant(self): + self.m.head(self.url, status_code=200, headers={"api-version": "3.2.0"}) + + with self.assertRaisesMessage( + CommandError, "API version must be 3.4.0 or higher" + ): + self._call_command() + + def test_command_fails_if_http_error(self): + self.m.get(f"{self.url}objecttypes", status_code=404) + with self.assertRaises(CommandError): + self._call_command() + + def test_new_objecttypes_are_created(self): + self.assertEqual(ObjectType.objects.count(), 0) + self.assertEqual(ObjectTypeVersion.objects.count(), 0) + + uuid1 = str(uuid.uuid4()) + uuid2 = str(uuid.uuid4()) + + self.m.get(f"{self.url}objecttypes", json=mock_objecttypes(uuid1, uuid2)) + self.m.get( + f"{self.url}objecttypes/{uuid1}/versions", + json=mock_objecttype_versions(uuid1), + ) + self.m.get( + f"{self.url}objecttypes/{uuid2}/versions", + json=mock_objecttype_versions(uuid2), + ) + + self._call_command() + + self.assertEqual(ObjectType.objects.count(), 2) + + objecttype = ObjectType.objects.get(uuid=uuid1) + self.assertEqual(objecttype.is_imported, True) + self.assertEqual(objecttype.name, "Melding") + self.assertEqual(objecttype.name_plural, "Meldingen") + self.assertEqual(objecttype.description, "") + self.assertEqual(objecttype.data_classification, "intern") + self.assertEqual(objecttype.maintainer_organization, "Dimpact") + self.assertEqual(objecttype.maintainer_department, "") + self.assertEqual(objecttype.contact_person, "Ad Alarm") + self.assertEqual(objecttype.contact_email, "") + self.assertEqual(objecttype.source, "") + self.assertEqual(objecttype.update_frequency, "unknown") + self.assertEqual(objecttype.provider_organization, "") + self.assertEqual(objecttype.documentation_url, "") + self.assertEqual(objecttype.labels, {}) + self.assertEqual(objecttype.linkable_to_zaken, False) + self.assertEqual(str(objecttype.created_at), "2020-12-01") + self.assertEqual(str(objecttype.modified_at), "2020-12-01") + self.assertEqual(objecttype.allow_geometry, True) + + self.assertEqual(ObjectTypeVersion.objects.count(), 4) + + version = ObjectTypeVersion.objects.get(object_type=objecttype, version=1) + self.assertEqual(str(version.created_at), "2020-12-01") + self.assertEqual(str(version.modified_at), "2020-12-01") + self.assertEqual(str(version.published_at), "2020-10-02") + self.assertEqual( + version.json_schema, + { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + ) + self.assertEqual(str(version.status), "published") + + def test_existing_objecttypes_are_updated(self): + objecttype1 = ObjectTypeFactory(service=self.service) + objecttype2 = ObjectTypeFactory(service=self.service) + + self.m.get( + f"{self.url}objecttypes", + json=mock_objecttypes(str(objecttype1.uuid), str(objecttype2.uuid)), + ) + self.m.get( + f"{self.url}objecttypes/{str(objecttype1.uuid)}/versions", + json=mock_objecttype_versions(str(objecttype1.uuid)), + ) + self.m.get( + f"{self.url}objecttypes/{str(objecttype2.uuid)}/versions", + json=mock_objecttype_versions(str(objecttype2.uuid)), + ) + + self._call_command() + self.assertEqual(ObjectType.objects.count(), 2) + self.assertEqual(ObjectTypeVersion.objects.count(), 4) + + objecttype = ObjectType.objects.get(uuid=objecttype1.uuid) + self.assertEqual(objecttype.is_imported, True) + self.assertEqual(objecttype.name, "Melding") + + version = ObjectTypeVersion.objects.get(object_type=objecttype, version=1) + self.assertEqual( + version.json_schema, + { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + ) diff --git a/src/objects/core/tests/test_objecttypeversion_generate.py b/src/objects/core/tests/test_objecttypeversion_generate.py new file mode 100644 index 00000000..49a7c52f --- /dev/null +++ b/src/objects/core/tests/test_objecttypeversion_generate.py @@ -0,0 +1,33 @@ +from django.test import TestCase + +from objects.core.models import ObjectTypeVersion +from objects.core.tests.factories import ObjectTypeFactory, ObjectTypeVersionFactory + +JSON_SCHEMA = { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["diameter"], + "properties": {"diameter": {"type": "integer", "description": "size in cm."}}, +} + + +class GenerateVersionTests(TestCase): + def test_generate_version_for_new_objecttype(self): + object_type = ObjectTypeFactory.create() + + object_version = ObjectTypeVersion.objects.create( + json_schema=JSON_SCHEMA, object_type=object_type + ) + + self.assertEqual(object_version.version, 1) + + def test_generate_version_for_objecttype_with_existed_version(self): + object_type = ObjectTypeFactory.create() + ObjectTypeVersionFactory.create(object_type=object_type, version=1) + + object_version = ObjectTypeVersion.objects.create( + json_schema=JSON_SCHEMA, object_type=object_type + ) + + self.assertEqual(object_version.version, 2) diff --git a/src/objects/core/utils.py b/src/objects/core/utils.py index ff6e65ca..3879d711 100644 --- a/src/objects/core/utils.py +++ b/src/objects/core/utils.py @@ -3,6 +3,8 @@ import jsonschema import requests +from jsonschema.exceptions import SchemaError +from jsonschema.validators import validator_for from objects.core import models from objects.utils.cache import cache @@ -27,8 +29,8 @@ def get_objecttype_version_response(): ) try: - vesion_data = get_objecttype_version_response() - jsonschema.validate(data, vesion_data["jsonSchema"]) + version_data = get_objecttype_version_response() + jsonschema.validate(data, version_data["jsonSchema"]) except KeyError: raise ValidationError( f"{object_type.versions_url} does not appear to be a valid objecttype.", @@ -49,3 +51,11 @@ def can_connect_to_objecttypes() -> bool: if not client.can_connect: return False return True + + +def check_json_schema(json_schema: dict): + schema_validator = validator_for(json_schema) + try: + schema_validator.check_schema(json_schema) + except SchemaError as exc: + raise ValidationError(exc.args[0]) from exc diff --git a/src/objects/tests/utils.py b/src/objects/tests/utils.py index 9f7043f8..1ea4ae4e 100644 --- a/src/objects/tests/utils.py +++ b/src/objects/tests/utils.py @@ -55,6 +55,118 @@ def mock_objecttype(url: str, attrs=None) -> dict: return response +def mock_objecttypes(uuid1, uuid2): + return { + "count": 2, + "next": None, + "previous": None, + "results": [ + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}", + "uuid": uuid1, + "name": "Melding", + "namePlural": "Meldingen", + "description": "", + "dataClassification": "intern", + "maintainerOrganization": "Dimpact", + "maintainerDepartment": "", + "contactPerson": "Ad Alarm", + "contactEmail": "", + "source": "", + "updateFrequency": "unknown", + "providerOrganization": "", + "documentationUrl": "", + "labels": {}, + "linkableToZaken": False, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "allowGeometry": True, + "versions": [ + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}/versions/1", + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid1}/versions/2", + ], + }, + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}", + "uuid": uuid2, + "name": "Straatverlichting", + "namePlural": "Straatverlichting", + "description": "", + "dataClassification": "open", + "maintainerOrganization": "Maykin Media", + "maintainerDepartment": "", + "contactPerson": "Desiree Lumen", + "contactEmail": "", + "source": "", + "updateFrequency": "unknown", + "providerOrganization": "", + "documentationUrl": "", + "labels": {}, + "linkableToZaken": False, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "allowGeometry": True, + "versions": [ + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}/versions/1", + f"http://127.0.0.1:8000/api/v2/objecttypes/{uuid2}/versions/2", + ], + }, + ], + } + + +def mock_objecttype_versions(objecttype_uuid: str): + return { + "count": 2, + "next": None, + "previous": None, + "results": [ + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}/versions/2", + "version": 2, + "objectType": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}", + "status": "published", + "jsonSchema": { + "type": "object", + "title": "Tree", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + "createdAt": "2020-11-12", + "modifiedAt": "2020-11-27", + "publishedAt": "2020-11-27", + }, + { + "url": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}/versions/1", + "version": 1, + "objectType": f"http://127.0.0.1:8000/api/v2/objecttypes/{objecttype_uuid}", + "status": "published", + "jsonSchema": { + "type": "object", + "title": "Melding", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["description"], + "properties": { + "description": { + "type": "string", + "description": "Explanation what happened", + } + }, + }, + "createdAt": "2020-12-01", + "modifiedAt": "2020-12-01", + "publishedAt": "2020-10-02", + }, + ], + } + + def mock_objecttype_version(url: str, attrs=None) -> dict: attrs = attrs or {} response = { diff --git a/src/objects/utils/client.py b/src/objects/utils/client.py index e7989a57..880e3d11 100644 --- a/src/objects/utils/client.py +++ b/src/objects/utils/client.py @@ -80,6 +80,11 @@ def get_objecttype_version( response.raise_for_status() return response.json() + def get_objecttypes_api_version(self) -> str | None: + response = self.head("") + response.raise_for_status() + return response.headers.get("api-version") + def get_objecttypes_client(service) -> ObjecttypesClient: assert service is not None