From f790b364f90c3e3923a24e75a1867d21f8b3c947 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Tue, 30 Jan 2024 16:16:02 +0100 Subject: [PATCH 01/10] Allow the test branch for symfony 5.4 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 9267b3c..ed8e19b 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "minimum-stability": "dev", "require": { "php": ">=7.4", - "commongateway/corebundle": "^1.1.52" + "commongateway/corebundle": "^1.1.52 | dev-dev-feature/GW-1596/Symfony54" }, "require-dev": { "symfony/dependency-injection": "~3.4|~4.1|~5.0" From dfb213318c60fe159d1817a1d2593df22ae51807 Mon Sep 17 00:00:00 2001 From: GitHub Actions <> Date: Tue, 30 Jan 2024 15:16:28 +0000 Subject: [PATCH 02/10] Update src from PHP Codesniffer --- .../SyncOpenWooDocumentHandler.php | 2 +- src/Service/SitemapService.php | 25 +- src/Service/SyncOpenWooService.php | 860 +++++++++--------- 3 files changed, 443 insertions(+), 444 deletions(-) diff --git a/src/ActionHandler/SyncOpenWooDocumentHandler.php b/src/ActionHandler/SyncOpenWooDocumentHandler.php index 87456ce..bbea8b5 100644 --- a/src/ActionHandler/SyncOpenWooDocumentHandler.php +++ b/src/ActionHandler/SyncOpenWooDocumentHandler.php @@ -62,7 +62,7 @@ public function getConfiguration(): array 'sourceEndpoint', ], 'properties' => [ - 'endpoint' => [ + 'endpoint' => [ 'type' => 'string', 'description' => 'The endpoint reference for documents.', 'example' => 'buren', diff --git a/src/Service/SitemapService.php b/src/Service/SitemapService.php index df8f824..5ab9597 100644 --- a/src/Service/SitemapService.php +++ b/src/Service/SitemapService.php @@ -145,14 +145,14 @@ public function sitemapHandler(array $data, array $configuration): array $parameters = array_merge($this->data['path'], $this->data['query']); switch ($this->configuration['type']) { - case 'sitemap': - return $this->getSitemap($parameters); - case 'sitemapindex': - return $this->getSitemapindex($parameters); - case 'robot.txt': - return $this->getRobot($parameters); - default: - $this->logger->error('Invalid action configuration type.', ['plugin' => 'common-gateway/woo-bundle']); + case 'sitemap': + return $this->getSitemap($parameters); + case 'sitemapindex': + return $this->getSitemapindex($parameters); + case 'robot.txt': + return $this->getRobot($parameters); + default: + $this->logger->error('Invalid action configuration type.', ['plugin' => 'common-gateway/woo-bundle']); } $this->data['response'] = $this->createResponse(['Message' => 'Invalid action configuration type.'], 409, 'error'); @@ -189,17 +189,16 @@ private function getSitemap(array $parameters): array unset($filter['oin'], $filter['sitemaps'], $filter['sitemap']); - $filter = [ - '_limit' => 50000,]; + $filter = ['_limit' => 50000]; // Get all the publication objects with the given query. $objects = $this->cacheService->searchObjects(null, $filter, [$publicatieSchema->getId()->toString()])['results']; $sitemap = []; foreach ($objects as $object) { - $document = $this->entityManager->getRepository('App:ObjectEntity')->find($object['_id']); - $file = $document->getValueObject('url')->getFiles()->first(); - $mappedObject = $this->mappingService->mapping($mapping, ['object' => json_decode(json_encode($object), true), 'file' => $file]); + $document = $this->entityManager->getRepository('App:ObjectEntity')->find($object['_id']); + $file = $document->getValueObject('url')->getFiles()->first(); + $mappedObject = $this->mappingService->mapping($mapping, ['object' => json_decode(json_encode($object), true), 'file' => $file]); $sitemap['url'][] = $mappedObject; } diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index f687a71..211bbcd 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -1,430 +1,430 @@ -, Barry Brands . - * @license EUPL - * - * @package CommonGateway\WOOBundle - * @category Service - */ -class SyncOpenWooService -{ - - /** - * @var GatewayResourceService - */ - private GatewayResourceService $resourceService; - - /** - * @var CallService - */ - private CallService $callService; - - /** - * @var SynchronizationService - */ - private SynchronizationService $syncService; - - /** - * @var MappingService - */ - private MappingService $mappingService; - - /** - * @var EntityManagerInterface - */ - private EntityManagerInterface $entityManager; - - /** - * @var SymfonyStyle|null - */ - private ?SymfonyStyle $style = null; - - /** - * @var LoggerInterface $logger. - */ - private LoggerInterface $logger; - - /** - * @var ValidationService $validationService. - */ - private ValidationService $validationService; - - /** - * @var CacheService $cacheService. - */ - private CacheService $cacheService; - - private FileService $fileService; - - private ObjectEntityService $gatewayOEService; - - /** - * @var array - */ - private array $data; - - /** - * @var array - */ - private array $configuration; - - - /** - * SyncOpenWooService constructor. - * - * @param GatewayResourceService $resourceService - * @param CallService $callService - * @param SynchronizationService $syncService - * @param EntityManagerInterface $entityManager - * @param MappingService $mappingService - * @param LoggerInterface $pluginLogger - * @param FileService $fileService - * - */ - public function __construct( - GatewayResourceService $resourceService, - CallService $callService, - SynchronizationService $syncService, - EntityManagerInterface $entityManager, - MappingService $mappingService, - LoggerInterface $pluginLogger, - ValidationService $validationService, - CacheService $cacheService, - FileService $fileService, - ObjectEntityService $gatewayOEService - ) { - $this->resourceService = $resourceService; - $this->callService = $callService; - $this->syncService = $syncService; - $this->entityManager = $entityManager; - $this->mappingService = $mappingService; - $this->logger = $pluginLogger; - $this->validationService = $validationService; - $this->cacheService = $cacheService; - $this->fileService = $fileService; - $this->gatewayOEService = $gatewayOEService; - - }//end __construct() - - - /** - * Set symfony style in order to output to the console. - * - * @param SymfonyStyle $style - * - * @return self - * - * @todo change to monolog - */ - public function setStyle(SymfonyStyle $style): self - { - $this->style = $style; - - return $this; - - }//end setStyle() - - - /** - * Checks if existing objects still exist in the source, if not deletes them. - * - * @param array $idsSynced ID's from objects we just synced from the source. - * @param Source $source These objects belong to. - * @param string $schemaRef These objects belong to. - * @param string $categorie The categorie these objects came from. - * - * @return int Count of deleted objects. - */ - private function deleteNonExistingObjects(array $idsSynced, Source $source, string $schemaRef, string $categorie): int - { - // Get all existing sourceIds. - $source = $this->entityManager->find('App:Gateway', $source->getId()->toString()); - $existingSourceIds = []; - $existingObjects = []; - foreach ($source->getSynchronizations() as $synchronization) { - if ($synchronization->getEntity()->getReference() === $schemaRef && $synchronization->getSourceId() !== null && $synchronization->getObject()->getValue('categorie') === $categorie) { - $existingSourceIds[] = $synchronization->getSourceId(); - $existingObjects[] = $synchronization->getObject(); - } - } - - // Check if existing sourceIds are in the array of new synced sourceIds. - $objectIdsToDelete = array_diff($existingSourceIds, $idsSynced); - - // If not it means the object does not exist in the source anymore and should be deleted here. - $deletedObjectsCount = 0; - foreach ($objectIdsToDelete as $key => $id) { - $this->logger->info("Object $id does not exist at the source, deleting.", ['plugin' => 'common-gateway/woo-bundle']); - $this->entityManager->remove($existingObjects[$key]); - $deletedObjectsCount++; - } - - $this->entityManager->flush(); - - return $deletedObjectsCount; - - }//end deleteNonExistingObjects() - - - /** - * Fetches objects from openWoo with pagination. - * - * @param Source $source The source entity that provides the source of the result data. - * @param int|null $page The page we are fetching, increments each iteration. - * @param array $results The results from xxllnc api we merge each iteration. - * @param string $categorie The type of object we are fetching. - * - * @return array The fetched objects. - */ - private function fetchObjects(Source $source, ?int $page=1, array $results=[], string $categorie) - { - $response = $this->callService->call($source, $this->configuration['sourceEndpoint'], 'GET', ['query' => ['page' => $page]]); - $decodedResponse = $this->callService->decodeResponse($source, $response); - - switch ($categorie) { - case 'Woo verzoek': - $results = array_merge($results, $decodedResponse['WOOverzoeken']); - break; - case 'Convenant': - $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); - break; - } - - // Pagination xxllnc. - if (isset($decodedResponse['pagination']) === true && $decodedResponse['pagination']['pages']['current'] < $decodedResponse['pagination']['pages']['total']) { - $page++; - $results = $this->fetchObjects($source, $page, $results, $categorie); - } - - return $results; - - }//end fetchObjects() - - - /** - * Handles the synchronization of openwoo objects. - * - * @param array $data - * @param array $configuration - * - * @throws CacheException|InvalidArgumentException - * - * @return array - */ - public function syncOpenWooHandler(array $data, array $configuration): array - { - $this->data = $data; - $this->configuration = $configuration; - - isset($this->style) === true && $this->style->success('SyncOpenWooService triggered'); - $this->logger->info('SyncOpenWooService triggered', ['plugin' => 'common-gateway/woo-bundle']); - - if (isset($this->configuration['source']) === false - || isset($this->configuration['oin']) === false - || isset($this->configuration['organisatie']) === false - || isset($this->configuration['portalUrl']) === false - || isset($this->configuration['schema']) === false - || isset($this->configuration['mapping']) === false - || isset($this->configuration['sourceEndpoint']) === false - ) { - isset($this->style) === true && $this->style->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler'); - $this->logger->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler', ['plugin' => 'common-gateway/woo-bundle']); - - return []; - }//end if - - $source = $this->resourceService->getSource($this->configuration['source'], 'common-gateway/woo-bundle'); - $schema = $this->resourceService->getSchema($this->configuration['schema'], 'common-gateway/woo-bundle'); - $mapping = $this->resourceService->getMapping($this->configuration['mapping'], 'common-gateway/woo-bundle'); - $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.categorie.mapping.json', 'common-gateway/woo-bundle'); - if ($source instanceof Source === false - || $schema instanceof Schema === false - || $mapping instanceof Mapping === false - ) { - isset($this->style) === true && $this->style->error("{$this->configuration['source']}, {$this->configuration['schema']} or {$this->configuration['mapping']} not found, ending syncOpenWooHandler"); - - return []; - }//end if - - $categorie = ''; - switch ($mapping->getReference()) { - case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': - $categorie = 'Woo verzoek'; - break; - case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': - $categorie = 'Convenant'; - break; - } - - isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}"); - $this->logger->info("Fetching objects from {$source->getLocation()}", ['plugin' => 'common-gateway/woo-bundle']); - - $results = $this->fetchObjects($source, 1, [], $categorie); - if (empty($results) === true) { - $this->logger->info('No results found, ending SyncOpenWooService', ['plugin' => 'common-gateway/woo-bundle']); - return $this->data; - } - - $this->entityManager->flush(); - - $customFields = [ - 'organisatie' => [ - 'oin' => $this->configuration['oin'], - 'naam' => $this->configuration['organisatie'], - ], - 'categorie' => $categorie, - ]; - - $idsSynced = []; - $responseItems = []; - $documents = []; - $hydrationService = new HydrationService($this->syncService, $this->entityManager); - foreach ($results as $result) { - try { - $result = array_merge($result, $customFields); - $mappedResult = $this->mappingService->mapping($mapping, $result); - // Map categories to prevent multiple variants of the same categorie. - $mappedResult = $this->mappingService->mapping($categorieMapping, $mappedResult); - - $validationErrors = $this->validationService->validateData($mappedResult, $schema, 'POST'); - if ($validationErrors !== null) { - $validationErrors = implode(', ', $validationErrors); - $this->logger->warning("SyncOpenWoo validation errors: $validationErrors", ['plugin' => 'common-gateway/woo-bundle']); - isset($this->style) === true && $this->style->warning("SyncOpenWoo validation errors: $validationErrors"); - continue; - } - - $object = $hydrationService->searchAndReplaceSynchronizations( - $mappedResult, - $source, - $schema, - true, - true - ); - - // Get all synced sourceIds. - if (empty($object->getSynchronizations()) === false && $object->getSynchronizations()[0]->getSourceId() !== null) { - $idsSynced[] = $object->getSynchronizations()[0]->getSourceId(); - } - - $this->entityManager->persist($object); - $this->cacheService->cacheObject($object); - $responseItems[] = $object; - - $renderedObject = $object->toArray(); - $documents = array_merge($documents, $renderedObject['bijlagen']); - if(isset($renderedObject['metadata']['verzoek']['informatieverzoek']) === true) { - $documents[] = $renderedObject['metadata']['verzoek']['informatieverzoek']; - } - if(isset($renderedObject['verzoek']['besluit']) === true) { - $documents[] = $renderedObject['metadata']['verzoek']['besluit']; - } - - } catch (Exception $exception) { - $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['UUID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle']); - continue; - }//end try - }//end foreach - - $this->entityManager->flush(); - - - foreach($documents as $document) { - $documentData['document'] = $document; - $documentData['source'] = $source->getReference(); - $this->gatewayOEService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); - } - - $deletedObjectsCount = $this->deleteNonExistingObjects($idsSynced, $source, $this->configuration['schema'], $categorie); - - $this->data['response'] = new Response(json_encode($responseItems), 200); - - $countItems = count($responseItems); - $logMessage = "Synchronized $countItems cases to woo objects for ".$source->getName()." and deleted $deletedObjectsCount objects"; - isset($this->style) === true && $this->style->success($logMessage); - $this->logger->info($logMessage, ['plugin' => 'common-gateway/woo-bundle']); - - return $this->data; - - }//end syncOpenWooHandler() - - public function syncOpenWooDocumentHandler(array $data, array $config): array - { - $source = $this->resourceService->getSource($data['source'], 'common-gateway/woo-bundle'); - $document = $data['document']; - $endpoint = $this->resourceService->getEndpoint($config['endpoint'], 'common-gateway/woo-bundle'); - - if(substr($document['url'], 0, strlen($source->getLocation())) === $source->getLocation()) { - $path = substr($document['url'], strlen($source->getLocation())); - } else { - $this->logger->error('Url of document does not correspond with source'); - - return $data; - } - - $bijlageObject = $this->entityManager->getRepository('App:ObjectEntity')->find($document['_self']['id']); - if($bijlageObject instanceof ObjectEntity === false) { - return $data; - } - - $value = $bijlageObject->getValueObject('url'); - - if($value->getFiles()->count() > 0) { - $file = $value->getFiles()->first(); - } else { - $file = new File(); - } - - $response = $this->callService->call($source, $path); - - $file->setBase64(base64_encode($response->getBody())); - $file->setMimeType($response->getHeader('content-type')[0]); - $file->setSize($response->getHeader('content-length')[0]); - $file->setName($document['titel'] ?? $document['url']); - - $explodedFilename = explode('.', $document['titel'] ?? $document['url']); - $file->setExtension(end($explodedFilename)); - $file->setValue($value); - - $this->entityManager->persist($file); - - $bijlageObject->hydrate(['url' => $this->fileService->generateDownloadEndpoint($file->getId()->toString(), $endpoint)]); - - $this->entityManager->persist($bijlageObject); - - $this->entityManager->flush(); - - $data['document'] = $bijlageObject->toArray(); - - return $data; - } - - -}//end class +, Barry Brands . + * @license EUPL + * + * @package CommonGateway\WOOBundle + * @category Service + */ +class SyncOpenWooService +{ + + /** + * @var GatewayResourceService + */ + private GatewayResourceService $resourceService; + + /** + * @var CallService + */ + private CallService $callService; + + /** + * @var SynchronizationService + */ + private SynchronizationService $syncService; + + /** + * @var MappingService + */ + private MappingService $mappingService; + + /** + * @var EntityManagerInterface + */ + private EntityManagerInterface $entityManager; + + /** + * @var SymfonyStyle|null + */ + private ?SymfonyStyle $style = null; + + /** + * @var LoggerInterface $logger. + */ + private LoggerInterface $logger; + + /** + * @var ValidationService $validationService. + */ + private ValidationService $validationService; + + /** + * @var CacheService $cacheService. + */ + private CacheService $cacheService; + + private FileService $fileService; + + private ObjectEntityService $gatewayOEService; + + /** + * @var array + */ + private array $data; + + /** + * @var array + */ + private array $configuration; + + + /** + * SyncOpenWooService constructor. + * + * @param GatewayResourceService $resourceService + * @param CallService $callService + * @param SynchronizationService $syncService + * @param EntityManagerInterface $entityManager + * @param MappingService $mappingService + * @param LoggerInterface $pluginLogger + * @param FileService $fileService + */ + public function __construct( + GatewayResourceService $resourceService, + CallService $callService, + SynchronizationService $syncService, + EntityManagerInterface $entityManager, + MappingService $mappingService, + LoggerInterface $pluginLogger, + ValidationService $validationService, + CacheService $cacheService, + FileService $fileService, + ObjectEntityService $gatewayOEService + ) { + $this->resourceService = $resourceService; + $this->callService = $callService; + $this->syncService = $syncService; + $this->entityManager = $entityManager; + $this->mappingService = $mappingService; + $this->logger = $pluginLogger; + $this->validationService = $validationService; + $this->cacheService = $cacheService; + $this->fileService = $fileService; + $this->gatewayOEService = $gatewayOEService; + + }//end __construct() + + + /** + * Set symfony style in order to output to the console. + * + * @param SymfonyStyle $style + * + * @return self + * + * @todo change to monolog + */ + public function setStyle(SymfonyStyle $style): self + { + $this->style = $style; + + return $this; + + }//end setStyle() + + + /** + * Checks if existing objects still exist in the source, if not deletes them. + * + * @param array $idsSynced ID's from objects we just synced from the source. + * @param Source $source These objects belong to. + * @param string $schemaRef These objects belong to. + * @param string $categorie The categorie these objects came from. + * + * @return int Count of deleted objects. + */ + private function deleteNonExistingObjects(array $idsSynced, Source $source, string $schemaRef, string $categorie): int + { + // Get all existing sourceIds. + $source = $this->entityManager->find('App:Gateway', $source->getId()->toString()); + $existingSourceIds = []; + $existingObjects = []; + foreach ($source->getSynchronizations() as $synchronization) { + if ($synchronization->getEntity()->getReference() === $schemaRef && $synchronization->getSourceId() !== null && $synchronization->getObject()->getValue('categorie') === $categorie) { + $existingSourceIds[] = $synchronization->getSourceId(); + $existingObjects[] = $synchronization->getObject(); + } + } + + // Check if existing sourceIds are in the array of new synced sourceIds. + $objectIdsToDelete = array_diff($existingSourceIds, $idsSynced); + + // If not it means the object does not exist in the source anymore and should be deleted here. + $deletedObjectsCount = 0; + foreach ($objectIdsToDelete as $key => $id) { + $this->logger->info("Object $id does not exist at the source, deleting.", ['plugin' => 'common-gateway/woo-bundle']); + $this->entityManager->remove($existingObjects[$key]); + $deletedObjectsCount++; + } + + $this->entityManager->flush(); + + return $deletedObjectsCount; + + }//end deleteNonExistingObjects() + + + /** + * Fetches objects from openWoo with pagination. + * + * @param Source $source The source entity that provides the source of the result data. + * @param int|null $page The page we are fetching, increments each iteration. + * @param array $results The results from xxllnc api we merge each iteration. + * @param string $categorie The type of object we are fetching. + * + * @return array The fetched objects. + */ + private function fetchObjects(Source $source, ?int $page=1, array $results=[], string $categorie) + { + $response = $this->callService->call($source, $this->configuration['sourceEndpoint'], 'GET', ['query' => ['page' => $page]]); + $decodedResponse = $this->callService->decodeResponse($source, $response); + + switch ($categorie) { + case 'Woo verzoek': + $results = array_merge($results, $decodedResponse['WOOverzoeken']); + break; + case 'Convenant': + $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); + break; + } + + // Pagination xxllnc. + if (isset($decodedResponse['pagination']) === true && $decodedResponse['pagination']['pages']['current'] < $decodedResponse['pagination']['pages']['total']) { + $page++; + $results = $this->fetchObjects($source, $page, $results, $categorie); + } + + return $results; + + }//end fetchObjects() + + + /** + * Handles the synchronization of openwoo objects. + * + * @param array $data + * @param array $configuration + * + * @throws CacheException|InvalidArgumentException + * + * @return array + */ + public function syncOpenWooHandler(array $data, array $configuration): array + { + $this->data = $data; + $this->configuration = $configuration; + + isset($this->style) === true && $this->style->success('SyncOpenWooService triggered'); + $this->logger->info('SyncOpenWooService triggered', ['plugin' => 'common-gateway/woo-bundle']); + + if (isset($this->configuration['source']) === false + || isset($this->configuration['oin']) === false + || isset($this->configuration['organisatie']) === false + || isset($this->configuration['portalUrl']) === false + || isset($this->configuration['schema']) === false + || isset($this->configuration['mapping']) === false + || isset($this->configuration['sourceEndpoint']) === false + ) { + isset($this->style) === true && $this->style->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler'); + $this->logger->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler', ['plugin' => 'common-gateway/woo-bundle']); + + return []; + }//end if + + $source = $this->resourceService->getSource($this->configuration['source'], 'common-gateway/woo-bundle'); + $schema = $this->resourceService->getSchema($this->configuration['schema'], 'common-gateway/woo-bundle'); + $mapping = $this->resourceService->getMapping($this->configuration['mapping'], 'common-gateway/woo-bundle'); + $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.categorie.mapping.json', 'common-gateway/woo-bundle'); + if ($source instanceof Source === false + || $schema instanceof Schema === false + || $mapping instanceof Mapping === false + ) { + isset($this->style) === true && $this->style->error("{$this->configuration['source']}, {$this->configuration['schema']} or {$this->configuration['mapping']} not found, ending syncOpenWooHandler"); + + return []; + }//end if + + $categorie = ''; + switch ($mapping->getReference()) { + case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': + $categorie = 'Woo verzoek'; + break; + case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': + $categorie = 'Convenant'; + break; + } + + isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}"); + $this->logger->info("Fetching objects from {$source->getLocation()}", ['plugin' => 'common-gateway/woo-bundle']); + + $results = $this->fetchObjects($source, 1, [], $categorie); + if (empty($results) === true) { + $this->logger->info('No results found, ending SyncOpenWooService', ['plugin' => 'common-gateway/woo-bundle']); + return $this->data; + } + + $this->entityManager->flush(); + + $customFields = [ + 'organisatie' => [ + 'oin' => $this->configuration['oin'], + 'naam' => $this->configuration['organisatie'], + ], + 'categorie' => $categorie, + ]; + + $idsSynced = []; + $responseItems = []; + $documents = []; + $hydrationService = new HydrationService($this->syncService, $this->entityManager); + foreach ($results as $result) { + try { + $result = array_merge($result, $customFields); + $mappedResult = $this->mappingService->mapping($mapping, $result); + // Map categories to prevent multiple variants of the same categorie. + $mappedResult = $this->mappingService->mapping($categorieMapping, $mappedResult); + + $validationErrors = $this->validationService->validateData($mappedResult, $schema, 'POST'); + if ($validationErrors !== null) { + $validationErrors = implode(', ', $validationErrors); + $this->logger->warning("SyncOpenWoo validation errors: $validationErrors", ['plugin' => 'common-gateway/woo-bundle']); + isset($this->style) === true && $this->style->warning("SyncOpenWoo validation errors: $validationErrors"); + continue; + } + + $object = $hydrationService->searchAndReplaceSynchronizations( + $mappedResult, + $source, + $schema, + true, + true + ); + + // Get all synced sourceIds. + if (empty($object->getSynchronizations()) === false && $object->getSynchronizations()[0]->getSourceId() !== null) { + $idsSynced[] = $object->getSynchronizations()[0]->getSourceId(); + } + + $this->entityManager->persist($object); + $this->cacheService->cacheObject($object); + $responseItems[] = $object; + + $renderedObject = $object->toArray(); + $documents = array_merge($documents, $renderedObject['bijlagen']); + if (isset($renderedObject['metadata']['verzoek']['informatieverzoek']) === true) { + $documents[] = $renderedObject['metadata']['verzoek']['informatieverzoek']; + } + + if (isset($renderedObject['verzoek']['besluit']) === true) { + $documents[] = $renderedObject['metadata']['verzoek']['besluit']; + } + } catch (Exception $exception) { + $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['UUID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle']); + continue; + }//end try + }//end foreach + + $this->entityManager->flush(); + + foreach ($documents as $document) { + $documentData['document'] = $document; + $documentData['source'] = $source->getReference(); + $this->gatewayOEService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); + } + + $deletedObjectsCount = $this->deleteNonExistingObjects($idsSynced, $source, $this->configuration['schema'], $categorie); + + $this->data['response'] = new Response(json_encode($responseItems), 200); + + $countItems = count($responseItems); + $logMessage = "Synchronized $countItems cases to woo objects for ".$source->getName()." and deleted $deletedObjectsCount objects"; + isset($this->style) === true && $this->style->success($logMessage); + $this->logger->info($logMessage, ['plugin' => 'common-gateway/woo-bundle']); + + return $this->data; + + }//end syncOpenWooHandler() + + + public function syncOpenWooDocumentHandler(array $data, array $config): array + { + $source = $this->resourceService->getSource($data['source'], 'common-gateway/woo-bundle'); + $document = $data['document']; + $endpoint = $this->resourceService->getEndpoint($config['endpoint'], 'common-gateway/woo-bundle'); + + if (substr($document['url'], 0, strlen($source->getLocation())) === $source->getLocation()) { + $path = substr($document['url'], strlen($source->getLocation())); + } else { + $this->logger->error('Url of document does not correspond with source'); + + return $data; + } + + $bijlageObject = $this->entityManager->getRepository('App:ObjectEntity')->find($document['_self']['id']); + if ($bijlageObject instanceof ObjectEntity === false) { + return $data; + } + + $value = $bijlageObject->getValueObject('url'); + + if ($value->getFiles()->count() > 0) { + $file = $value->getFiles()->first(); + } else { + $file = new File(); + } + + $response = $this->callService->call($source, $path); + + $file->setBase64(base64_encode($response->getBody())); + $file->setMimeType($response->getHeader('content-type')[0]); + $file->setSize($response->getHeader('content-length')[0]); + $file->setName(($document['titel'] ?? $document['url'])); + + $explodedFilename = explode('.', ($document['titel'] ?? $document['url'])); + $file->setExtension(end($explodedFilename)); + $file->setValue($value); + + $this->entityManager->persist($file); + + $bijlageObject->hydrate(['url' => $this->fileService->generateDownloadEndpoint($file->getId()->toString(), $endpoint)]); + + $this->entityManager->persist($bijlageObject); + + $this->entityManager->flush(); + + $data['document'] = $bijlageObject->toArray(); + + return $data; + + }//end syncOpenWooDocumentHandler() + + +}//end class From 1bc039d6b8847b138e3ee35d9d3fda7650b07dea Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Tue, 30 Jan 2024 16:17:12 +0100 Subject: [PATCH 03/10] fix typo in branch name --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index ed8e19b..2485262 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "minimum-stability": "dev", "require": { "php": ">=7.4", - "commongateway/corebundle": "^1.1.52 | dev-dev-feature/GW-1596/Symfony54" + "commongateway/corebundle": "^1.1.52 | dev-feature/GW-1596/Symfony54" }, "require-dev": { "symfony/dependency-injection": "~3.4|~4.1|~5.0" From 88ca51136dd37ee05b7ee427f747a48e823b51ae Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 31 Jan 2024 13:41:30 +0100 Subject: [PATCH 04/10] Allow symfony 6.4 testbranch --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2485262..9b7bf48 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ "minimum-stability": "dev", "require": { "php": ">=7.4", - "commongateway/corebundle": "^1.1.52 | dev-feature/GW-1596/Symfony54" + "commongateway/corebundle": "^1.1.52 | dev-feature/GW-1596/Symfony54 | dev-feature/GW-1595/Symfony64" }, "require-dev": { "symfony/dependency-injection": "~3.4|~4.1|~5.0" From 40ef723384e9b028d30d5d47ab176c4705bb0fb5 Mon Sep 17 00:00:00 2001 From: GitHub Actions <> Date: Wed, 31 Jan 2024 12:41:55 +0000 Subject: [PATCH 05/10] Update src from PHP Codesniffer --- src/Service/SitemapService.php | 844 ++++++++++++++-------------- src/Service/SyncOpenWooService.php | 860 ++++++++++++++--------------- 2 files changed, 852 insertions(+), 852 deletions(-) diff --git a/src/Service/SitemapService.php b/src/Service/SitemapService.php index ecd2442..6c772a0 100644 --- a/src/Service/SitemapService.php +++ b/src/Service/SitemapService.php @@ -1,422 +1,422 @@ -, Sarai Misidjan , Wilco Louwerse . - * @license EUPL - * - * @package CommonGateway\WOOBundle - * @category Service - */ -class SitemapService -{ - - /** - * @var EntityManagerInterface $entityManager - */ - private EntityManagerInterface $entityManager; - - /** - * @var RequestStack $requestStack - */ - private RequestStack $requestStack; - - /** - * @var LoggerInterface $logger. - */ - private LoggerInterface $logger; - - /** - * @var GatewayResourceService $resourceService. - */ - private GatewayResourceService $resourceService; - - /** - * @var CacheService $cacheService. - */ - private CacheService $cacheService; - - /** - * @var MappingService $mappingService. - */ - private MappingService $mappingService; - - /** - * @var DownloadService $downloadService. - */ - private DownloadService $downloadService; - - /** - * @var ApplicationService $applicationService. - */ - private ApplicationService $applicationService; - - /** - * @var array $data - */ - private array $data; - - /** - * @var array $configuration - */ - private array $configuration; - - - /** - * SitemapService constructor. - * - * @param EntityManagerInterface $entityManager The Entity Manager Interface - * @param RequestStack $requestStack The Request Stack - * @param LoggerInterface $pluginLogger The Logger Interface - * @param GatewayResourceService $resourceService The Gateway Resource Service - * @param CacheService $cacheService The Cache Service - * @param MappingService $mappingService The Mapping Service - * @param DownloadService $downloadService The Download Service - * @param ApplicationService $applicationService The Application Service - */ - public function __construct( - EntityManagerInterface $entityManager, - RequestStack $requestStack, - LoggerInterface $pluginLogger, - GatewayResourceService $resourceService, - CacheService $cacheService, - MappingService $mappingService, - DownloadService $downloadService, - ApplicationService $applicationService - ) { - $this->entityManager = $entityManager; - $this->requestStack = $requestStack; - $this->logger = $pluginLogger; - $this->resourceService = $resourceService; - $this->cacheService = $cacheService; - $this->mappingService = $mappingService; - $this->downloadService = $downloadService; - $this->applicationService = $applicationService; - - }//end __construct() - - - /** - * Generates a sitemap, sitemapindex or robot.txt for the given organization - * - * @param array $data The data passed by the action. - * @param array $configuration The configuration of the action. - * - * @return array - */ - public function sitemapHandler(array $data, array $configuration): array - { - $this->data = $data; - $this->configuration = $configuration; - - // Get the type from the action so that we know what to generate. - if (key_exists('type', $this->configuration) === false) { - $this->logger->error('The type in the configuration of the action is not set.', ['plugin' => 'common-gateway/woo-bundle']); - $this->data['response'] = $this->createResponse(['Message' => 'The type in the configuration of the action is not set.'], 409, 'error'); - return $this->data; - } - - // Get the parameters from the call. This has to be any identification for an organization. - $parameters = array_merge($this->data['path'], $this->data['query']); - - switch ($this->configuration['type']) { - case 'sitemap': - return $this->getSitemap($parameters); - case 'sitemapindex': - return $this->getSitemapindex($parameters); - case 'robot.txt': - return $this->getRobot($parameters); - default: - $this->logger->error('Invalid action configuration type.', ['plugin' => 'common-gateway/woo-bundle']); - } - - $this->data['response'] = $this->createResponse(['Message' => 'Invalid action configuration type.'], 409, 'error'); - return $this->data; - - }//end sitemapHandler() - - - /** - * Finds all subobjects of the type 'bijlage' for a specified object of type 'publicatie'. - * - * @param array $object The 'publicatie' object to find documents for. - * - * @return array The resulting 'bijlage' subobjects. - */ - private function getAllDocumentsForObject(array $object): array - { - $documents = $object['bijlagen']; - - if (isset($object['metadata']['informatieverzoek']['verzoek']) === true) { - $documents[] = $object['metadata']['informatieverzoek']['verzoek']; - } - - if (isset($object['metadata']['informatieverzoek']['besluit']) === true) { - $documents[] = $object['metadata']['informatieverzoek']['besluit']; - } - - return $documents; - - }//end getAllDocumentsForObject() - - - /** - * Generates a sitemap for the given organization - * - * @param array $parameters The parameter array from the request. - * - * @return array Handler data with added 'response'. - */ - private function getSitemap(array $parameters): array - { - // Get the publication schema and the sitemap mapping. - $mapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemap.mapping.json', 'common-gateway/woo-bundle'); - $publicatieSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.publicatie.schema.json', 'common-gateway/woo-bundle'); - if ($publicatieSchema instanceof Schema === false || $mapping instanceof Mapping === false) { - $this->logger->error('The publication schema or the sitemap mapping cannot be found.', ['plugin' => 'common-gateway/woo-bundle']); - $this->data['response'] = $this->createResponse(['Message' => 'The publication schema or the sitemap mapping cannot be found.'], 409, 'error'); - return $this->data; - } - - $filter = array_merge( - $parameters, - [ - 'organisatie.oin' => $parameters['oin'], - '_limit' => 50000, - ] - ); - - $publisherSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.sitemap.schema.json', 'common-gateway/woo-bundle'); - $publishers = $this->cacheService->searchObjects(null, ['oin' => $parameters['oin']], [$publisherSchema->getId()->toString()])['results']; - - unset($filter['oin'], $filter['sitemaps'], $filter['sitemap']); - - // $filter = ['_limit' => 50000]; - // Get all the publication objects with the given query. - $objects = $this->cacheService->searchObjects(null, $filter, [$publicatieSchema->getId()->toString()])['results']; - - $sitemap = []; - foreach ($objects as $object) { - $objectArray = json_decode(json_encode($object), true); - $documents = $this->getAllDocumentsForObject($objectArray); - - $publisher = []; - $publisher['name'] = ($objectArray['organisatie']['naam'] ?? ''); - $publisher['resource'] = $publishers[0]['organisatiecode']; - - $mappedObject = $this->mappingService->mapping($mapping, ['object' => $objectArray, 'documents' => $documents, 'publisher' => $publisher]); - - $sitemap = array_merge_recursive($sitemap, $mappedObject); - } - - // Return the sitemap response. - $this->data['response'] = $this->createResponse($sitemap, 200, 'urlset'); - return $this->data; - - }//end getSitemap() - - - /** - * Generates a sitemapindex for the given organization - * - * @param array $parameters The query array from the request. - * - * @return array Handler data with added 'response'. - */ - private function getSitemapindex(array $parameters): array - { - $mapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.mapping.json', 'common-gateway/woo-bundle'); - $publicatieSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.publicatie.schema.json', 'common-gateway/woo-bundle'); - $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.informatiecategorie.mapping.json', 'common-gateway/woo-bundle'); - - if ($publicatieSchema instanceof Schema === false || $mapping instanceof Mapping === false || $categorieMapping instanceof Mapping === false) { - $this->logger->error('The publication schema, the sitemap index mapping or categorie mapping cannot be found.'); - $this->data['response'] = $this->createResponse(['Message' => 'The publication schema, the sitemap index mapping or categorie mapping cannot be found.'], 409, 'error'); - return $this->data; - } - - $filter = $parameters; - if ($parameters['oin'] === '00000000000000000000') { - $filter = array_merge($filter, ['organisatie.oin' => $parameters['oin']]); - } - - unset($filter['oin']); - - $categorieStr = ''; - if (isset($parameters['sitemapindex']) === true) { - $categorie = $this->mappingService->mapping($categorieMapping, [$parameters['sitemapindex'] => '']); - $categorieDot = new Dot($categorie); - - if ($categorieDot->has($parameters['sitemapindex']) === false) { - $this->logger->error('Invalid informatiecategorie.'); - $this->data['response'] = $this->createResponse(['Message' => 'Invalid informatiecategorie.'], 400, 'error'); - return $this->data; - } - - $filter['categorie'] = $categorieDot->get($parameters['sitemapindex']); - $categorieStr = 'categorie='.urlencode($categorieDot->get($parameters['sitemapindex'])); - unset($filter['sitemapindex']); - } - - // Count all the publication objects with the given query. - $count = $this->cacheService->countObjects(null, $filter, [$publicatieSchema->getId()->toString()]); - $pages = ((int) (($count - 1) / 50000) + 1); - - // Get the domain of the request. - $domain = $this->requestStack->getMainRequest()->getSchemeAndHttpHost(); - - $sitemapindex = []; - for ($i = 1; $i <= $pages; $i++) { - // The location of the sitemap file is the endpoint of the sitemap. - $location['location'] = $this->nonAsciiUrlEncode( - $domain.'/api/sitemaps/'.$parameters['oin'].'/sitemap?'.$categorieStr.'&_page='.$i - ); - $sitemapindex['sitemap'][] = $this->mappingService->mapping($mapping, $location); - } - - // Return the sitemapindex response. - $this->data['response'] = $this->createResponse($sitemapindex, 200, 'sitemapindex'); - return $this->data; - - }//end getSitemapindex() - - - /** - * Generates a robot.txt for the given organization - * - * @param array $parameters The query array from the request. - * - * @return array Handler data with added 'response'. - */ - private function getRobot(array $parameters): array - { - - $sitemapSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.sitemap.schema.json', 'common-gateway/woo-bundle'); - $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.informatiecategorie.mapping.json', 'common-gateway/woo-bundle'); - if ($sitemapSchema instanceof Schema === false || $categorieMapping instanceof Mapping === false) { - $this->logger->error('The sitemap schema or categorie mapping cannot be found.', ['plugin' => 'common-gateway/woo-bundle']); - $this->data['response'] = $this->createResponse(['Message' => 'The sitemap schema or categorie mapping cannot be found.'], 409, 'error'); - return $this->data; - } - - $host = $this->requestStack->getMainRequest()->getHost(); - - $sitemaps = $this->cacheService->searchObjects(null, ['domains' => $host], [$sitemapSchema->getId()->toString()]); - - if (count($sitemaps['results']) === 1) { - $oin = $sitemaps['results'][0]['oin']; - } else if (count($sitemaps['results']) > 1) { - $oin = '00000000000000000000'; - if (isset($parameters['oin']) === true) { - $oin = $parameters['oin']; - } - } else { - $this->logger->warning('No oin found for this domain, returning no sitemaps'); - return $this->data; - } - - $categories = array_keys($categorieMapping->getMapping()); - - // Get the domain of the request. - $domain = $this->requestStack->getMainRequest()->getSchemeAndHttpHost(); - - foreach ($categories as $category) { - // The location of the robot.txt file is the endpoint of the sitemapindex. - $robotArray['locations'][] = $this->nonAsciiUrlEncode( - $domain.'/api/sitemaps/'.$oin.'/'.$category, - false - ); - } - - // Set the id of the schema to the array so that the downloadService can work with that. - $robotArray['_self']['schema']['id'] = $sitemapSchema->getId()->toString(); - $robot = $this->downloadService->render($robotArray); - - $this->data['response'] = new Response($robot, 200, ['Content-Type' => 'text/plain']); - $this->data['response']->headers->set('Content-Disposition', 'attachment; filename="Robot.txt"'); - - return $this->data; - - }//end getRobot() - - - /** - * URL encodes all characters in a string that are non ASCII characters. - * And does a htmlspecialchars() on $str after that unless $htmlspecialchars is set to false. - * - * @param string $str The input string. - * @param bool $htmlspecialchars True by default, if set to false htmlspecialchars() will not be used on $str. - * - * @return string The updated string. - */ - private function nonAsciiUrlEncode(string $str, bool $htmlspecialchars=true): string - { - $str = preg_replace_callback( - '/[^\x20-\x7e]/', - function ($matches) { - return urlencode($matches[0]); - }, - $str - ); - - if ($htmlspecialchars === false) { - return $str; - } - - return htmlspecialchars($str); - - }//end nonAsciiUrlEncode() - - - /** - * Creates a response based on content. - * - * @param array $content The content to incorporate in the response - * @param int $status The status code of the response - * @param string $rootName The rootName of the xml. - * - * @return Response - */ - private function createResponse(array $content, int $status, string $rootName): Response - { - $this->logger->debug('Creating XML response', ['plugin' => 'common-gateway/woo-bundle']); - $xmlEncoder = new XmlEncoder(['xml_root_node_name' => $rootName]); - $xml = ['@xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9']; - $content = array_merge($xml, $content); - - $contentString = $xmlEncoder->encode($content, 'xml', ['xml_encoding' => 'utf-8', 'remove_empty_tags' => true]); - - // Remove CDATA - $contentString = str_replace([""], "", $contentString); - - return new Response($contentString, $status); - - }//end createResponse() - - -}//end class +, Sarai Misidjan , Wilco Louwerse . + * @license EUPL + * + * @package CommonGateway\WOOBundle + * @category Service + */ +class SitemapService +{ + + /** + * @var EntityManagerInterface $entityManager + */ + private EntityManagerInterface $entityManager; + + /** + * @var RequestStack $requestStack + */ + private RequestStack $requestStack; + + /** + * @var LoggerInterface $logger. + */ + private LoggerInterface $logger; + + /** + * @var GatewayResourceService $resourceService. + */ + private GatewayResourceService $resourceService; + + /** + * @var CacheService $cacheService. + */ + private CacheService $cacheService; + + /** + * @var MappingService $mappingService. + */ + private MappingService $mappingService; + + /** + * @var DownloadService $downloadService. + */ + private DownloadService $downloadService; + + /** + * @var ApplicationService $applicationService. + */ + private ApplicationService $applicationService; + + /** + * @var array $data + */ + private array $data; + + /** + * @var array $configuration + */ + private array $configuration; + + + /** + * SitemapService constructor. + * + * @param EntityManagerInterface $entityManager The Entity Manager Interface + * @param RequestStack $requestStack The Request Stack + * @param LoggerInterface $pluginLogger The Logger Interface + * @param GatewayResourceService $resourceService The Gateway Resource Service + * @param CacheService $cacheService The Cache Service + * @param MappingService $mappingService The Mapping Service + * @param DownloadService $downloadService The Download Service + * @param ApplicationService $applicationService The Application Service + */ + public function __construct( + EntityManagerInterface $entityManager, + RequestStack $requestStack, + LoggerInterface $pluginLogger, + GatewayResourceService $resourceService, + CacheService $cacheService, + MappingService $mappingService, + DownloadService $downloadService, + ApplicationService $applicationService + ) { + $this->entityManager = $entityManager; + $this->requestStack = $requestStack; + $this->logger = $pluginLogger; + $this->resourceService = $resourceService; + $this->cacheService = $cacheService; + $this->mappingService = $mappingService; + $this->downloadService = $downloadService; + $this->applicationService = $applicationService; + + }//end __construct() + + + /** + * Generates a sitemap, sitemapindex or robot.txt for the given organization + * + * @param array $data The data passed by the action. + * @param array $configuration The configuration of the action. + * + * @return array + */ + public function sitemapHandler(array $data, array $configuration): array + { + $this->data = $data; + $this->configuration = $configuration; + + // Get the type from the action so that we know what to generate. + if (key_exists('type', $this->configuration) === false) { + $this->logger->error('The type in the configuration of the action is not set.', ['plugin' => 'common-gateway/woo-bundle']); + $this->data['response'] = $this->createResponse(['Message' => 'The type in the configuration of the action is not set.'], 409, 'error'); + return $this->data; + } + + // Get the parameters from the call. This has to be any identification for an organization. + $parameters = array_merge($this->data['path'], $this->data['query']); + + switch ($this->configuration['type']) { + case 'sitemap': + return $this->getSitemap($parameters); + case 'sitemapindex': + return $this->getSitemapindex($parameters); + case 'robot.txt': + return $this->getRobot($parameters); + default: + $this->logger->error('Invalid action configuration type.', ['plugin' => 'common-gateway/woo-bundle']); + } + + $this->data['response'] = $this->createResponse(['Message' => 'Invalid action configuration type.'], 409, 'error'); + return $this->data; + + }//end sitemapHandler() + + + /** + * Finds all subobjects of the type 'bijlage' for a specified object of type 'publicatie'. + * + * @param array $object The 'publicatie' object to find documents for. + * + * @return array The resulting 'bijlage' subobjects. + */ + private function getAllDocumentsForObject(array $object): array + { + $documents = $object['bijlagen']; + + if (isset($object['metadata']['informatieverzoek']['verzoek']) === true) { + $documents[] = $object['metadata']['informatieverzoek']['verzoek']; + } + + if (isset($object['metadata']['informatieverzoek']['besluit']) === true) { + $documents[] = $object['metadata']['informatieverzoek']['besluit']; + } + + return $documents; + + }//end getAllDocumentsForObject() + + + /** + * Generates a sitemap for the given organization + * + * @param array $parameters The parameter array from the request. + * + * @return array Handler data with added 'response'. + */ + private function getSitemap(array $parameters): array + { + // Get the publication schema and the sitemap mapping. + $mapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemap.mapping.json', 'common-gateway/woo-bundle'); + $publicatieSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.publicatie.schema.json', 'common-gateway/woo-bundle'); + if ($publicatieSchema instanceof Schema === false || $mapping instanceof Mapping === false) { + $this->logger->error('The publication schema or the sitemap mapping cannot be found.', ['plugin' => 'common-gateway/woo-bundle']); + $this->data['response'] = $this->createResponse(['Message' => 'The publication schema or the sitemap mapping cannot be found.'], 409, 'error'); + return $this->data; + } + + $filter = array_merge( + $parameters, + [ + 'organisatie.oin' => $parameters['oin'], + '_limit' => 50000, + ] + ); + + $publisherSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.sitemap.schema.json', 'common-gateway/woo-bundle'); + $publishers = $this->cacheService->searchObjects(null, ['oin' => $parameters['oin']], [$publisherSchema->getId()->toString()])['results']; + + unset($filter['oin'], $filter['sitemaps'], $filter['sitemap']); + + // $filter = ['_limit' => 50000]; + // Get all the publication objects with the given query. + $objects = $this->cacheService->searchObjects(null, $filter, [$publicatieSchema->getId()->toString()])['results']; + + $sitemap = []; + foreach ($objects as $object) { + $objectArray = json_decode(json_encode($object), true); + $documents = $this->getAllDocumentsForObject($objectArray); + + $publisher = []; + $publisher['name'] = ($objectArray['organisatie']['naam'] ?? ''); + $publisher['resource'] = $publishers[0]['organisatiecode']; + + $mappedObject = $this->mappingService->mapping($mapping, ['object' => $objectArray, 'documents' => $documents, 'publisher' => $publisher]); + + $sitemap = array_merge_recursive($sitemap, $mappedObject); + } + + // Return the sitemap response. + $this->data['response'] = $this->createResponse($sitemap, 200, 'urlset'); + return $this->data; + + }//end getSitemap() + + + /** + * Generates a sitemapindex for the given organization + * + * @param array $parameters The query array from the request. + * + * @return array Handler data with added 'response'. + */ + private function getSitemapindex(array $parameters): array + { + $mapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.mapping.json', 'common-gateway/woo-bundle'); + $publicatieSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.publicatie.schema.json', 'common-gateway/woo-bundle'); + $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.informatiecategorie.mapping.json', 'common-gateway/woo-bundle'); + + if ($publicatieSchema instanceof Schema === false || $mapping instanceof Mapping === false || $categorieMapping instanceof Mapping === false) { + $this->logger->error('The publication schema, the sitemap index mapping or categorie mapping cannot be found.'); + $this->data['response'] = $this->createResponse(['Message' => 'The publication schema, the sitemap index mapping or categorie mapping cannot be found.'], 409, 'error'); + return $this->data; + } + + $filter = $parameters; + if ($parameters['oin'] === '00000000000000000000') { + $filter = array_merge($filter, ['organisatie.oin' => $parameters['oin']]); + } + + unset($filter['oin']); + + $categorieStr = ''; + if (isset($parameters['sitemapindex']) === true) { + $categorie = $this->mappingService->mapping($categorieMapping, [$parameters['sitemapindex'] => '']); + $categorieDot = new Dot($categorie); + + if ($categorieDot->has($parameters['sitemapindex']) === false) { + $this->logger->error('Invalid informatiecategorie.'); + $this->data['response'] = $this->createResponse(['Message' => 'Invalid informatiecategorie.'], 400, 'error'); + return $this->data; + } + + $filter['categorie'] = $categorieDot->get($parameters['sitemapindex']); + $categorieStr = 'categorie='.urlencode($categorieDot->get($parameters['sitemapindex'])); + unset($filter['sitemapindex']); + } + + // Count all the publication objects with the given query. + $count = $this->cacheService->countObjects(null, $filter, [$publicatieSchema->getId()->toString()]); + $pages = ((int) (($count - 1) / 50000) + 1); + + // Get the domain of the request. + $domain = $this->requestStack->getMainRequest()->getSchemeAndHttpHost(); + + $sitemapindex = []; + for ($i = 1; $i <= $pages; $i++) { + // The location of the sitemap file is the endpoint of the sitemap. + $location['location'] = $this->nonAsciiUrlEncode( + $domain.'/api/sitemaps/'.$parameters['oin'].'/sitemap?'.$categorieStr.'&_page='.$i + ); + $sitemapindex['sitemap'][] = $this->mappingService->mapping($mapping, $location); + } + + // Return the sitemapindex response. + $this->data['response'] = $this->createResponse($sitemapindex, 200, 'sitemapindex'); + return $this->data; + + }//end getSitemapindex() + + + /** + * Generates a robot.txt for the given organization + * + * @param array $parameters The query array from the request. + * + * @return array Handler data with added 'response'. + */ + private function getRobot(array $parameters): array + { + + $sitemapSchema = $this->resourceService->getSchema('https://commongateway.nl/woo.sitemap.schema.json', 'common-gateway/woo-bundle'); + $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.sitemapindex.informatiecategorie.mapping.json', 'common-gateway/woo-bundle'); + if ($sitemapSchema instanceof Schema === false || $categorieMapping instanceof Mapping === false) { + $this->logger->error('The sitemap schema or categorie mapping cannot be found.', ['plugin' => 'common-gateway/woo-bundle']); + $this->data['response'] = $this->createResponse(['Message' => 'The sitemap schema or categorie mapping cannot be found.'], 409, 'error'); + return $this->data; + } + + $host = $this->requestStack->getMainRequest()->getHost(); + + $sitemaps = $this->cacheService->searchObjects(null, ['domains' => $host], [$sitemapSchema->getId()->toString()]); + + if (count($sitemaps['results']) === 1) { + $oin = $sitemaps['results'][0]['oin']; + } else if (count($sitemaps['results']) > 1) { + $oin = '00000000000000000000'; + if (isset($parameters['oin']) === true) { + $oin = $parameters['oin']; + } + } else { + $this->logger->warning('No oin found for this domain, returning no sitemaps'); + return $this->data; + } + + $categories = array_keys($categorieMapping->getMapping()); + + // Get the domain of the request. + $domain = $this->requestStack->getMainRequest()->getSchemeAndHttpHost(); + + foreach ($categories as $category) { + // The location of the robot.txt file is the endpoint of the sitemapindex. + $robotArray['locations'][] = $this->nonAsciiUrlEncode( + $domain.'/api/sitemaps/'.$oin.'/'.$category, + false + ); + } + + // Set the id of the schema to the array so that the downloadService can work with that. + $robotArray['_self']['schema']['id'] = $sitemapSchema->getId()->toString(); + $robot = $this->downloadService->render($robotArray); + + $this->data['response'] = new Response($robot, 200, ['Content-Type' => 'text/plain']); + $this->data['response']->headers->set('Content-Disposition', 'attachment; filename="Robot.txt"'); + + return $this->data; + + }//end getRobot() + + + /** + * URL encodes all characters in a string that are non ASCII characters. + * And does a htmlspecialchars() on $str after that unless $htmlspecialchars is set to false. + * + * @param string $str The input string. + * @param bool $htmlspecialchars True by default, if set to false htmlspecialchars() will not be used on $str. + * + * @return string The updated string. + */ + private function nonAsciiUrlEncode(string $str, bool $htmlspecialchars=true): string + { + $str = preg_replace_callback( + '/[^\x20-\x7e]/', + function ($matches) { + return urlencode($matches[0]); + }, + $str + ); + + if ($htmlspecialchars === false) { + return $str; + } + + return htmlspecialchars($str); + + }//end nonAsciiUrlEncode() + + + /** + * Creates a response based on content. + * + * @param array $content The content to incorporate in the response + * @param int $status The status code of the response + * @param string $rootName The rootName of the xml. + * + * @return Response + */ + private function createResponse(array $content, int $status, string $rootName): Response + { + $this->logger->debug('Creating XML response', ['plugin' => 'common-gateway/woo-bundle']); + $xmlEncoder = new XmlEncoder(['xml_root_node_name' => $rootName]); + $xml = ['@xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9']; + $content = array_merge($xml, $content); + + $contentString = $xmlEncoder->encode($content, 'xml', ['xml_encoding' => 'utf-8', 'remove_empty_tags' => true]); + + // Remove CDATA + $contentString = str_replace([""], "", $contentString); + + return new Response($contentString, $status); + + }//end createResponse() + + +}//end class diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index 7bf2544..0e82a93 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -1,430 +1,430 @@ -, Barry Brands . - * @license EUPL - * - * @package CommonGateway\WOOBundle - * @category Service - */ -class SyncOpenWooService -{ - - /** - * @var GatewayResourceService - */ - private GatewayResourceService $resourceService; - - /** - * @var CallService - */ - private CallService $callService; - - /** - * @var SynchronizationService - */ - private SynchronizationService $syncService; - - /** - * @var MappingService - */ - private MappingService $mappingService; - - /** - * @var EntityManagerInterface - */ - private EntityManagerInterface $entityManager; - - /** - * @var SymfonyStyle|null - */ - private ?SymfonyStyle $style = null; - - /** - * @var LoggerInterface $logger. - */ - private LoggerInterface $logger; - - /** - * @var ValidationService $validationService. - */ - private ValidationService $validationService; - - /** - * @var CacheService $cacheService. - */ - private CacheService $cacheService; - - private FileService $fileService; - - private ObjectEntityService $gatewayOEService; - - /** - * @var array - */ - private array $data; - - /** - * @var array - */ - private array $configuration; - - - /** - * SyncOpenWooService constructor. - * - * @param GatewayResourceService $resourceService - * @param CallService $callService - * @param SynchronizationService $syncService - * @param EntityManagerInterface $entityManager - * @param MappingService $mappingService - * @param LoggerInterface $pluginLogger - * @param FileService $fileService - */ - public function __construct( - GatewayResourceService $resourceService, - CallService $callService, - SynchronizationService $syncService, - EntityManagerInterface $entityManager, - MappingService $mappingService, - LoggerInterface $pluginLogger, - ValidationService $validationService, - CacheService $cacheService, - FileService $fileService, - ObjectEntityService $gatewayOEService - ) { - $this->resourceService = $resourceService; - $this->callService = $callService; - $this->syncService = $syncService; - $this->entityManager = $entityManager; - $this->mappingService = $mappingService; - $this->logger = $pluginLogger; - $this->validationService = $validationService; - $this->cacheService = $cacheService; - $this->fileService = $fileService; - $this->gatewayOEService = $gatewayOEService; - - }//end __construct() - - - /** - * Set symfony style in order to output to the console. - * - * @param SymfonyStyle $style - * - * @return self - * - * @todo change to monolog - */ - public function setStyle(SymfonyStyle $style): self - { - $this->style = $style; - - return $this; - - }//end setStyle() - - - /** - * Checks if existing objects still exist in the source, if not deletes them. - * - * @param array $idsSynced ID's from objects we just synced from the source. - * @param Source $source These objects belong to. - * @param string $schemaRef These objects belong to. - * @param string $categorie The categorie these objects came from. - * - * @return int Count of deleted objects. - */ - private function deleteNonExistingObjects(array $idsSynced, Source $source, string $schemaRef, string $categorie): int - { - // Get all existing sourceIds. - $source = $this->entityManager->find('App:Gateway', $source->getId()->toString()); - $existingSourceIds = []; - $existingObjects = []; - foreach ($source->getSynchronizations() as $synchronization) { - if ($synchronization->getEntity()->getReference() === $schemaRef && $synchronization->getSourceId() !== null && $synchronization->getObject()->getValue('categorie') === $categorie) { - $existingSourceIds[] = $synchronization->getSourceId(); - $existingObjects[] = $synchronization->getObject(); - } - } - - // Check if existing sourceIds are in the array of new synced sourceIds. - $objectIdsToDelete = array_diff($existingSourceIds, $idsSynced); - - // If not it means the object does not exist in the source anymore and should be deleted here. - $deletedObjectsCount = 0; - foreach ($objectIdsToDelete as $key => $id) { - $this->logger->info("Object $id does not exist at the source, deleting.", ['plugin' => 'common-gateway/woo-bundle']); - $this->entityManager->remove($existingObjects[$key]); - $deletedObjectsCount++; - } - - $this->entityManager->flush(); - - return $deletedObjectsCount; - - }//end deleteNonExistingObjects() - - - /** - * Fetches objects from openWoo with pagination. - * - * @param Source $source The source entity that provides the source of the result data. - * @param int|null $page The page we are fetching, increments each iteration. - * @param array $results The results from xxllnc api we merge each iteration. - * @param string $categorie The type of object we are fetching. - * - * @return array The fetched objects. - */ - private function fetchObjects(Source $source, ?int $page=1, array $results=[], string $categorie) - { - $response = $this->callService->call($source, $this->configuration['sourceEndpoint'], 'GET', ['query' => ['page' => $page]]); - $decodedResponse = $this->callService->decodeResponse($source, $response); - - switch ($categorie) { - case 'Woo verzoek': - $results = array_merge($results, $decodedResponse['WOOverzoeken']); - break; - case 'Convenant': - $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); - break; - } - - // Pagination xxllnc. - if (isset($decodedResponse['pagination']) === true && $decodedResponse['pagination']['pages']['current'] < $decodedResponse['pagination']['pages']['total']) { - $page++; - $results = $this->fetchObjects($source, $page, $results, $categorie); - } - - return $results; - - }//end fetchObjects() - - - /** - * Handles the synchronization of openwoo objects. - * - * @param array $data - * @param array $configuration - * - * @throws CacheException|InvalidArgumentException - * - * @return array - */ - public function syncOpenWooHandler(array $data, array $configuration): array - { - $this->data = $data; - $this->configuration = $configuration; - - isset($this->style) === true && $this->style->success('SyncOpenWooService triggered'); - $this->logger->info('SyncOpenWooService triggered', ['plugin' => 'common-gateway/woo-bundle']); - - if (isset($this->configuration['source']) === false - || isset($this->configuration['oin']) === false - || isset($this->configuration['organisatie']) === false - || isset($this->configuration['portalUrl']) === false - || isset($this->configuration['schema']) === false - || isset($this->configuration['mapping']) === false - || isset($this->configuration['sourceEndpoint']) === false - ) { - isset($this->style) === true && $this->style->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler'); - $this->logger->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler', ['plugin' => 'common-gateway/woo-bundle']); - - return []; - }//end if - - $source = $this->resourceService->getSource($this->configuration['source'], 'common-gateway/woo-bundle'); - $schema = $this->resourceService->getSchema($this->configuration['schema'], 'common-gateway/woo-bundle'); - $mapping = $this->resourceService->getMapping($this->configuration['mapping'], 'common-gateway/woo-bundle'); - $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.categorie.mapping.json', 'common-gateway/woo-bundle'); - if ($source instanceof Source === false - || $schema instanceof Schema === false - || $mapping instanceof Mapping === false - ) { - isset($this->style) === true && $this->style->error("{$this->configuration['source']}, {$this->configuration['schema']} or {$this->configuration['mapping']} not found, ending syncOpenWooHandler"); - - return []; - }//end if - - $categorie = ''; - switch ($mapping->getReference()) { - case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': - $categorie = 'Woo verzoek'; - break; - case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': - $categorie = 'Convenant'; - break; - } - - isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}"); - $this->logger->info("Fetching objects from {$source->getLocation()}", ['plugin' => 'common-gateway/woo-bundle']); - - $results = $this->fetchObjects($source, 1, [], $categorie); - if (empty($results) === true) { - $this->logger->info('No results found, ending SyncOpenWooService', ['plugin' => 'common-gateway/woo-bundle']); - return $this->data; - } - - $this->entityManager->flush(); - - $customFields = [ - 'organisatie' => [ - 'oin' => $this->configuration['oin'], - 'naam' => $this->configuration['organisatie'], - ], - 'categorie' => $categorie, - ]; - - $idsSynced = []; - $responseItems = []; - $documents = []; - $hydrationService = new HydrationService($this->syncService, $this->entityManager); - foreach ($results as $result) { - try { - $result = array_merge($result, $customFields); - $mappedResult = $this->mappingService->mapping($mapping, $result); - // Map categories to prevent multiple variants of the same categorie. - $mappedResult = $this->mappingService->mapping($categorieMapping, $mappedResult); - - $validationErrors = $this->validationService->validateData($mappedResult, $schema, 'POST'); - if ($validationErrors !== null) { - $validationErrors = implode(', ', $validationErrors); - $this->logger->warning("SyncOpenWoo validation errors: $validationErrors", ['plugin' => 'common-gateway/woo-bundle']); - isset($this->style) === true && $this->style->warning("SyncOpenWoo validation errors: $validationErrors"); - continue; - } - - $object = $hydrationService->searchAndReplaceSynchronizations( - $mappedResult, - $source, - $schema, - true, - true - ); - - // Get all synced sourceIds. - if (empty($object->getSynchronizations()) === false && $object->getSynchronizations()[0]->getSourceId() !== null) { - $idsSynced[] = $object->getSynchronizations()[0]->getSourceId(); - } - - $this->entityManager->persist($object); - $this->cacheService->cacheObject($object); - $responseItems[] = $object; - - $renderedObject = $object->toArray(); - $documents = array_merge($documents, $renderedObject['bijlagen']); - if (isset($renderedObject['metadata']['verzoek']['informatieverzoek']) === true) { - $documents[] = $renderedObject['metadata']['verzoek']['informatieverzoek']; - } - - if (isset($renderedObject['verzoek']['besluit']) === true) { - $documents[] = $renderedObject['metadata']['verzoek']['besluit']; - } - } catch (Exception $exception) { - $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['UUID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle']); - continue; - }//end try - }//end foreach - - $this->entityManager->flush(); - - foreach ($documents as $document) { - $documentData['document'] = $document; - $documentData['source'] = $source->getReference(); - $this->gatewayOEService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); - } - - $deletedObjectsCount = $this->deleteNonExistingObjects($idsSynced, $source, $this->configuration['schema'], $categorie); - - $this->data['response'] = new Response(json_encode($responseItems), 200); - - $countItems = count($responseItems); - $logMessage = "Synchronized $countItems cases to woo objects for ".$source->getName()." and deleted $deletedObjectsCount objects"; - isset($this->style) === true && $this->style->success($logMessage); - $this->logger->info($logMessage, ['plugin' => 'common-gateway/woo-bundle']); - - return $this->data; - - }//end syncOpenWooHandler() - - - public function syncOpenWooDocumentHandler(array $data, array $config): array - { - $source = $this->resourceService->getSource($data['source'], 'common-gateway/woo-bundle'); - $document = $data['document']; - $endpoint = $this->resourceService->getEndpoint($config['endpoint'], 'common-gateway/woo-bundle'); - - if (substr($document['url'], 0, strlen($source->getLocation())) === $source->getLocation()) { - $path = substr($document['url'], strlen($source->getLocation())); - } else { - $this->logger->error('Url of document does not correspond with source'); - - return $data; - } - - $bijlageObject = $this->entityManager->getRepository('App:ObjectEntity')->find($document['_self']['id']); - if ($bijlageObject instanceof ObjectEntity === false) { - return $data; - } - - $value = $bijlageObject->getValueObject('url'); - - if ($value->getFiles()->count() > 0) { - $file = $value->getFiles()->first(); - } else { - $file = new File(); - } - - $response = $this->callService->call($source, $path); - - $file->setBase64(base64_encode($response->getBody())); - $file->setMimeType($response->getHeader('content-type')[0]); - $file->setSize($response->getHeader('content-length')[0]); - $file->setName(($document['titel'] ?? $document['url'])); - - $explodedFilename = explode('.', ($document['titel'] ?? $document['url'])); - $file->setExtension(end($explodedFilename)); - $file->setValue($value); - - $this->entityManager->persist($file); - - $bijlageObject->hydrate(['url' => $this->fileService->generateDownloadEndpoint($file->getId()->toString(), $endpoint), 'extension' => end($explodedFilename)]); - - $this->entityManager->persist($bijlageObject); - - $this->entityManager->flush(); - - $data['document'] = $bijlageObject->toArray(); - - return $data; - - }//end syncOpenWooDocumentHandler() - - -}//end class +, Barry Brands . + * @license EUPL + * + * @package CommonGateway\WOOBundle + * @category Service + */ +class SyncOpenWooService +{ + + /** + * @var GatewayResourceService + */ + private GatewayResourceService $resourceService; + + /** + * @var CallService + */ + private CallService $callService; + + /** + * @var SynchronizationService + */ + private SynchronizationService $syncService; + + /** + * @var MappingService + */ + private MappingService $mappingService; + + /** + * @var EntityManagerInterface + */ + private EntityManagerInterface $entityManager; + + /** + * @var SymfonyStyle|null + */ + private ?SymfonyStyle $style = null; + + /** + * @var LoggerInterface $logger. + */ + private LoggerInterface $logger; + + /** + * @var ValidationService $validationService. + */ + private ValidationService $validationService; + + /** + * @var CacheService $cacheService. + */ + private CacheService $cacheService; + + private FileService $fileService; + + private ObjectEntityService $gatewayOEService; + + /** + * @var array + */ + private array $data; + + /** + * @var array + */ + private array $configuration; + + + /** + * SyncOpenWooService constructor. + * + * @param GatewayResourceService $resourceService + * @param CallService $callService + * @param SynchronizationService $syncService + * @param EntityManagerInterface $entityManager + * @param MappingService $mappingService + * @param LoggerInterface $pluginLogger + * @param FileService $fileService + */ + public function __construct( + GatewayResourceService $resourceService, + CallService $callService, + SynchronizationService $syncService, + EntityManagerInterface $entityManager, + MappingService $mappingService, + LoggerInterface $pluginLogger, + ValidationService $validationService, + CacheService $cacheService, + FileService $fileService, + ObjectEntityService $gatewayOEService + ) { + $this->resourceService = $resourceService; + $this->callService = $callService; + $this->syncService = $syncService; + $this->entityManager = $entityManager; + $this->mappingService = $mappingService; + $this->logger = $pluginLogger; + $this->validationService = $validationService; + $this->cacheService = $cacheService; + $this->fileService = $fileService; + $this->gatewayOEService = $gatewayOEService; + + }//end __construct() + + + /** + * Set symfony style in order to output to the console. + * + * @param SymfonyStyle $style + * + * @return self + * + * @todo change to monolog + */ + public function setStyle(SymfonyStyle $style): self + { + $this->style = $style; + + return $this; + + }//end setStyle() + + + /** + * Checks if existing objects still exist in the source, if not deletes them. + * + * @param array $idsSynced ID's from objects we just synced from the source. + * @param Source $source These objects belong to. + * @param string $schemaRef These objects belong to. + * @param string $categorie The categorie these objects came from. + * + * @return int Count of deleted objects. + */ + private function deleteNonExistingObjects(array $idsSynced, Source $source, string $schemaRef, string $categorie): int + { + // Get all existing sourceIds. + $source = $this->entityManager->find('App:Gateway', $source->getId()->toString()); + $existingSourceIds = []; + $existingObjects = []; + foreach ($source->getSynchronizations() as $synchronization) { + if ($synchronization->getEntity()->getReference() === $schemaRef && $synchronization->getSourceId() !== null && $synchronization->getObject()->getValue('categorie') === $categorie) { + $existingSourceIds[] = $synchronization->getSourceId(); + $existingObjects[] = $synchronization->getObject(); + } + } + + // Check if existing sourceIds are in the array of new synced sourceIds. + $objectIdsToDelete = array_diff($existingSourceIds, $idsSynced); + + // If not it means the object does not exist in the source anymore and should be deleted here. + $deletedObjectsCount = 0; + foreach ($objectIdsToDelete as $key => $id) { + $this->logger->info("Object $id does not exist at the source, deleting.", ['plugin' => 'common-gateway/woo-bundle']); + $this->entityManager->remove($existingObjects[$key]); + $deletedObjectsCount++; + } + + $this->entityManager->flush(); + + return $deletedObjectsCount; + + }//end deleteNonExistingObjects() + + + /** + * Fetches objects from openWoo with pagination. + * + * @param Source $source The source entity that provides the source of the result data. + * @param int|null $page The page we are fetching, increments each iteration. + * @param array $results The results from xxllnc api we merge each iteration. + * @param string $categorie The type of object we are fetching. + * + * @return array The fetched objects. + */ + private function fetchObjects(Source $source, ?int $page=1, array $results=[], string $categorie) + { + $response = $this->callService->call($source, $this->configuration['sourceEndpoint'], 'GET', ['query' => ['page' => $page]]); + $decodedResponse = $this->callService->decodeResponse($source, $response); + + switch ($categorie) { + case 'Woo verzoek': + $results = array_merge($results, $decodedResponse['WOOverzoeken']); + break; + case 'Convenant': + $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); + break; + } + + // Pagination xxllnc. + if (isset($decodedResponse['pagination']) === true && $decodedResponse['pagination']['pages']['current'] < $decodedResponse['pagination']['pages']['total']) { + $page++; + $results = $this->fetchObjects($source, $page, $results, $categorie); + } + + return $results; + + }//end fetchObjects() + + + /** + * Handles the synchronization of openwoo objects. + * + * @param array $data + * @param array $configuration + * + * @throws CacheException|InvalidArgumentException + * + * @return array + */ + public function syncOpenWooHandler(array $data, array $configuration): array + { + $this->data = $data; + $this->configuration = $configuration; + + isset($this->style) === true && $this->style->success('SyncOpenWooService triggered'); + $this->logger->info('SyncOpenWooService triggered', ['plugin' => 'common-gateway/woo-bundle']); + + if (isset($this->configuration['source']) === false + || isset($this->configuration['oin']) === false + || isset($this->configuration['organisatie']) === false + || isset($this->configuration['portalUrl']) === false + || isset($this->configuration['schema']) === false + || isset($this->configuration['mapping']) === false + || isset($this->configuration['sourceEndpoint']) === false + ) { + isset($this->style) === true && $this->style->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler'); + $this->logger->error('No source, schema, mapping, oin, organisatie, sourceEndpoint or portalUrl configured on this action, ending syncOpenWooHandler', ['plugin' => 'common-gateway/woo-bundle']); + + return []; + }//end if + + $source = $this->resourceService->getSource($this->configuration['source'], 'common-gateway/woo-bundle'); + $schema = $this->resourceService->getSchema($this->configuration['schema'], 'common-gateway/woo-bundle'); + $mapping = $this->resourceService->getMapping($this->configuration['mapping'], 'common-gateway/woo-bundle'); + $categorieMapping = $this->resourceService->getMapping('https://commongateway.nl/mapping/woo.categorie.mapping.json', 'common-gateway/woo-bundle'); + if ($source instanceof Source === false + || $schema instanceof Schema === false + || $mapping instanceof Mapping === false + ) { + isset($this->style) === true && $this->style->error("{$this->configuration['source']}, {$this->configuration['schema']} or {$this->configuration['mapping']} not found, ending syncOpenWooHandler"); + + return []; + }//end if + + $categorie = ''; + switch ($mapping->getReference()) { + case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': + $categorie = 'Woo verzoek'; + break; + case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': + $categorie = 'Convenant'; + break; + } + + isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}"); + $this->logger->info("Fetching objects from {$source->getLocation()}", ['plugin' => 'common-gateway/woo-bundle']); + + $results = $this->fetchObjects($source, 1, [], $categorie); + if (empty($results) === true) { + $this->logger->info('No results found, ending SyncOpenWooService', ['plugin' => 'common-gateway/woo-bundle']); + return $this->data; + } + + $this->entityManager->flush(); + + $customFields = [ + 'organisatie' => [ + 'oin' => $this->configuration['oin'], + 'naam' => $this->configuration['organisatie'], + ], + 'categorie' => $categorie, + ]; + + $idsSynced = []; + $responseItems = []; + $documents = []; + $hydrationService = new HydrationService($this->syncService, $this->entityManager); + foreach ($results as $result) { + try { + $result = array_merge($result, $customFields); + $mappedResult = $this->mappingService->mapping($mapping, $result); + // Map categories to prevent multiple variants of the same categorie. + $mappedResult = $this->mappingService->mapping($categorieMapping, $mappedResult); + + $validationErrors = $this->validationService->validateData($mappedResult, $schema, 'POST'); + if ($validationErrors !== null) { + $validationErrors = implode(', ', $validationErrors); + $this->logger->warning("SyncOpenWoo validation errors: $validationErrors", ['plugin' => 'common-gateway/woo-bundle']); + isset($this->style) === true && $this->style->warning("SyncOpenWoo validation errors: $validationErrors"); + continue; + } + + $object = $hydrationService->searchAndReplaceSynchronizations( + $mappedResult, + $source, + $schema, + true, + true + ); + + // Get all synced sourceIds. + if (empty($object->getSynchronizations()) === false && $object->getSynchronizations()[0]->getSourceId() !== null) { + $idsSynced[] = $object->getSynchronizations()[0]->getSourceId(); + } + + $this->entityManager->persist($object); + $this->cacheService->cacheObject($object); + $responseItems[] = $object; + + $renderedObject = $object->toArray(); + $documents = array_merge($documents, $renderedObject['bijlagen']); + if (isset($renderedObject['metadata']['verzoek']['informatieverzoek']) === true) { + $documents[] = $renderedObject['metadata']['verzoek']['informatieverzoek']; + } + + if (isset($renderedObject['verzoek']['besluit']) === true) { + $documents[] = $renderedObject['metadata']['verzoek']['besluit']; + } + } catch (Exception $exception) { + $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['UUID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle']); + continue; + }//end try + }//end foreach + + $this->entityManager->flush(); + + foreach ($documents as $document) { + $documentData['document'] = $document; + $documentData['source'] = $source->getReference(); + $this->gatewayOEService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); + } + + $deletedObjectsCount = $this->deleteNonExistingObjects($idsSynced, $source, $this->configuration['schema'], $categorie); + + $this->data['response'] = new Response(json_encode($responseItems), 200); + + $countItems = count($responseItems); + $logMessage = "Synchronized $countItems cases to woo objects for ".$source->getName()." and deleted $deletedObjectsCount objects"; + isset($this->style) === true && $this->style->success($logMessage); + $this->logger->info($logMessage, ['plugin' => 'common-gateway/woo-bundle']); + + return $this->data; + + }//end syncOpenWooHandler() + + + public function syncOpenWooDocumentHandler(array $data, array $config): array + { + $source = $this->resourceService->getSource($data['source'], 'common-gateway/woo-bundle'); + $document = $data['document']; + $endpoint = $this->resourceService->getEndpoint($config['endpoint'], 'common-gateway/woo-bundle'); + + if (substr($document['url'], 0, strlen($source->getLocation())) === $source->getLocation()) { + $path = substr($document['url'], strlen($source->getLocation())); + } else { + $this->logger->error('Url of document does not correspond with source'); + + return $data; + } + + $bijlageObject = $this->entityManager->getRepository('App:ObjectEntity')->find($document['_self']['id']); + if ($bijlageObject instanceof ObjectEntity === false) { + return $data; + } + + $value = $bijlageObject->getValueObject('url'); + + if ($value->getFiles()->count() > 0) { + $file = $value->getFiles()->first(); + } else { + $file = new File(); + } + + $response = $this->callService->call($source, $path); + + $file->setBase64(base64_encode($response->getBody())); + $file->setMimeType($response->getHeader('content-type')[0]); + $file->setSize($response->getHeader('content-length')[0]); + $file->setName(($document['titel'] ?? $document['url'])); + + $explodedFilename = explode('.', ($document['titel'] ?? $document['url'])); + $file->setExtension(end($explodedFilename)); + $file->setValue($value); + + $this->entityManager->persist($file); + + $bijlageObject->hydrate(['url' => $this->fileService->generateDownloadEndpoint($file->getId()->toString(), $endpoint), 'extension' => end($explodedFilename)]); + + $this->entityManager->persist($bijlageObject); + + $this->entityManager->flush(); + + $data['document'] = $bijlageObject->toArray(); + + return $data; + + }//end syncOpenWooDocumentHandler() + + +}//end class From bbc3170788308c6af44d3e54315a80fa405e71e9 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 1 Feb 2024 15:42:52 +0100 Subject: [PATCH 06/10] Update syncopenwoo-constructor, use actionservice for dispatching --- composer.json | 2 +- src/Service/SyncOpenWooService.php | 80 +++++------------------------- 2 files changed, 14 insertions(+), 68 deletions(-) diff --git a/composer.json b/composer.json index 9b7bf48..56f729d 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "license": "EUPL-1.2", "minimum-stability": "dev", "require": { - "php": ">=7.4", + "php": ">=8.2", "commongateway/corebundle": "^1.1.52 | dev-feature/GW-1596/Symfony54 | dev-feature/GW-1595/Symfony64" }, "require-dev": { diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index 7bf2544..04117d9 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -37,54 +37,9 @@ class SyncOpenWooService { /** - * @var GatewayResourceService - */ - private GatewayResourceService $resourceService; - - /** - * @var CallService - */ - private CallService $callService; - - /** - * @var SynchronizationService - */ - private SynchronizationService $syncService; - - /** - * @var MappingService - */ - private MappingService $mappingService; - - /** - * @var EntityManagerInterface - */ - private EntityManagerInterface $entityManager; - - /** - * @var SymfonyStyle|null - */ - private ?SymfonyStyle $style = null; - - /** - * @var LoggerInterface $logger. + * @var LoggerInterface $logger The plugin logger. */ private LoggerInterface $logger; - - /** - * @var ValidationService $validationService. - */ - private ValidationService $validationService; - - /** - * @var CacheService $cacheService. - */ - private CacheService $cacheService; - - private FileService $fileService; - - private ObjectEntityService $gatewayOEService; - /** * @var array */ @@ -96,6 +51,7 @@ class SyncOpenWooService private array $configuration; + /** * SyncOpenWooService constructor. * @@ -108,28 +64,18 @@ class SyncOpenWooService * @param FileService $fileService */ public function __construct( - GatewayResourceService $resourceService, - CallService $callService, - SynchronizationService $syncService, - EntityManagerInterface $entityManager, - MappingService $mappingService, - LoggerInterface $pluginLogger, - ValidationService $validationService, - CacheService $cacheService, - FileService $fileService, - ObjectEntityService $gatewayOEService + private readonly GatewayResourceService $resourceService, + private readonly CallService $callService, + private readonly SynchronizationService $syncService, + private readonly EntityManagerInterface $entityManager, + private readonly MappingService $mappingService, + LoggerInterface $pluginLogger, + private readonly ValidationService $validationService, + private readonly CacheService $cacheService, + private readonly FileService $fileService, + private ActionService $actionService ) { - $this->resourceService = $resourceService; - $this->callService = $callService; - $this->syncService = $syncService; - $this->entityManager = $entityManager; - $this->mappingService = $mappingService; $this->logger = $pluginLogger; - $this->validationService = $validationService; - $this->cacheService = $cacheService; - $this->fileService = $fileService; - $this->gatewayOEService = $gatewayOEService; - }//end __construct() @@ -357,7 +303,7 @@ public function syncOpenWooHandler(array $data, array $configuration): array foreach ($documents as $document) { $documentData['document'] = $document; $documentData['source'] = $source->getReference(); - $this->gatewayOEService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); + $this->actionService->dispatchEvent('commongateway.action.event', $documentData, 'woo.openwoo.document.created'); } $deletedObjectsCount = $this->deleteNonExistingObjects($idsSynced, $source, $this->configuration['schema'], $categorie); From 470f5265383b92520913a52e51bfa23c6a426edb Mon Sep 17 00:00:00 2001 From: GitHub Actions <> Date: Thu, 1 Feb 2024 14:43:25 +0000 Subject: [PATCH 07/10] Update src from PHP Codesniffer --- src/Service/SyncOpenWooService.php | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index d951af0..450900e 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -40,6 +40,7 @@ class SyncOpenWooService * @var LoggerInterface $logger The plugin logger. */ private LoggerInterface $logger; + /** * @var array */ @@ -51,7 +52,6 @@ class SyncOpenWooService private array $configuration; - /** * SyncOpenWooService constructor. * @@ -65,17 +65,18 @@ class SyncOpenWooService */ public function __construct( private readonly GatewayResourceService $resourceService, - private readonly CallService $callService, + private readonly CallService $callService, private readonly SynchronizationService $syncService, private readonly EntityManagerInterface $entityManager, - private readonly MappingService $mappingService, - LoggerInterface $pluginLogger, - private readonly ValidationService $validationService, - private readonly CacheService $cacheService, - private readonly FileService $fileService, - private ActionService $actionService + private readonly MappingService $mappingService, + LoggerInterface $pluginLogger, + private readonly ValidationService $validationService, + private readonly CacheService $cacheService, + private readonly FileService $fileService, + private ActionService $actionService ) { - $this->logger = $pluginLogger; + $this->logger = $pluginLogger; + }//end __construct() From 6091b876416e36c0157252da082af47c8774d5e4 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 1 Feb 2024 15:45:12 +0100 Subject: [PATCH 08/10] Add actionservice to use --- src/Service/SyncOpenWooService.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index d951af0..b45198d 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -8,6 +8,7 @@ use App\Entity\ObjectEntity; use App\Service\ObjectEntityService; use App\Service\SynchronizationService; +use CommonGateway\CoreBundle\Service\ActionService; use CommonGateway\CoreBundle\Service\CallService; use CommonGateway\CoreBundle\Service\GatewayResourceService; use CommonGateway\CoreBundle\Service\MappingService; From ac8fa840f99d8ac8c3e96986697a2434c8fc838d Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Mon, 26 Feb 2024 15:31:24 +0100 Subject: [PATCH 09/10] Fix on calling entity repository --- src/Service/SyncOpenWooService.php | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index 3c3ec46..0e986d2 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -15,7 +15,6 @@ use CommonGateway\CoreBundle\Service\HydrationService; use CommonGateway\CoreBundle\Service\ValidationService; use CommonGateway\CoreBundle\Service\CacheService; -use CommonGateway\WOOBundle\Service\FileService; use Doctrine\ORM\EntityManagerInterface; use Psr\Cache\CacheException; use Psr\Cache\InvalidArgumentException; @@ -112,7 +111,7 @@ public function setStyle(SymfonyStyle $style): self private function deleteNonExistingObjects(array $idsSynced, Source $source, string $schemaRef, string $categorie): int { // Get all existing sourceIds. - $source = $this->entityManager->find('App:Gateway', $source->getId()->toString()); + $source = $this->entityManager->find(Source::class, $source->getId()->toString()); $existingSourceIds = []; $existingObjects = []; foreach ($source->getSynchronizations() as $synchronization) { @@ -156,12 +155,12 @@ private function fetchObjects(Source $source, ?int $page=1, array $results=[], s $decodedResponse = $this->callService->decodeResponse($source, $response); switch ($categorie) { - case 'Woo verzoek': - $results = array_merge($results, $decodedResponse['WOOverzoeken']); - break; - case 'Convenant': - $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); - break; + case 'Woo verzoek': + $results = array_merge($results, $decodedResponse['WOOverzoeken']); + break; + case 'Convenant': + $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); + break; } // Pagination xxllnc. @@ -222,12 +221,12 @@ public function syncOpenWooHandler(array $data, array $configuration): array $categorie = ''; switch ($mapping->getReference()) { - case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': - $categorie = 'Woo verzoek'; - break; - case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': - $categorie = 'Convenant'; - break; + case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': + $categorie = 'Woo verzoek'; + break; + case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': + $categorie = 'Convenant'; + break; } isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}"); @@ -295,7 +294,7 @@ public function syncOpenWooHandler(array $data, array $configuration): array $documents[] = $renderedObject['metadata']['verzoek']['besluit']; } } catch (Exception $exception) { - $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['UUID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle']); + $this->logger->error("Something wen't wrong synchronizing sourceId: {$result['ID']} with error: {$exception->getMessage()}", ['plugin' => 'common-gateway/woo-bundle', 'trace' => $exception->getTraceAsString()]); continue; }//end try }//end foreach @@ -336,7 +335,7 @@ public function syncOpenWooDocumentHandler(array $data, array $config): array return $data; } - $bijlageObject = $this->entityManager->getRepository('App:ObjectEntity')->find($document['_self']['id']); + $bijlageObject = $this->entityManager->getRepository(ObjectEntity::class)->find($document['_self']['id']); if ($bijlageObject instanceof ObjectEntity === false) { return $data; } From 2ac90915dfdf8575ef13268c1de0453cb1fddcd0 Mon Sep 17 00:00:00 2001 From: GitHub Actions <> Date: Mon, 26 Feb 2024 14:31:52 +0000 Subject: [PATCH 10/10] Update src from PHP Codesniffer --- src/Service/SyncOpenWooService.php | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Service/SyncOpenWooService.php b/src/Service/SyncOpenWooService.php index 0e986d2..7b65a58 100644 --- a/src/Service/SyncOpenWooService.php +++ b/src/Service/SyncOpenWooService.php @@ -155,12 +155,12 @@ private function fetchObjects(Source $source, ?int $page=1, array $results=[], s $decodedResponse = $this->callService->decodeResponse($source, $response); switch ($categorie) { - case 'Woo verzoek': - $results = array_merge($results, $decodedResponse['WOOverzoeken']); - break; - case 'Convenant': - $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); - break; + case 'Woo verzoek': + $results = array_merge($results, $decodedResponse['WOOverzoeken']); + break; + case 'Convenant': + $results = array_merge($results, $decodedResponse['Convenantenverzoeken']); + break; } // Pagination xxllnc. @@ -221,12 +221,12 @@ public function syncOpenWooHandler(array $data, array $configuration): array $categorie = ''; switch ($mapping->getReference()) { - case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': - $categorie = 'Woo verzoek'; - break; - case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': - $categorie = 'Convenant'; - break; + case 'https://commongateway.nl/mapping/woo.openWooToWoo.mapping.json': + $categorie = 'Woo verzoek'; + break; + case 'https://commongateway.nl/mapping/woo.openConvenantToWoo.mapping.json': + $categorie = 'Convenant'; + break; } isset($this->style) === true && $this->style->info("Fetching objects from {$source->getLocation()}");