Merge remote-tracking branch 'origin/master' into public-beta

Author: jnm

kpi/deployment_backends/kobocat_backend.py

@@ -15,6 +15,7 @@
 import requests
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from lxml import etree
 from django.core.files import File
 from django.db.models.query import QuerySet
 from django.utils.translation import gettext_lazy as t
@@ -44,7 +45,7 @@
 from kpi.utils.log import logging
 from kpi.utils.mongo_helper import MongoHelper
 from kpi.utils.permissions import is_user_anonymous
-from kpi.utils.xml import edit_submission_xml, strip_nodes
+from kpi.utils.xml import edit_submission_xml
 from .base_backend import BaseDeploymentBackend
 from .kc_access.shadow_models import (
     KobocatOneTimeAuthToken,
@@ -152,7 +153,7 @@ def bulk_update_submissions(
         update_data = self.__prepare_bulk_update_data(data['data'])
         kc_responses = []
         for submission in submissions:
-            xml_parsed = ET.fromstring(submission)
+            xml_parsed = etree.fromstring(submission)
 
             _uuid, uuid_formatted = self.generate_new_instance_id()
 
@@ -166,7 +167,7 @@ def bulk_update_submissions(
             deprecated_id_or_new = (
                 deprecated_id
                 if deprecated_id is not None
-                else ET.SubElement(xml_parsed.find('meta'), 'deprecatedID')
+                else etree.SubElement(xml_parsed.find('meta'), 'deprecatedID')
             )
             deprecated_id_or_new.text = instance_id.text
             instance_id.text = uuid_formatted
@@ -181,7 +182,7 @@ def bulk_update_submissions(
 
             # TODO: Might be worth refactoring this as it is also used when
             # duplicating a submission
-            file_tuple = (_uuid, io.BytesIO(ET.tostring(xml_parsed)))
+            file_tuple = (_uuid, io.BytesIO(etree.tostring(xml_parsed)))
             files = {'xml_submission_file': file_tuple}
             # `POST` is required by OpenRosa spec https://docs.getodk.org/openrosa-form-submission
             headers = {}
@@ -666,12 +667,6 @@ def get_data_download_links(self):
             'reports',
             self.backend_response['id_string']
         ))
-        forms_base_url = '/'.join((
-            settings.KOBOCAT_URL.rstrip('/'),
-            self.asset.owner.username,
-            'forms',
-            self.backend_response['id_string']
-        ))
         links = {
             # To be displayed in iframes
             'xls_legacy': '/'.join((exports_base_url, 'xls/')),
@@ -700,7 +695,7 @@ def get_enketo_survey_links(self):
                 data=data
             )
             response.raise_for_status()
-        except requests.exceptions.RequestException as e:
+        except requests.exceptions.RequestException:
             # Don't 500 the entire asset view if Enketo is unreachable
             logging.error(
                 'Failed to retrieve links from Enketo', exc_info=True)

kpi/deployment_backends/mock_backend.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 import copy
 import os
+import re
 import time
 import uuid
 from datetime import datetime
@@ -12,6 +13,8 @@
 from dicttoxml import dicttoxml
 from django.conf import settings
 from django.urls import reverse
+from django.utils.translation import gettext as t
+from lxml import etree
 from rest_framework import status
 
 from kpi.constants import (
@@ -29,61 +32,86 @@
 )
 from kpi.interfaces.sync_backend_media import SyncBackendMediaInterface
 from kpi.models.asset_file import AssetFile
-from kpi.utils.mongo_helper import MongoHelper, drop_mock_only
 from kpi.tests.utils.mock import MockAttachment
+from kpi.utils.mongo_helper import MongoHelper, drop_mock_only
+from kpi.utils.xml import edit_submission_xml
 from .base_backend import BaseDeploymentBackend
+from ..exceptions import KobocatBulkUpdateSubmissionsClientException
 
 
 class MockDeploymentBackend(BaseDeploymentBackend):
     """
     Only used for unit testing and interface testing.
     """
 
+    PROTECTED_XML_FIELDS = [
+        '__version__',
+        'formhub',
+        'meta',
+    ]
+
     def bulk_assign_mapped_perms(self):
         pass
 
     def bulk_update_submissions(
         self, data: dict, user: 'auth.User'
     ) -> dict:
-
         submission_ids = self.validate_access_with_partial_perms(
             user=user,
             perm=PERM_CHANGE_SUBMISSIONS,
             submission_ids=data['submission_ids'],
             query=data['query'],
         )
 
-        if not submission_ids:
+        if submission_ids:
+            data['query'] = {}
+        else:
             submission_ids = data['submission_ids']
 
         submissions = self.get_submissions(
             user=user,
-            format_type=SUBMISSION_FORMAT_TYPE_JSON,
-            submission_ids=submission_ids
+            format_type=SUBMISSION_FORMAT_TYPE_XML,
+            submission_ids=submission_ids,
+            query=data['query'],
         )
 
-        submission_ids = [int(id_) for id_ in submission_ids]
+        if not self.current_submissions_count:
+            raise KobocatBulkUpdateSubmissionsClientException(
+                detail=t('No submissions match the given `submission_ids`')
+            )
 
-        responses = []
+        update_data = self.__prepare_bulk_update_data(data['data'])
+        kc_responses = []
         for submission in submissions:
-            if submission['_id'] in submission_ids:
-                _uuid = uuid.uuid4()
-                submission['meta/deprecatedID'] = submission['meta/instanceID']
-                submission['meta/instanceID'] = f'uuid:{_uuid}'
-                for k, v in data['data'].items():
-                    submission[k] = v
-
-                # Mirror KobocatDeploymentBackend responses
-                responses.append(
-                    {
-                        'uuid': _uuid,
-                        'status_code': status.HTTP_201_CREATED,
-                        'message': 'Successful submission'
-                    }
-                )
-
-        self.mock_submissions(submissions)
-        return self.__prepare_bulk_update_response(responses)
+            # Remove XML declaration from submission
+            submission = re.sub(r'(<\?.*\?>)', '', submission)
+            xml_parsed = etree.fromstring(submission)
+
+            _uuid, uuid_formatted = self.generate_new_instance_id()
+
+            instance_id = xml_parsed.find('meta/instanceID')
+            deprecated_id = xml_parsed.find('meta/deprecatedID')
+            deprecated_id_or_new = (
+                deprecated_id
+                if deprecated_id is not None
+                else etree.SubElement(xml_parsed.find('meta'), 'deprecatedID')
+            )
+            deprecated_id_or_new.text = instance_id.text
+            instance_id.text = uuid_formatted
+
+            for path, value in update_data.items():
+                edit_submission_xml(xml_parsed, path, value)
+
+            kc_responses.append(
+                {
+                    'uuid': _uuid,
+                    'status_code': status.HTTP_201_CREATED,
+                    'message': 'Successful submission',
+                    'updated_submission': etree.tostring(xml_parsed) # only for testing
+                }
+            )
+
+        return self.__prepare_bulk_update_response(kc_responses)
 
     def calculated_submission_count(self, user: 'auth.User', **kwargs) -> int:
         params = self.validate_submission_list_params(user,
@@ -529,6 +557,16 @@ def set_validation_statuses(self, user: 'auth.User', data: dict) -> dict:
             }
         }
 
+    @staticmethod
+    def generate_new_instance_id() -> (str, str):
+        """
+        Returns:
+            - Generated uuid
+            - Formatted uuid for OpenRosa xml
+        """
+        _uuid = str(uuid.uuid4())
+        return _uuid, f'uuid:{_uuid}'
+
     @property
     def submission_list_url(self):
         # This doesn't really need to be implemented.
@@ -545,6 +583,22 @@ def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA):
         for obj in queryset:
             assert issubclass(obj.__class__, SyncBackendMediaInterface)
 
+    @classmethod
+    def __prepare_bulk_update_data(cls, updates: dict) -> dict:
+        """
+        Preparing the request payload for bulk updating of submissions
+        """
+        # Sanitizing the payload of potentially destructive keys
+        sanitized_updates = copy.deepcopy(updates)
+        for key in updates:
+            if (
+                key in cls.PROTECTED_XML_FIELDS
+                or '/' in key and key.split('/')[0] in cls.PROTECTED_XML_FIELDS
+            ):
+                sanitized_updates.pop(key)
+
+        return sanitized_updates
+
     @staticmethod
     def __prepare_bulk_update_response(kc_responses: list) -> dict:
         total_update_attempts = len(kc_responses)

kpi/tests/api/v2/test_api_submissions.py

@@ -1385,7 +1385,8 @@ def setUp(self):
         self.updated_submission_data = {
             'submission_ids': [rs['_id'] for rs in random_submissions],
             'data': {
-                'q1': '🕺',
+                'q1': 'Updated value',
+                'q_new': 'A new question and value'
             },
         }